Store filelists in generated repositories

bpkg-repogen

@@ -6,7 +6,6 @@ import zipfile
 import ast
 import json
 import logging
-from utils import *
 
 log = logging.getLogger(__name__)
 
@@ -16,15 +15,15 @@ SCHEMA_VERSION = 1
 class BadAddon(Exception):
     pass
 
-def iter_addons(path: Path) -> (Path, dict):
+def iter_addons(path: Path) -> (Path, dict, list):
     """
-    Generator, yields (path, bl_info) of blender addons in `path`.
+    Generator, yields (path, bl_info, filelist) of blender addons in `path`.
     """
     for item in path.iterdir():
         try:
-            yield (item, extract_blinfo(item))
+            yield (item, extract_blinfo(item), get_filelist(item))
         except BadAddon as err:
-            log.debug("Skipping '{}': {}".format(item.name, err))
+            log.info("Skipping '{}': {}".format(item.name, err))
 
 def parse_blinfo(source: str) -> dict:
     """Parse a python file and return its bl_info dict, if there is one (else return None)"""
@@ -46,6 +45,41 @@ def parse_blinfo(source: str) -> dict:
 
     raise BadAddon('No bl_info found')
 
+def filelist_from_zip(zippath: Path) -> list:
+    """Return list of files in the root of a zipfile"""
+    rootlist = []
+    with zipfile.ZipFile(str(zippath), 'r') as z:
+        for f in z.namelist():
+            # Get all names which have no path separators (root level files)
+            # or have a single path separator at the end (root level directories).
+            if len(f.rstrip('/').split('/')) == 1:
+                rootlist.append(f)
+
+    return rootlist
+
+def get_filelist(addon_path: Path) -> list:
+    """
+    Extract filelist from addon at path (can be single file, python package, or zip)
+    """
+
+    if not addon_path.exists():
+        raise FileNotFoundError("Cannot extract blinfo from '%s'; no such file or directory" % addon_path)
+
+    if addon_path.is_dir():
+        return [str(addon_path)]
+    
+    if addon_path.is_file():
+        ext = addon_path.suffix.lower()
+
+        if ext == '.zip':
+            return filelist_from_zip(addon_path)
+        elif ext == '.py':
+            return [str(addon_path)]
+        else:
+            raise BadAddon("File '%s' doesn't have a .zip or .py extension; not an addon" % addon_path)
+
+    # This should not happen
+    raise RuntimeError("Could not read addon '%s'" % addon_path.name)
 
 def blinfo_from_zip(item: Path) -> dict:
     try:
@@ -115,7 +149,8 @@ class Package:
     def __init__(self, path: Path, bl_info: dict):
         self.bl_info = bl_info
         self.path    = path
-        self.url     = None
+        self.url     = ""
+        self.files   = []
 
     def to_dict(self) -> dict:
         """
@@ -124,8 +159,14 @@ class Package:
         return {
                 'bl_info': self.bl_info,
                 'url': self.url,
+                'files': self.files,
                 }
 
+    # def raise_for_missing(self):
+    #     """Ensure all fields are set"""
+    #     if len(self.files) == 0:
+    #         log.warning("Filelist for %s is empty", self.bl_info['name'])
+
 class Repository:
     """
     Stores repository metadata (including a list of packages)
@@ -171,17 +212,28 @@ def make_repo(path: Path, name: str, baseurl: str) -> Repository:
     if not path.is_dir():
         raise FileNotFoundError(path)
 
-    for addon, bl_info in iter_addons(path):
+    for addon_path, bl_info, filelist in iter_addons(path):
 
         # Check if we have all bl_info fields we want
         if not REQUIRED_KEYS.issubset(set(bl_info)):
             log.warning(
-                    "Required key(s) '{}' not found in bl_info of '{}'".format(
-                        "', '".join(REQUIRED_KEYS.difference(set(bl_info))), addon)
+                    "Skipping '{}': Required key(s) '{}' not found in bl_info".format(
+                        addon_path,
+                        "', '".join(REQUIRED_KEYS.difference(set(bl_info)))
+                        )
                     )
+            continue
 
-        package = Package(addon, bl_info)
+        if addon_path.is_dir():
+            log.warning(
+                    "Skipping '{}': Addon not zipped".format(addon_path)
+                    )
+            continue
+
+        package = Package(addon_path, bl_info)
         package.url = baseurl + package.path.name
+        package.files = filelist
+
         repo.add_package(package)
 
     log.info("Repository generation successful")
@@ -210,7 +262,7 @@ def main():
 
     args = parser.parse_args()
 
-    logging.basicConfig(format='%(levelname)8s: %(message)s', level=logging.INFO)
+    logging.basicConfig(format='%(levelname)8s: %(message)s', level=logging.WARNING)
     log.level += args.verbose
 
     if args.name is None: