Split repo generation functionality out of blenderpack.py

Moved repo.json generation to make_repo.py
Package/addon parsing is getting a bit messy, this can be
cleaned up when we have a clearer idea of what a package is.

For now just make it work.

blenderpack.py

@@ -60,94 +60,6 @@ def fetch(url, pipe):
 #     def json(self):
 #         return json.dumps(self.__dict__)
 
-def parse_blinfo(source: str) -> dict:
-    """Parse a python file and return its bl_info dict, if there is one (else return None)"""
-
-    tree = ast.parse(source)
-
-    for body in tree.body:
-        if body.__class__ != ast.Assign:
-            continue
-        if len(body.targets) != 1:
-            continue
-        if getattr(body.targets[0], 'id', '') != 'bl_info':
-            continue
-
-        return ast.literal_eval(body.value)
-
-    raise BadAddon('No bl_info found')
-
-
-def extract_blinfo(path: pathlib.Path) -> dict:
-    """Extract bl_info dict from addon at path (can be single file, module, or zip)"""
-
-    source = None
-    # get last component of path
-    addon_name = path.name
-
-    if path.is_dir():
-        with open(path / '__init__.py', 'r') as f:
-            source = f.read()
-    else:
-
-        # HACK: perhaps not the best approach determining filetype..?
-        try:
-            with zipfile.ZipFile(str(path), 'r') as z:
-                for fname in z.namelist():
-                    # HACK: this seems potentially fragile; depends on zipfile listing root contents first
-                    if fname.endswith('__init__.py'):
-                        source = z.read(fname)
-                        break
-        except zipfile.BadZipFile:
-            with path.open() as f:
-                source = f.read()
-
-    if source == None:
-        raise RuntimeError("Could not read addon '%s'" % addon_name)
-
-    return parse_blinfo(source)
-
-
-
-def make_repo(repopath: pathlib.Path):
-    """Make repo.json for files in directory 'repopath'"""
-
-    repo_data = {}
-    package_data = []
-
-    if not repopath.is_dir():
-        raise FileNotFoundError(repopath)
-
-    for addon_path in repopath.iterdir():
-        package_datum = {}
-        addon = addon_path.name
-
-        try:
-            bl_info = extract_blinfo(addon_path)
-        except BadAddon as err:
-            log.warning('Could not extract bl_info from {}: {}'.format(addon_path, err))
-            continue
-
-        if not REQUIRED_KEYS.issubset(set(bl_info)):
-            log.warning(
-                    "Required key(s) '{}' not found in bl_info of '{}'".format(
-                        "', '".join(REQUIRED_KEYS.difference(set(bl_info))), addon)
-                    )
-
-        package_datum['bl_info'] = bl_info
-        package_datum['type'] = 'addon'
-        package_data.append(package_datum)
-
-    repo_data['packages'] = package_data
-
-    log.info("Repository generation successful")
-    cwd = pathlib.Path.cwd()
-    dump_repo(cwd, repo_data)
-    log.info("repo.json written to %s" % cwd)
-
-def dump_repo(repo_path: pathlib.Path, repo_data: dict):
-    with (repo_path / 'repo.json').open('w', encoding='utf-8') as repo_file:
-        json.dump(repo_data, repo_file, indent=4, sort_keys=True)
 
 
 def main():

make_repo.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+import argparse
+import zipfile
+import ast
+import json
+import logging
+from utils import *
+
+log = logging.getLogger(__name__)
+
+REQUIRED_KEYS = set(['name', 'blender', 'version'])
+SCHEMA_VERSION = 1
+
+class BadAddon(Exception):
+    pass
+
+def iter_addons(path: Path) -> (Path, dict):
+    """
+    Generator, yields (path, bl_info) of blender addons in `path` (non recursive).
+    """
+    pass
+    # for item in addons_dir.iterdir():
+    #     base = item.name
+    #
+    #         yield (base, fname, '.zip')
+    #     else:
+    # yield (base, item.path, '.py') 
+
+def parse_blinfo(source: str) -> dict:
+    """Parse a python file and return its bl_info dict, if there is one (else return None)"""
+
+    try:
+        tree = ast.parse(source)
+    except SyntaxError as e:
+        raise BadAddon("Syntax error") from e
+
+    for body in tree.body:
+        if body.__class__ != ast.Assign:
+            continue
+        if len(body.targets) != 1:
+            continue
+        if getattr(body.targets[0], 'id', '') != 'bl_info':
+            continue
+
+        return ast.literal_eval(body.value)
+
+    raise BadAddon('No bl_info found')
+
+
+def extract_blinfo(item: Path) -> dict:
+    """
+    Extract bl_info dict from addon at path (can be single file, python package, or zip)
+    """
+
+    blinfo = None
+    addon_name = item.name
+    
+    if not item.exists():
+        raise FileNotFoundError("Cannot extract blinfo from '%s'; no such file or directory" % item)
+
+    if item.is_dir():
+        fname = item / '__init__.py'
+        try:
+            with fname.open("r") as f:
+                blinfo = parse_blinfo(f.read())
+        except FileNotFoundError as err:
+            # directory with no __init__.py: not an addon
+            raise BadAddon("Directory '%s' doesn't contain __init__.py; not a python package" % item) from err
+    
+    elif item.is_file():
+        try:
+            with zipfile.ZipFile(str(item), 'r') as z:
+                if len(z.namelist()) == 1:
+                    # zipfile with one item: just read that item
+                    blinfo = parse_blinfo(z.read(z.namelist()[0]))
+                else:
+                    # zipfile with multiple items: try all __init__.py files
+                    for fname in z.namelist():
+                        # TODO: zips with multiple bl_infos might be a problem,
+                        # not sure how such cases should be handled (if at all)
+                        if fname.endswith('__init__.py'):
+                            try:
+                                blinfo = parse_blinfo(z.read(fname))
+                                break
+                            except BadAddon:
+                                continue
+                    raise BadAddon("Zipfile '%s' doesn't contain any readable bl_info dict" % item)
+
+        except zipfile.BadZipFile:
+            # Assume file is 
+            with item.open() as f:
+                blinfo = parse_blinfo(f.read())
+
+    # This should not happen
+    if blinfo == None:
+        raise RuntimeError("Could not read addon '%s'" % addon_name)
+
+    return blinfo
+
+
+
+def make_repo(repopath: Path):
+    """Make repo.json for files in directory 'repopath'"""
+
+    repo_data = {}
+    package_data = []
+
+    if not repopath.is_dir():
+        raise FileNotFoundError(repopath)
+
+    for addon_path in repopath.iterdir():
+        package_datum = {}
+        addon = addon_path.name
+
+        try:
+            bl_info = extract_blinfo(addon_path)
+        except BadAddon as err:
+            log.warning('Could not extract bl_info from {}: {}'.format(addon_path, err))
+            continue
+
+        if not REQUIRED_KEYS.issubset(set(bl_info)):
+            log.warning(
+                    "Required key(s) '{}' not found in bl_info of '{}'".format(
+                        "', '".join(REQUIRED_KEYS.difference(set(bl_info))), addon)
+                    )
+
+        package_datum['bl_info'] = bl_info
+        package_datum['type'] = 'addon'
+        package_data.append(package_datum)
+
+    repo_data['packages'] = package_data
+
+    log.info("Repository generation successful")
+    cwd = Path.cwd()
+    dump_repo(cwd, repo_data)
+    log.info("repo.json written to %s" % cwd)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Generate a blender package repository from a directory of addons')
+
+    parser.add_argument('-v', '--verbose',
+            help="Increase verbosity (can be used multiple times)",
+            action="count",
+            default=0)
+    parser.add_argument('path',
+            type=Path,
+            nargs='?',
+            default=Path.cwd(),
+            help="Path to addon directory")
+
+    args = parser.parse_args()
+    log.level = args.verbose
+    logging.basicConfig(level=logging.INFO,
+            format='%(levelname)8s: %(message)s')
+
+    make_repo(args.path)
+
+if __name__ == '__main__':
+    main()

tests/test_helpers/addons/dir_invalid_addon/__init__.py

@@ -0,0 +1,12 @@
+bl_info = {
+    "author": "testscreenings, PKHG, TrumanBlending",
+    "version": (0, 1, 2),
+    "blender": (2, 59, 0),
+    "location": "View3D > Add > Curve",
+    "description": "Adds generated ivy to a mesh object starting "
+                   "at the 3D cursor",
+    "warning": "",
+    "wiki_url": "https://wiki.blender.org/index.php/Extensions:2.6/Py/"
+                "Scripts/Curve/Ivy_Gen",
+    "category": "Add Curve",
+}

tests/test_helpers/addons/singlefile_addon.py

@@ -18,21 +18,36 @@
 
 # <pep8-80 compliant>
 
+# bl_info = {
+#     "name": "IvyGen",
+#     "author": "testscreenings, PKHG, TrumanBlending",
+#     "version": (0, 1, 2),
+#     "blender": (2, 59, 0),
+#     "location": "View3D > Add > Curve",
+#     "description": "Adds generated ivy to a mesh object starting "
+#                    "at the 3D cursor",
+#     "warning": "",
+#     "wiki_url": "https://wiki.blender.org/index.php/Extensions:2.6/Py/"
+#                 "Scripts/Curve/Ivy_Gen",
+#     "category": "Add Curve",
+# }
+
+# just use one blinfo for all addons to simplify testing
 bl_info = {
-    "name": "IvyGen",
+    "name": "Extra Objects",
-    "author": "testscreenings, PKHG, TrumanBlending",
+    "author": "Multiple Authors",
     "version": (0, 1, 2),
-    "blender": (2, 59, 0),
+    "blender": (2, 76, 0),
-    "location": "View3D > Add > Curve",
+    "location": "View3D > Add > Curve > Extra Objects",
-    "description": "Adds generated ivy to a mesh object starting "
+    "description": "Add extra curve object types",
-                   "at the 3D cursor",
     "warning": "",
     "wiki_url": "https://wiki.blender.org/index.php/Extensions:2.6/Py/"
-                "Scripts/Curve/Ivy_Gen",
+                "Scripts/Curve/Curve_Objects",
-    "category": "Add Curve",
+    "category": "Add Curve"
 }
 
 
+
 import bpy
 from bpy.props import (
         FloatProperty,

tests/test_helpers/dir_addon_output

@@ -1 +0,0 @@
-{'name': 'Extra Objects', 'author': 'Multiple Authors', 'version': (0, 1, 2), 'blender': (2, 76, 0), 'location': 'View3D > Add > Curve > Extra Objects', 'description': 'Add extra curve object types', 'warning': '', 'wiki_url': 'https://wiki.blender.org/index.php/Extensions:2.6/Py/Scripts/Curve/Curve_Objects', 'category': 'Add Curve'}

tests/test_helpers/expected_blinfo

@@ -0,0 +1,13 @@
+{
+    "name": "Extra Objects",
+    "author": "Multiple Authors",
+    "version": (0, 1, 2),
+    "blender": (2, 76, 0),
+    "location": "View3D > Add > Curve > Extra Objects",
+    "description": "Add extra curve object types",
+    "warning": "",
+    "wiki_url": "https://wiki.blender.org/index.php/Extensions:2.6/Py/"
+                "Scripts/Curve/Curve_Objects",
+    "category": "Add Curve"
+ }
+

tests/test_helpers/real_addon.py_output

@@ -1 +0,0 @@
-{'name': 'IvyGen', 'author': 'testscreenings, PKHG, TrumanBlending', 'version': (0, 1, 2), 'blender': (2, 59, 0), 'location': 'View3D > Add > Curve', 'description': 'Adds generated ivy to a mesh object starting at the 3D cursor', 'warning': '', 'wiki_url': 'https://wiki.blender.org/index.php/Extensions:2.6/Py/Scripts/Curve/Ivy_Gen', 'category': 'Add Curve'}

tests/test_helpers/zipped_addon.zip_output

@@ -1 +0,0 @@
-{'name': 'Extra Objects', 'author': 'Multiple Authors', 'version': (0, 1, 2), 'blender': (2, 76, 0), 'location': 'View3D > Add > Curve > Extra Objects', 'description': 'Add extra curve object types', 'warning': '', 'wiki_url': 'https://wiki.blender.org/index.php/Extensions:2.6/Py/Scripts/Curve/Curve_Objects', 'category': 'Add Curve'}

tests/test_make_repo.py

@@ -2,11 +2,15 @@
 
 import unittest
 from pathlib import Path
-import os
+import logging
+import ast
 import json
-import blenderpack
+import make_repo
 
-class test_blenderpack_make_repo(unittest.TestCase):
+logging.basicConfig(level=logging.DEBUG,
+    format='%(levelname)8s: %(message)s')
+
+class test_make_repo(unittest.TestCase):
 
     helper_path = Path('tests', 'test_helpers')
     addon_path = helper_path / 'addons'
@@ -15,20 +19,12 @@ class test_blenderpack_make_repo(unittest.TestCase):
         test_file = 'file_that_doesnt_exist'
         self.assertRaises(
                 FileNotFoundError,
-                blenderpack.extract_blinfo,
+                make_repo.extract_blinfo,
-                self.addon_path / test_file
-                )
-
-    def test_extract_blinfo_from_nonaddon(self):
-        test_file = 'not_an_addon.py'
-        self.assertRaises(
-                blenderpack.BadAddon,
-                blenderpack.extract_blinfo,
                 self.addon_path / test_file
                 )
 
     def test_make_repo_valid(self):
-        blenderpack.make_repo(self.helper_path / 'addons')
+        make_repo.make_repo(self.helper_path / 'addons')
         repojson = Path.cwd() / 'repo.json'
 
         try:
@@ -40,26 +36,46 @@ class test_blenderpack_make_repo(unittest.TestCase):
         self.fail('unfinished test')
 
     def test_make_repo_from_nonexistent(self):
-        blenderpack.make_repo(self.helper_path / 'addons')
+        make_repo.make_repo(self.helper_path / 'addons')
         self.fail('unfinished test')
 
+# addons which should contain bl_infos
+yes_blinfo = [
+        f for f in test_make_repo.addon_path.iterdir()
+        if not f.match('*nonaddon*') and not f.match('*invalid_addon*')
+        ]
+# addons which should throw BadAddon because they have no blinfo
+no_blinfo = [
+        f for f in test_make_repo.addon_path.iterdir()
+        if f.match('*nonaddon*')
+        ]
 
-# testname: filename
+def generate_good_blinfo_test(test_file: Path):
-bl_info_tests = {
-    'test_extract_blinfo_from_file': 'real_addon.py',
-    'test_extract_blinfo_from_zip': 'zipped_addon.zip',
-    'test_extract_blinfo_from_dir': 'dir_addon',
-}
-
-def generate_test(test_file):
     def test(self):
-        reality = str(blenderpack.extract_blinfo(self.addon_path / test_file))
+        reality = make_repo.extract_blinfo(test_file)
-        with (self.helper_path / (test_file + '_output')).open() as f:
+        with (self.helper_path / 'expected_blinfo').open("r") as f:
-            expectation = f.read()
+            expectation = ast.literal_eval(f.read())
         self.assertEqual(expectation, reality)
     return test
 
-for name, param in bl_info_tests.items():
+def generate_bad_blinfo_test(test_file: Path):
-    test_func = generate_test(param)
+    def test(self):
-    setattr(test_blenderpack_make_repo, 'test_{}'.format(name), test_func)
+        self.assertRaises(
+                make_repo.BadAddon,
+                make_repo.extract_blinfo,
+                test_file
+                )
+    return test
+
+# Add test method retur
+def add_generated_tests(test_generator, params, destclass):
+    """
+    Add a test method (as returned by 'test_generator') to destclass for every param
+    """
+    for param in params:
+        test_func = test_generator(param)
+        setattr(destclass, 'test_{}'.format(param), test_func)
+
+add_generated_tests(generate_good_blinfo_test, yes_blinfo, test_make_repo)
+add_generated_tests(generate_bad_blinfo_test, no_blinfo, test_make_repo)
 

utils.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+import json
+
+def dump_repo(repo_path: Path, repo_data: dict):
+    with (repo_path / 'repo.json').open('w', encoding='utf-8') as repo_file:
+        json.dump(repo_data, repo_file, indent=4, sort_keys=True)