Include installed packages in listing

bpkg/__init__.py

@@ -18,11 +18,13 @@ if 'bpy' in locals():
     import importlib
 
     subproc = importlib.reload(subproc)
+    Package = subproc.Package
 else:
     from . import subproc
+    from .subproc import Package
 
 import bpy
-
+from collections import OrderedDict
 
 class SubprocMixin:
     """Mix-in class for things that need to be run in a subprocess."""
@@ -218,6 +220,7 @@ class BPKG_OT_install(SubprocMixin, bpy.types.Operator):
 
     def _subproc_success(self, success: subproc.Success):
         self.report({'INFO'}, 'Package installed successfully')
+        getattr(bpy.ops, __package__).refresh_packages()
         self.quit()
 
     def _subproc_aborted(self, aborted: subproc.Aborted):
@@ -232,10 +235,34 @@ class BPKG_OT_install(SubprocMixin, bpy.types.Operator):
             self.log.error('Process died without telling us! Exit code was 0 though')
             self.report({'WARNING'}, 'Error downloading package, but process finished OK. This is weird.')
 
-class BPKG_OT_refresh(SubprocMixin, bpy.types.Operator):
+def get_packages_from_disk(refresh=False) -> list:
-    bl_idname = "bpkg.refresh"
+    """Get list of packages installed on disk"""
+    import addon_utils
+    return [Package.from_module(mod) for mod in addon_utils.modules(refresh=refresh)]
+
+def get_packages_from_repo() -> list:
+    """Get list of packages from cached repository lists (does not refresh them from server)"""
+    import pathlib
+    storage_path = pathlib.Path(bpy.utils.user_resource('CONFIG', 'packages', create=True))
+    return subproc._load_repo(storage_path).packages
+
+class BPKG_OT_refresh_packages(bpy.types.Operator):
+    bl_idname = "bpkg.refresh_packages"
     bl_label = "Refresh Packages"
-    bl_description = 'Check for new and updated packages'
+    bl_description = "Scan for packages on disk"
+
+    log = logging.getLogger(__name__ + ".BPKG_OT_refresh_packages")
+
+    def execute(self, context):
+        installed_packages = get_packages_from_disk()
+        USERPREF_PT_packages.all_packages = combine_packagelists(installed_packages, USERPREF_PT_packages.available_packages)
+
+        return {'FINISHED'}
+
+class BPKG_OT_refresh_repositories(SubprocMixin, bpy.types.Operator):
+    bl_idname = "bpkg.refresh_repositories"
+    bl_label = "Refresh Repositories"
+    bl_description = 'Check repositories for new and updated packages'
     bl_options = {'REGISTER'}
 
     log = logging.getLogger(__name__ + ".BPKG_OT_refresh")
@@ -250,6 +277,7 @@ class BPKG_OT_refresh(SubprocMixin, bpy.types.Operator):
 
         import multiprocessing
 
+        #TODO: make sure all possible messages are handled
         self.msg_handlers = {
             subproc.Progress: self._subproc_progress,
             subproc.SubprocError: self._subproc_error,
@@ -290,13 +318,11 @@ class BPKG_OT_refresh(SubprocMixin, bpy.types.Operator):
         self.quit()
 
     def _subproc_success(self, success: subproc.Success):
-        self.report({'INFO'}, 'Package list retrieved successfully')
         self.quit()
 
     def _subproc_repository_result(self, result: subproc.RepositoryResult):
-        bpy.context.window_manager['package_repo'] = result.repository
+        USERPREF_PT_packages.available_packages = result.repository['packages']
         self.report({'INFO'}, 'Package list retrieved successfully')
-        self.quit()
 
     def _subproc_aborted(self, aborted: subproc.Aborted):
         self.report({'ERROR'}, 'Package list retrieval aborted per your request')
@@ -310,6 +336,19 @@ class BPKG_OT_refresh(SubprocMixin, bpy.types.Operator):
             self.log.error('Process died without telling us! Exit code was 0 though')
             self.report({'WARNING'}, 'Error refreshing package lists, but process finished OK. This is weird.')
 
+#TODO:
+# monkey patch refresh_repositories and add refresh_packages in the success callback
+# this way refresh_packages is always called after repositories have been refreshed
+class BPKG_OT_refresh(bpy.types.Operator):
+    bl_idname = "bpkg.refresh"
+    bl_label = "Refresh"
+    bl_description = "Check for new and updated packages"
+
+    def execute(self, context):
+        getattr(bpy.ops, __package__).refresh_repositories()
+        getattr(bpy.ops, __package__).refresh_packages()
+        return {'FINISHED'}
+
 
 class BPKG_OT_hang(SubprocMixin, bpy.types.Operator):
     bl_idname = 'bpkg.hang'
@@ -405,6 +444,11 @@ class USERPREF_PT_packages(bpy.types.Panel):
 
     log = logging.getLogger(__name__ + '.USERPREF_PT_packages')
 
+    all_packages = OrderedDict()
+    available_packages = []
+    installed_packages = []
+    displayed_packages = []
+
     @classmethod
     def poll(cls, context):
         userpref = context.user_preferences
@@ -428,8 +472,8 @@ class USERPREF_PT_packages(bpy.types.Panel):
         spl_r = spl.row()
         spl_r.prop(wm, "package_install_filter", expand=True)
 
-        def filtered(filters: dict, packages: list) -> list:
+        def filtered_packages(filters: dict, packages: OrderedDict) -> list:
-            """Returns filtered and sorted list of packages which match filters defined in dict"""
+            """Returns filtered and sorted list of names of packages which match filters"""
 
             #TODO: using lower() for case-insensitive comparison doesn't work in some languages
             def match_contains(blinfo) -> bool:
@@ -456,27 +500,27 @@ class USERPREF_PT_packages(bpy.types.Panel):
             contains = []
             startswith = []
 
-            for pkg in packages:
+            for pkgname, pkg in packages.items():
-                blinfo = pkg['bl_info']
+                blinfo = pkg.versions[0].bl_info
                 if match_category(blinfo):
                     if len(filters['search']) == 0:
-                        startswith.append(pkg)
+                        startswith.append(pkgname)
                         continue
                     if match_startswith(blinfo):
-                        startswith.append(pkg)
+                        startswith.append(pkgname)
                         continue
                     if match_contains(blinfo):
-                        contains.append(pkg)
+                        contains.append(pkgname)
                         continue
 
             return startswith + contains
 
-        def draw_package(pkg, layout):# {{{
+        def draw_package(pkg: ViewPackage, layout: bpy.types.UILayout):# {{{
             """Draws the given package"""
             pkgbox = layout.box()
             spl = pkgbox.split(.8)
             left = spl.row(align=True)
-            blinfo = pkg['bl_info']
+            blinfo = pkg.versions[0].bl_info
 
             # for install/uninstall buttons
             right = spl.row()
@@ -486,9 +530,9 @@ class USERPREF_PT_packages(bpy.types.Panel):
             # for collapse/expand button
             left.operator(
                     WM_OT_package_toggle_expand.bl_idname,
-                    icon='TRIA_DOWN' if pkg.get('expand') else 'TRIA_RIGHT',
+                    icon='TRIA_DOWN' if pkg.expanded else 'TRIA_RIGHT',
                     emboss=False,
-                    ).package_id=pkg['id']
+                    ).package_name=blinfo['name']
 
             # for metadata
             leftcol = left.column(align=True)
@@ -501,9 +545,9 @@ class USERPREF_PT_packages(bpy.types.Panel):
                 lr2.label(text=blinfo.get('description', ""))
                 lr2.enabled = False #Give name more visual weight
 
-            if pkg.get('url'):
+            if pkg.versions[0].url:
                 right.operator(BPKG_OT_install.bl_idname,
-                    text="Install").package_url=pkg.get('url')
+                    text="Install").package_url=pkg.versions[0].url
 
             def expanded():
                 row1 = leftcol.row()
@@ -541,7 +585,7 @@ class USERPREF_PT_packages(bpy.types.Panel):
                         spl.label("{}:".format(prop.title()))
                         spl.label(str(blinfo[prop]))
 
-            if pkg.get('expand'):
+            if pkg.expanded:
                 expanded()
             else:
                 collapsed()# }}}
@@ -553,59 +597,76 @@ class USERPREF_PT_packages(bpy.types.Panel):
             row.alignment='CENTER'
             row.scale_y = 10
 
-        try:
+        if len(USERPREF_PT_packages.all_packages) == 0:
-            repo = wm['package_repo']
+            center_message(pkgzone, "No packages found.")
-        except KeyError:
-            center_message(pkgzone, "Loading Repositories...")
 
-            import pathlib
+            # TODO: read repository and installed packages synchronously for now;
-            # TODO: read repository synchronously for now; can't run an operator to do async monitoring from draw code
+            # can't run an operator from draw code to do async monitoring
-            storage_path = pathlib.Path(bpy.utils.user_resource('CONFIG', 'packages', create=True))
+            installed_packages = get_packages_from_disk()
             try:
-                res = subproc._load_repo(storage_path)
+                available_packages = get_packages_from_repo()
-                wm['package_repo'] = res.to_dict(sort=True, ids=True)
             except FileNotFoundError:
-                wm['package_repo'] = None
+                center_message(pkgzone, "No repositories found")
-            return
+                return
+
+            all_packages = combine_packagelists(installed_packages, available_packages)
+            if len(all_packages) == 0:
+                center_message(pkgzone, "No packages found")
+                return
+
+            USERPREF_PT_packages.all_packages = all_packages
 
-        if repo is None:
-            center_message(pkgzone, "No repositories found.")
-            return
 
         filters = {
                 'category': bpy.context.window_manager.addon_filter,
                 'search': bpy.context.window_manager.package_search,
                 }
-        filtered_packages = filtered(filters, repo['packages'])
+        USERPREF_PT_packages.displayed_packages = filtered_packages(filters, USERPREF_PT_packages.all_packages)
 
-        for pkg in filtered_packages:
+        for pkgname in USERPREF_PT_packages.displayed_packages:
             row = pkgzone.row()
-            draw_package(pkg, row)
+            draw_package(USERPREF_PT_packages.all_packages[pkgname], row)
 
+class ViewPackage:
+    """
+    Stores a grouping of different versions of the same packages,
+    and view-specific data used for drawing
+    """
+    def __init__(self, pkg=None):
+        self.versions = []
+        self.expanded = False
+        self.installed = False
+
+        if pkg is not None:
+            self.add_version(pkg)
+
+    def add_version(self, pkg: Package):
+        self.versions.append(pkg)
+
+    def __iter__(self):
+        return (pkg for pkg in self.versions)
 
 class WM_OT_package_toggle_expand(bpy.types.Operator):
     bl_idname = "wm.package_toggle_expand"
     bl_label = ""
-    bl_description = "Toggle display of all information for given package"
+    bl_description = "Toggle display of extended information for given package"
     bl_options = {'INTERNAL'}
 
     log = logging.getLogger(__name__ + ".WM_OT_package_toggle_expand")
 
-    package_id = bpy.props.StringProperty(
+    package_name = bpy.props.StringProperty(
-            name="Package ID",
+            name="Package Name",
-            description="ID of package to expand/shrink",
+            description="Name of package to expand/collapse",
             )
 
     def execute(self, context):
-        repo = context.window_manager.get('package_repo')
+        try:
-
+            pkg = USERPREF_PT_packages.all_packages[self.package_name]
-        if not repo:
+        except KeyError:
+            log.error("Couldn't find package '%s'", self.package_name)
             return {'CANCELLED'}
         
-        for pkg in repo['packages']:
+        pkg.expanded = not pkg.expanded
-            if pkg.get('id') == self.package_id:
-                # if pkg['expand'] is unset, it's not expanded
-                pkg['expand'] = not pkg.get('expand', False)
 
         return {'FINISHED'}
 
@@ -629,9 +690,50 @@ class PackageManagerPreferences(bpy.types.AddonPreferences):
         temp_box.prop(self, 'repository_url')
         temp_box.operator(BPKG_OT_refresh.bl_idname)
 
+def validate_packagelist(pkglist: list) -> list:
+    """Ensures all packages have required fields; strips out bad packages and returns them in a list"""
+    pass
+
+def combine_packagelists(installed: list, available: list) -> OrderedDict:
+    """Merge list of installed and available packages into one dict, keyed by package name"""
+
+    log = logging.getLogger(__name__ + ".combine_packagelists")
+
+    masterlist = {}
+
+    def packages_are_equivilent(pkg1: Package, pkg2: Package) -> bool:
+        """Check that packages are the same version and provide the same files"""
+        blinfo1 = pkg1.bl_info
+        blinfo2 = pkg2.bl_info
+        return  blinfo1['version'] == blinfo2['version']\
+            and blinfo1['files']   == blinfo2['files']
+
+    for pkg in available:
+        pkgname = pkg.bl_info['name']
+        if pkgname in masterlist:
+            masterlist[pkgname].add_version(pkg)
+        else:
+            masterlist[pkgname] = ViewPackage(pkg)
+
+    for pkg in installed:
+        pkgname = pkg.bl_info['name']
+        if pkgname in masterlist:
+            for masterpkg in masterlist[pkgname]:
+                if packages_are_equivilent(pkg, masterpkg):
+                    masterpkg.installed = True
+                    break
+            else:
+                pkg.installed = True
+                masterlist[pkgname].add_version(pkg)
+        else:
+            masterlist[pkgname] = ViewPackage(pkg)
+
+    return OrderedDict(sorted(masterlist.items()))
 
 def register():
     bpy.utils.register_class(BPKG_OT_install)
+    bpy.utils.register_class(BPKG_OT_refresh_repositories)
+    bpy.utils.register_class(BPKG_OT_refresh_packages)
     bpy.utils.register_class(BPKG_OT_refresh)
     bpy.utils.register_class(BPKG_OT_load_repositories)
     bpy.utils.register_class(BPKG_OT_hang)
@@ -655,6 +757,8 @@ def register():
 
 def unregister():
     bpy.utils.unregister_class(BPKG_OT_install)
+    bpy.utils.unregister_class(BPKG_OT_refresh_repositories)
+    bpy.utils.unregister_class(BPKG_OT_refresh_packages)
     bpy.utils.unregister_class(BPKG_OT_refresh)
     bpy.utils.unregister_class(BPKG_OT_load_repositories)
     bpy.utils.unregister_class(BPKG_OT_hang)

bpkg/subproc.py

@@ -141,10 +141,13 @@ class Package:
     Stores package methods and metadata
     """
 
-    log = logging.getLogger(__name__ + ".Repository")
+    log = logging.getLogger(__name__ + ".Package")
 
     def __init__(self, package_dict:dict = None):
-        self.from_dict(package_dict)
+        self.bl_info = {}
+        self.url     = ""
+        self.files   = []
+        self.set_from_dict(package_dict)
 
     def to_dict(self) -> dict:
         """
@@ -153,18 +156,61 @@ class Package:
         return {
                 'bl_info': self.bl_info,
                 'url': self.url,
+                'files': self.files,
                 }
 
-    def from_dict(self, package_dict: dict):
+    def set_from_dict(self, package_dict: dict):
         """
         Get attributes from a dict such as produced by `to_dict`
         """
         if package_dict is None:
             package_dict = {}
         
-        for attr in ('name', 'url', 'bl_info'):
+        for attr in ('files', 'url', 'bl_info'):
-            setattr(self, attr, package_dict.get(attr))
+            if package_dict.get(attr) is not None:
+                setattr(self, attr, package_dict[attr])
 
+    #bl_info convenience getters
+    def get_name() -> str:
+        """Get name from bl_info"""
+        return self.bl_info['name']
+
+    def get_description() -> str:
+        """Get description from bl_info"""
+        return self.bl_info['description']
+
+    # @classmethod
+    # def from_dict(cls, package_dict: dict):
+    #     """
+    #     Return a Package with values from dict
+    #     """
+    #     pkg = cls()
+    #     pkg.set_from_dict(package_dict)
+
+    @classmethod
+    def from_blinfo(cls, blinfo: dict):
+        """
+        Return a Package with bl_info filled in
+        """
+        return cls({'bl_info': blinfo})
+
+    @classmethod
+    def from_module(cls, module):
+        """
+        Return a Package object from an addon module
+        """
+        from pathlib import Path
+        filepath = Path(module.__file__)
+        if filepath.name == '__init__.py':
+            filepath = filepath.parent
+
+        pkg = cls()
+        pkg.files = [filepath]
+        try:
+            pkg.bl_info = module.bl_info
+        except AttributeError as err:
+            raise BadAddon("Module does not appear to be an addon; no bl_info attribute") from err
+        return pkg
 
 
 class Repository:
@@ -526,6 +572,7 @@ def refresh(pipe_to_blender, storage_path: pathlib.Path, repository_url: str):
 
     repo.to_file(repo_path) # TODO: this always writes even if repo wasn't changed
     pipe_to_blender.send(RepositoryResult(repo.to_dict(sort=True, ids=True)))
+    pipe_to_blender.send(Success())
 
 def load(pipe_to_blender, storage_path: pathlib.Path):
     """Reads the stored repository and sends the result to blender"""