Repository downloading

bpkg_manager/subproc.py

@@ -5,6 +5,7 @@ All the stuff that needs to run in a subprocess.
 import logging
 import pathlib
 import shutil
+import json
 
 
 class Message:
@@ -73,6 +74,16 @@ class Aborted(SubprocMessage):
 class InstallException(Exception):
     """Raised when there is an error during installation"""
 
+class DownloadException(Exception):
+    """Raised when there is an error downloading something"""
+
+    def __init__(self, status_code: int, message: str):
+        self.status_code = status_code
+        self.message = message
+
+class BadRepository(Exception):
+    """Raised when reading a repository results in an error"""
+
 
 class InplaceBackup:
     """Utility for moving a file out of the way by appending a '~'"""
@@ -116,6 +127,176 @@ class InplaceBackup:
 
 
 
+class Package:
+    """
+    Stores package methods and metadata
+    """
+
+    log = logging.getLogger(__name__ + ".Repository")
+
+    def __init__(self, package_dict:dict = None):
+        self.from_dict(package_dict)
+
+    def to_dict(self) -> dict:
+        """
+        Return a dict representation of the package
+        """
+        return {
+                'bl_info': self.bl_info,
+                'url': self.url,
+                }
+
+    def from_dict(self, package_dict: dict):
+        """
+        Get attributes from a dict such as produced by `to_dict`
+        """
+        if package_dict is None:
+            package_dict = {}
+
+        for attr in ('name', 'url', 'bl_info'):
+            setattr(self, attr, package_dict.get(attr))
+
+
+
+class Repository:
+    """
+    Stores repository metadata (including packages)
+    """
+
+    log = logging.getLogger(__name__ + ".Repository")
+
+    def __init__(self, url=None):
+        self.set_from_dict({'url': url})
+        self.log.debug("Initializing repository: %s", self.to_dict())
+
+    def refresh(self):
+        """
+        Requests repo.json from URL and embeds etag/last-modification headers
+        """
+        import requests
+
+        if self.url is None:
+            raise ValueError("Cannot refresh repository without a URL")
+
+        self.log.debug("Refreshing repository from %s", self.url)
+
+        req_headers = {}
+        # Do things this way to avoid adding empty objects/None to the req_headers dict
+        if self._headers:
+            try:
+                req_headers['If-None-Match'] = self._headers['etag']
+            except KeyError:
+                pass
+            try:
+                req_headers['If-Modified-Since'] = self._headers['last-modified']
+            except KeyError:
+                pass
+
+        resp = requests.get(self.url, headers=req_headers)
+
+        try:
+            resp.raise_for_status()
+        except requests.HTTPError as err:
+            self.log.error('Error downloading %s: %s', self.url, err)
+            raise DownloadException(resp.status_code, resp.reason) from err
+
+        if resp.status_code == requests.codes.not_modified:
+            self.log.debug("Packagelist not modified")
+            return
+
+        resp_headers = {}
+        try:
+            resp_headers['etag'] = resp.headers['etag']
+        except KeyError:
+            pass
+        try:
+            resp_headers['last-modified'] = resp.headers['last-modified']
+        except KeyError:
+            pass
+
+        self.log.debug("Found headers: %s", resp_headers)
+
+        
+        repodict = resp.json()
+        repodict['_headers'] = resp_headers
+
+        self.set_from_dict(repodict)
+
+
+    def to_dict(self) -> dict:
+        """
+        Return a dict representation of the repository
+        """
+        self.log.debug("Rendering to a dict")
+        self.log.debug("url: %s", self.url)
+        return {
+                'name':     self.name,
+                'packages': [p.to_dict() for p in self.packages] if self.packages is not None else None,
+                'url':      self.url,
+                '_headers': self._headers,
+                }
+
+    def set_from_dict(self, repodict: dict):
+        """
+        Get repository attributes from a dict such as produced by `to_dict`
+        """
+        if repodict is None:
+            repodict = {}
+
+        for attr in ('name', 'url', 'packages', '_headers'):
+            if attr == 'packages':
+                value = set(Package(pkg) for pkg in repodict.get('packages', []))
+            else:
+                value = repodict.get(attr)
+
+            if value is None:
+                try:
+                    value = getattr(self, attr)
+                except AttributeError:
+                    pass
+
+            setattr(self, attr, value)
+
+    @classmethod
+    def from_dict(cls, repodict: dict) -> Repository:
+        """
+        Like `set_from_dict`, but immutable
+        """
+        repo = cls()
+        repo.set_from_dict(repodict)
+        return repo
+
+    def to_file(self, path: pathlib.Path):
+        """
+        Dump repository to a json file at `path`.
+        """
+        if self.packages is None:
+            self.log.warning("Writing an empty repository")
+
+        with path.open('w', encoding='utf-8') as repo_file:
+            json.dump(self.to_dict(), repo_file, indent=4, sort_keys=True)
+        self.log.debug("Repository written to %s" % path)
+
+    @classmethod
+    def from_file(cls, path: pathlib.Path):
+        """
+        Read repository from a json file at `path`.
+        """
+        try:
+            repo_file = path.open('r', encoding='utf-8')
+        except IOError as err:
+            raise BadRepository from err
+
+        with repo_file:
+            try:
+                repo = cls.from_dict(json.load(repo_file))
+            except Exception as err:
+                raise BadRepository from err
+
+        cls.log.debug("Repository read from %s", path)
+        return repo
+
+
 def _download(pipe_to_blender, package_url: str, download_dir: pathlib.Path) -> pathlib.Path:
     """Downloads the given package
 
@@ -285,8 +466,33 @@ def download_and_install(pipe_to_blender, package_url: str, install_path: pathli
         _install(pipe_to_blender, downloaded, install_path, search_paths)
         pipe_to_blender.send(Success())
     except InstallException as err:
-        log.error("InstallException thrown")
+        #TODO
+        log.error("Failed to install package: %s", err)
+        pipe_to_blender.send(InstallError(err))
 
+def refresh(pipe_to_blender, storage_path: pathlib.Path, repository_url: str):
+    """Retrieves and stores the given repository"""
+
+    log = logging.getLogger(__name__ + '.refresh')
+
+    repo_path = storage_path / 'repo.json'
+    try:
+        repo = Repository.from_file(repo_path)
+    except BadRepository as err:
+        log.warning("Failed to read existing repository: %s. Continuing download.", err)
+        repo = Repository(repository_url)
+
+    if repo.url != repository_url:
+        # We're getting a new repository
+        repo = Repository(repository_url)
+
+    try:
+        repo.refresh()
+    except DownloadException as err:
+        pipe_to_blender.send(DownloadError(err.status_code, err.message))
+
+    repo.to_file(repo_path) # TODO: this always writes even if repo wasn't changed
+    pipe_to_blender.send(Success())
 
 def debug_hang():
     """Hangs for an hour. For testing purposes only."""