Repository downloading

This commit is contained in:
Ellwood Zwovic
2017-07-13 16:33:14 -07:00
parent 058b5a802f
commit 67b1857e58
2 changed files with 280 additions and 3 deletions

View File

@@ -232,6 +232,71 @@ class BPKG_OT_install(SubprocMixin, bpy.types.Operator):
self.log.error('Process died without telling us! Exit code was 0 though') self.log.error('Process died without telling us! Exit code was 0 though')
self.report({'WARNING'}, 'Error downloading package, but process finished OK. This is weird.') self.report({'WARNING'}, 'Error downloading package, but process finished OK. This is weird.')
class BPKG_OT_refresh(SubprocMixin, bpy.types.Operator):
bl_idname = "bpkg.refresh"
bl_label = "Refresh Packages"
bl_description = 'Check for new and updated packages'
bl_options = {'REGISTER'}
log = logging.getLogger(__name__ + ".BPKG_OT_refresh")
def invoke(self, context, event):
return super().invoke(context, event)
def create_subprocess(self):
"""Starts the download process.
Also registers the message handlers.
:rtype: multiprocessing.Process
"""
import multiprocessing
self.msg_handlers = {
subproc.Progress: self._subproc_progress,
subproc.SubprocError: self._subproc_error,
subproc.DownloadError: self._subproc_download_error,
subproc.Success: self._subproc_success,
subproc.Aborted: self._subproc_aborted,
}
import pathlib
storage_path = pathlib.Path(bpy.utils.user_resource('CONFIG', 'addons', create=True))
repository_url = bpy.context.user_preferences.addons[__package__].preferences.repository_url
proc = multiprocessing.Process(target=subproc.refresh,
args=(self.pipe_subproc, storage_path, repository_url))
return proc
def _subproc_progress(self, progress: subproc.Progress):
self.log.info('Task progress at %i%%', progress.progress * 100)
def _subproc_error(self, error: subproc.SubprocError):
self.report({'ERROR'}, 'Unable to refresh package list: %s' % error.message)
self.quit()
def _subproc_download_error(self, error: subproc.DownloadError):
self.report({'ERROR'}, 'Unable to download package list: %s' % error.description)
self.quit()
def _subproc_success(self, success: subproc.Success):
self.report({'INFO'}, 'Package list retrieved successfully')
self.quit()
def _subproc_aborted(self, aborted: subproc.Aborted):
self.report({'ERROR'}, 'Package list retrieval aborted per your request')
self.quit()
def report_process_died(self):
if self.process.exitcode:
self.log.error('Process died without telling us! Exit code was %i', self.process.exitcode)
self.report({'ERROR'}, 'Error refreshing package lists, exit code %i' % self.process.exitcode)
else:
self.log.error('Process died without telling us! Exit code was 0 though')
self.report({'WARNING'}, 'Error refreshing package lists, but process finished OK. This is weird.')
class BPKG_OT_hang(SubprocMixin, bpy.types.Operator): class BPKG_OT_hang(SubprocMixin, bpy.types.Operator):
bl_idname = 'bpkg.hang' bl_idname = 'bpkg.hang'
@@ -265,23 +330,29 @@ class PackageManagerPreferences(bpy.types.AddonPreferences):
name='Package URL', name='Package URL',
description='Just a temporary place to store the URL of a package to download') description='Just a temporary place to store the URL of a package to download')
repository_url = bpy.props.StringProperty(
name='Repository URL',
description='Temporary repository URL')
def draw(self, context): def draw(self, context):
layout = self.layout layout = self.layout
temp_box = layout.box() temp_box = layout.box()
temp_box.label(text="Temporary stuff while we're developing") temp_box.label(text="Temporary stuff while we're developing")
temp_box.prop(self, 'package_url') temp_box.prop(self, 'repository_url')
temp_box.operator(BPKG_OT_install.bl_idname).package_url = self.package_url temp_box.operator(BPKG_OT_refresh.bl_idname)
temp_box.operator(BPKG_OT_hang.bl_idname) temp_box.operator(BPKG_OT_hang.bl_idname)
def register(): def register():
bpy.utils.register_class(BPKG_OT_install) bpy.utils.register_class(BPKG_OT_install)
bpy.utils.register_class(BPKG_OT_refresh)
bpy.utils.register_class(BPKG_OT_hang) bpy.utils.register_class(BPKG_OT_hang)
bpy.utils.register_class(PackageManagerPreferences) bpy.utils.register_class(PackageManagerPreferences)
def unregister(): def unregister():
bpy.utils.unregister_class(BPKG_OT_install) bpy.utils.unregister_class(BPKG_OT_install)
bpy.utils.unregister_class(BPKG_OT_refresh)
bpy.utils.unregister_class(BPKG_OT_hang) bpy.utils.unregister_class(BPKG_OT_hang)
bpy.utils.unregister_class(PackageManagerPreferences) bpy.utils.unregister_class(PackageManagerPreferences)

View File

@@ -5,6 +5,7 @@ All the stuff that needs to run in a subprocess.
import logging import logging
import pathlib import pathlib
import shutil import shutil
import json
class Message: class Message:
@@ -73,6 +74,16 @@ class Aborted(SubprocMessage):
class InstallException(Exception): class InstallException(Exception):
"""Raised when there is an error during installation""" """Raised when there is an error during installation"""
class DownloadException(Exception):
"""Raised when there is an error downloading something"""
def __init__(self, status_code: int, message: str):
self.status_code = status_code
self.message = message
class BadRepository(Exception):
"""Raised when reading a repository results in an error"""
class InplaceBackup: class InplaceBackup:
"""Utility for moving a file out of the way by appending a '~'""" """Utility for moving a file out of the way by appending a '~'"""
@@ -116,6 +127,176 @@ class InplaceBackup:
class Package:
"""
Stores package methods and metadata
"""
log = logging.getLogger(__name__ + ".Repository")
def __init__(self, package_dict:dict = None):
self.from_dict(package_dict)
def to_dict(self) -> dict:
"""
Return a dict representation of the package
"""
return {
'bl_info': self.bl_info,
'url': self.url,
}
def from_dict(self, package_dict: dict):
"""
Get attributes from a dict such as produced by `to_dict`
"""
if package_dict is None:
package_dict = {}
for attr in ('name', 'url', 'bl_info'):
setattr(self, attr, package_dict.get(attr))
class Repository:
"""
Stores repository metadata (including packages)
"""
log = logging.getLogger(__name__ + ".Repository")
def __init__(self, url=None):
self.set_from_dict({'url': url})
self.log.debug("Initializing repository: %s", self.to_dict())
def refresh(self):
"""
Requests repo.json from URL and embeds etag/last-modification headers
"""
import requests
if self.url is None:
raise ValueError("Cannot refresh repository without a URL")
self.log.debug("Refreshing repository from %s", self.url)
req_headers = {}
# Do things this way to avoid adding empty objects/None to the req_headers dict
if self._headers:
try:
req_headers['If-None-Match'] = self._headers['etag']
except KeyError:
pass
try:
req_headers['If-Modified-Since'] = self._headers['last-modified']
except KeyError:
pass
resp = requests.get(self.url, headers=req_headers)
try:
resp.raise_for_status()
except requests.HTTPError as err:
self.log.error('Error downloading %s: %s', self.url, err)
raise DownloadException(resp.status_code, resp.reason) from err
if resp.status_code == requests.codes.not_modified:
self.log.debug("Packagelist not modified")
return
resp_headers = {}
try:
resp_headers['etag'] = resp.headers['etag']
except KeyError:
pass
try:
resp_headers['last-modified'] = resp.headers['last-modified']
except KeyError:
pass
self.log.debug("Found headers: %s", resp_headers)
repodict = resp.json()
repodict['_headers'] = resp_headers
self.set_from_dict(repodict)
def to_dict(self) -> dict:
"""
Return a dict representation of the repository
"""
self.log.debug("Rendering to a dict")
self.log.debug("url: %s", self.url)
return {
'name': self.name,
'packages': [p.to_dict() for p in self.packages] if self.packages is not None else None,
'url': self.url,
'_headers': self._headers,
}
def set_from_dict(self, repodict: dict):
"""
Get repository attributes from a dict such as produced by `to_dict`
"""
if repodict is None:
repodict = {}
for attr in ('name', 'url', 'packages', '_headers'):
if attr == 'packages':
value = set(Package(pkg) for pkg in repodict.get('packages', []))
else:
value = repodict.get(attr)
if value is None:
try:
value = getattr(self, attr)
except AttributeError:
pass
setattr(self, attr, value)
@classmethod
def from_dict(cls, repodict: dict) -> Repository:
"""
Like `set_from_dict`, but immutable
"""
repo = cls()
repo.set_from_dict(repodict)
return repo
def to_file(self, path: pathlib.Path):
"""
Dump repository to a json file at `path`.
"""
if self.packages is None:
self.log.warning("Writing an empty repository")
with path.open('w', encoding='utf-8') as repo_file:
json.dump(self.to_dict(), repo_file, indent=4, sort_keys=True)
self.log.debug("Repository written to %s" % path)
@classmethod
def from_file(cls, path: pathlib.Path):
"""
Read repository from a json file at `path`.
"""
try:
repo_file = path.open('r', encoding='utf-8')
except IOError as err:
raise BadRepository from err
with repo_file:
try:
repo = cls.from_dict(json.load(repo_file))
except Exception as err:
raise BadRepository from err
cls.log.debug("Repository read from %s", path)
return repo
def _download(pipe_to_blender, package_url: str, download_dir: pathlib.Path) -> pathlib.Path: def _download(pipe_to_blender, package_url: str, download_dir: pathlib.Path) -> pathlib.Path:
"""Downloads the given package """Downloads the given package
@@ -285,8 +466,33 @@ def download_and_install(pipe_to_blender, package_url: str, install_path: pathli
_install(pipe_to_blender, downloaded, install_path, search_paths) _install(pipe_to_blender, downloaded, install_path, search_paths)
pipe_to_blender.send(Success()) pipe_to_blender.send(Success())
except InstallException as err: except InstallException as err:
log.error("InstallException thrown") #TODO
log.error("Failed to install package: %s", err)
pipe_to_blender.send(InstallError(err))
def refresh(pipe_to_blender, storage_path: pathlib.Path, repository_url: str):
"""Retrieves and stores the given repository"""
log = logging.getLogger(__name__ + '.refresh')
repo_path = storage_path / 'repo.json'
try:
repo = Repository.from_file(repo_path)
except BadRepository as err:
log.warning("Failed to read existing repository: %s. Continuing download.", err)
repo = Repository(repository_url)
if repo.url != repository_url:
# We're getting a new repository
repo = Repository(repository_url)
try:
repo.refresh()
except DownloadException as err:
pipe_to_blender.send(DownloadError(err.status_code, err.message))
repo.to_file(repo_path) # TODO: this always writes even if repo wasn't changed
pipe_to_blender.send(Success())
def debug_hang(): def debug_hang():
"""Hangs for an hour. For testing purposes only.""" """Hangs for an hour. For testing purposes only."""