Repository downloading
This commit is contained in:
Ellwood Zwovic
@@ -232,6 +232,71 @@ class BPKG_OT_install(SubprocMixin, bpy.types.Operator):
|
||||
self.log.error('Process died without telling us! Exit code was 0 though')
|
||||
self.report({'WARNING'}, 'Error downloading package, but process finished OK. This is weird.')
|
||||
|
||||
class BPKG_OT_refresh(SubprocMixin, bpy.types.Operator):
|
||||
bl_idname = "bpkg.refresh"
|
||||
bl_label = "Refresh Packages"
|
||||
bl_description = 'Check for new and updated packages'
|
||||
bl_options = {'REGISTER'}
|
||||
|
||||
log = logging.getLogger(__name__ + ".BPKG_OT_refresh")
|
||||
|
||||
def invoke(self, context, event):
|
||||
return super().invoke(context, event)
|
||||
|
||||
def create_subprocess(self):
|
||||
"""Starts the download process.
|
||||
|
||||
Also registers the message handlers.
|
||||
|
||||
:rtype: multiprocessing.Process
|
||||
"""
|
||||
|
||||
import multiprocessing
|
||||
|
||||
self.msg_handlers = {
|
||||
subproc.Progress: self._subproc_progress,
|
||||
subproc.SubprocError: self._subproc_error,
|
||||
subproc.DownloadError: self._subproc_download_error,
|
||||
subproc.Success: self._subproc_success,
|
||||
subproc.Aborted: self._subproc_aborted,
|
||||
}
|
||||
|
||||
import pathlib
|
||||
|
||||
storage_path = pathlib.Path(bpy.utils.user_resource('CONFIG', 'addons', create=True))
|
||||
repository_url = bpy.context.user_preferences.addons[__package__].preferences.repository_url
|
||||
|
||||
proc = multiprocessing.Process(target=subproc.refresh,
|
||||
args=(self.pipe_subproc, storage_path, repository_url))
|
||||
return proc
|
||||
|
||||
def _subproc_progress(self, progress: subproc.Progress):
|
||||
self.log.info('Task progress at %i%%', progress.progress * 100)
|
||||
|
||||
def _subproc_error(self, error: subproc.SubprocError):
|
||||
self.report({'ERROR'}, 'Unable to refresh package list: %s' % error.message)
|
||||
self.quit()
|
||||
|
||||
def _subproc_download_error(self, error: subproc.DownloadError):
|
||||
self.report({'ERROR'}, 'Unable to download package list: %s' % error.description)
|
||||
self.quit()
|
||||
|
||||
def _subproc_success(self, success: subproc.Success):
|
||||
self.report({'INFO'}, 'Package list retrieved successfully')
|
||||
self.quit()
|
||||
|
||||
def _subproc_aborted(self, aborted: subproc.Aborted):
|
||||
self.report({'ERROR'}, 'Package list retrieval aborted per your request')
|
||||
self.quit()
|
||||
|
||||
def report_process_died(self):
|
||||
if self.process.exitcode:
|
||||
self.log.error('Process died without telling us! Exit code was %i', self.process.exitcode)
|
||||
self.report({'ERROR'}, 'Error refreshing package lists, exit code %i' % self.process.exitcode)
|
||||
else:
|
||||
self.log.error('Process died without telling us! Exit code was 0 though')
|
||||
self.report({'WARNING'}, 'Error refreshing package lists, but process finished OK. This is weird.')
|
||||
|
||||
|
||||
class BPKG_OT_hang(SubprocMixin, bpy.types.Operator):
|
||||
bl_idname = 'bpkg.hang'
|
||||
@@ -265,23 +330,29 @@ class PackageManagerPreferences(bpy.types.AddonPreferences):
|
||||
name='Package URL',
|
||||
description='Just a temporary place to store the URL of a package to download')
|
||||
|
||||
repository_url = bpy.props.StringProperty(
|
||||
name='Repository URL',
|
||||
description='Temporary repository URL')
|
||||
|
||||
def draw(self, context):
|
||||
layout = self.layout
|
||||
|
||||
temp_box = layout.box()
|
||||
temp_box.label(text="Temporary stuff while we're developing")
|
||||
temp_box.prop(self, 'package_url')
|
||||
temp_box.operator(BPKG_OT_install.bl_idname).package_url = self.package_url
|
||||
temp_box.prop(self, 'repository_url')
|
||||
temp_box.operator(BPKG_OT_refresh.bl_idname)
|
||||
temp_box.operator(BPKG_OT_hang.bl_idname)
|
||||
|
||||
|
||||
def register():
|
||||
bpy.utils.register_class(BPKG_OT_install)
|
||||
bpy.utils.register_class(BPKG_OT_refresh)
|
||||
bpy.utils.register_class(BPKG_OT_hang)
|
||||
bpy.utils.register_class(PackageManagerPreferences)
|
||||
|
||||
|
||||
def unregister():
|
||||
bpy.utils.unregister_class(BPKG_OT_install)
|
||||
bpy.utils.unregister_class(BPKG_OT_refresh)
|
||||
bpy.utils.unregister_class(BPKG_OT_hang)
|
||||
bpy.utils.unregister_class(PackageManagerPreferences)
|
||||
|
||||
@@ -5,6 +5,7 @@ All the stuff that needs to run in a subprocess.
|
||||
import logging
|
||||
import pathlib
|
||||
import shutil
|
||||
import json
|
||||
|
||||
|
||||
class Message:
|
||||
@@ -73,6 +74,16 @@ class Aborted(SubprocMessage):
|
||||
class InstallException(Exception):
|
||||
"""Raised when there is an error during installation"""
|
||||
|
||||
class DownloadException(Exception):
|
||||
"""Raised when there is an error downloading something"""
|
||||
|
||||
def __init__(self, status_code: int, message: str):
|
||||
self.status_code = status_code
|
||||
self.message = message
|
||||
|
||||
class BadRepository(Exception):
|
||||
"""Raised when reading a repository results in an error"""
|
||||
|
||||
|
||||
class InplaceBackup:
|
||||
"""Utility for moving a file out of the way by appending a '~'"""
|
||||
@@ -116,6 +127,176 @@ class InplaceBackup:
|
||||
|
||||
|
||||
|
||||
class Package:
|
||||
"""
|
||||
Stores package methods and metadata
|
||||
"""
|
||||
|
||||
log = logging.getLogger(__name__ + ".Repository")
|
||||
|
||||
def __init__(self, package_dict:dict = None):
|
||||
self.from_dict(package_dict)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""
|
||||
Return a dict representation of the package
|
||||
"""
|
||||
return {
|
||||
'bl_info': self.bl_info,
|
||||
'url': self.url,
|
||||
}
|
||||
|
||||
def from_dict(self, package_dict: dict):
|
||||
"""
|
||||
Get attributes from a dict such as produced by `to_dict`
|
||||
"""
|
||||
if package_dict is None:
|
||||
package_dict = {}
|
||||
|
||||
for attr in ('name', 'url', 'bl_info'):
|
||||
setattr(self, attr, package_dict.get(attr))
|
||||
|
||||
|
||||
|
||||
class Repository:
|
||||
"""
|
||||
Stores repository metadata (including packages)
|
||||
"""
|
||||
|
||||
log = logging.getLogger(__name__ + ".Repository")
|
||||
|
||||
def __init__(self, url=None):
|
||||
self.set_from_dict({'url': url})
|
||||
self.log.debug("Initializing repository: %s", self.to_dict())
|
||||
|
||||
def refresh(self):
|
||||
"""
|
||||
Requests repo.json from URL and embeds etag/last-modification headers
|
||||
"""
|
||||
import requests
|
||||
|
||||
if self.url is None:
|
||||
raise ValueError("Cannot refresh repository without a URL")
|
||||
|
||||
self.log.debug("Refreshing repository from %s", self.url)
|
||||
|
||||
req_headers = {}
|
||||
# Do things this way to avoid adding empty objects/None to the req_headers dict
|
||||
if self._headers:
|
||||
try:
|
||||
req_headers['If-None-Match'] = self._headers['etag']
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
req_headers['If-Modified-Since'] = self._headers['last-modified']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
resp = requests.get(self.url, headers=req_headers)
|
||||
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
except requests.HTTPError as err:
|
||||
self.log.error('Error downloading %s: %s', self.url, err)
|
||||
raise DownloadException(resp.status_code, resp.reason) from err
|
||||
|
||||
if resp.status_code == requests.codes.not_modified:
|
||||
self.log.debug("Packagelist not modified")
|
||||
return
|
||||
|
||||
resp_headers = {}
|
||||
try:
|
||||
resp_headers['etag'] = resp.headers['etag']
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
resp_headers['last-modified'] = resp.headers['last-modified']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
self.log.debug("Found headers: %s", resp_headers)
|
||||
|
||||
|
||||
repodict = resp.json()
|
||||
repodict['_headers'] = resp_headers
|
||||
|
||||
self.set_from_dict(repodict)
|
||||
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""
|
||||
Return a dict representation of the repository
|
||||
"""
|
||||
self.log.debug("Rendering to a dict")
|
||||
self.log.debug("url: %s", self.url)
|
||||
return {
|
||||
'name': self.name,
|
||||
'packages': [p.to_dict() for p in self.packages] if self.packages is not None else None,
|
||||
'url': self.url,
|
||||
'_headers': self._headers,
|
||||
}
|
||||
|
||||
def set_from_dict(self, repodict: dict):
|
||||
"""
|
||||
Get repository attributes from a dict such as produced by `to_dict`
|
||||
"""
|
||||
if repodict is None:
|
||||
repodict = {}
|
||||
|
||||
for attr in ('name', 'url', 'packages', '_headers'):
|
||||
if attr == 'packages':
|
||||
value = set(Package(pkg) for pkg in repodict.get('packages', []))
|
||||
else:
|
||||
value = repodict.get(attr)
|
||||
|
||||
if value is None:
|
||||
try:
|
||||
value = getattr(self, attr)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
setattr(self, attr, value)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, repodict: dict) -> Repository:
|
||||
"""
|
||||
Like `set_from_dict`, but immutable
|
||||
"""
|
||||
repo = cls()
|
||||
repo.set_from_dict(repodict)
|
||||
return repo
|
||||
|
||||
def to_file(self, path: pathlib.Path):
|
||||
"""
|
||||
Dump repository to a json file at `path`.
|
||||
"""
|
||||
if self.packages is None:
|
||||
self.log.warning("Writing an empty repository")
|
||||
|
||||
with path.open('w', encoding='utf-8') as repo_file:
|
||||
json.dump(self.to_dict(), repo_file, indent=4, sort_keys=True)
|
||||
self.log.debug("Repository written to %s" % path)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: pathlib.Path):
|
||||
"""
|
||||
Read repository from a json file at `path`.
|
||||
"""
|
||||
try:
|
||||
repo_file = path.open('r', encoding='utf-8')
|
||||
except IOError as err:
|
||||
raise BadRepository from err
|
||||
|
||||
with repo_file:
|
||||
try:
|
||||
repo = cls.from_dict(json.load(repo_file))
|
||||
except Exception as err:
|
||||
raise BadRepository from err
|
||||
|
||||
cls.log.debug("Repository read from %s", path)
|
||||
return repo
|
||||
|
||||
|
||||
def _download(pipe_to_blender, package_url: str, download_dir: pathlib.Path) -> pathlib.Path:
|
||||
"""Downloads the given package
|
||||
|
||||
@@ -285,8 +466,33 @@ def download_and_install(pipe_to_blender, package_url: str, install_path: pathli
|
||||
_install(pipe_to_blender, downloaded, install_path, search_paths)
|
||||
pipe_to_blender.send(Success())
|
||||
except InstallException as err:
|
||||
log.error("InstallException thrown")
|
||||
#TODO
|
||||
log.error("Failed to install package: %s", err)
|
||||
pipe_to_blender.send(InstallError(err))
|
||||
|
||||
def refresh(pipe_to_blender, storage_path: pathlib.Path, repository_url: str):
|
||||
"""Retrieves and stores the given repository"""
|
||||
|
||||
log = logging.getLogger(__name__ + '.refresh')
|
||||
|
||||
repo_path = storage_path / 'repo.json'
|
||||
try:
|
||||
repo = Repository.from_file(repo_path)
|
||||
except BadRepository as err:
|
||||
log.warning("Failed to read existing repository: %s. Continuing download.", err)
|
||||
repo = Repository(repository_url)
|
||||
|
||||
if repo.url != repository_url:
|
||||
# We're getting a new repository
|
||||
repo = Repository(repository_url)
|
||||
|
||||
try:
|
||||
repo.refresh()
|
||||
except DownloadException as err:
|
||||
pipe_to_blender.send(DownloadError(err.status_code, err.message))
|
||||
|
||||
repo.to_file(repo_path) # TODO: this always writes even if repo wasn't changed
|
||||
pipe_to_blender.send(Success())
|
||||
|
||||
def debug_hang():
|
||||
"""Hangs for an hour. For testing purposes only."""
|
||||
|
||||
Reference in New Issue
Block a user