From 2f6357e40e9e954b3feceef50b5069c13f038540 Mon Sep 17 00:00:00 2001 From: Ellwood Zwovic Date: Wed, 26 Jul 2017 02:08:01 -0700 Subject: [PATCH] Cleanup: Move package download/install code out of subproc.py Instead do such things in bpkg, and only handle interfacing between blender and bpkg in subproc.py --- package_manager/__init__.py | 170 +++++++----------- package_manager/bpkg/exceptions.py | 4 +- package_manager/bpkg/types.py | 28 ++- package_manager/bpkg/utils.py | 137 ++++++++++++++ package_manager/messages.py | 64 +++---- package_manager/subproc.py | 275 +++++------------------------ 6 files changed, 296 insertions(+), 382 deletions(-) diff --git a/package_manager/__init__.py b/package_manager/__init__.py index cfacaa1..4065716 100644 --- a/package_manager/__init__.py +++ b/package_manager/__init__.py @@ -30,12 +30,14 @@ if 'bpy' in locals(): return reloaded_mod subproc = recursive_reload(subproc) + messages = recursive_reload(messages) utils = recursive_reload(utils) bpkg = recursive_reload(bpkg) Package = bpkg.Package else: from . import subproc + from . import messages from . import bpkg from . import utils from .bpkg import Package @@ -114,8 +116,6 @@ class SubprocMixin: def invoke(self, context, event): import multiprocessing - self.log.info('Starting') - self.pipe_blender, self.pipe_subproc = multiprocessing.Pipe() # The subprocess should just be terminated when Blender quits. Without this, @@ -160,6 +160,7 @@ class SubprocMixin: if not self.process.is_alive(): self.report_process_died() + self.cancel(context) self._finish(context) return {'CANCELLED'} @@ -172,7 +173,7 @@ class SubprocMixin: self._abort_timeout = time.time() + 10 self._state = 'ABORTING' - self.pipe_blender.send(subproc.Abort()) + self.pipe_blender.send(messages.Abort()) def _finish(self, context): import multiprocessing @@ -227,13 +228,16 @@ class PACKAGE_OT_install(SubprocMixin, bpy.types.Operator): bl_description = 'Downloads and installs a Blender add-on package' bl_options = {'REGISTER'} - package_url = bpy.props.StringProperty(name='package_url', description='The URL of the file to download') + package_name = bpy.props.StringProperty( + name='package_name', + description='The name of the package to install' + ) log = logging.getLogger(__name__ + '.PACKAGE_OT_install') def invoke(self, context, event): - if not self.package_url: - self.report({'ERROR'}, 'Package URL not given') + if not self.package_name: + self.report({'ERROR'}, 'Package name not given') return {'CANCELLED'} return super().invoke(context, event) @@ -249,14 +253,16 @@ class PACKAGE_OT_install(SubprocMixin, bpy.types.Operator): import multiprocessing self.msg_handlers = { - subproc.Progress: self._subproc_progress, - subproc.DownloadError: self._subproc_download_error, - subproc.InstallError: self._subproc_install_error, - subproc.FileConflictError: self._subproc_conflict_error, - subproc.Success: self._subproc_success, - subproc.Aborted: self._subproc_aborted, + messages.Progress: self._subproc_progress, + messages.DownloadError: self._subproc_download_error, + messages.InstallError: self._subproc_install_error, + messages.Success: self._subproc_success, + messages.Aborted: self._subproc_aborted, } + global _packages + package = _packages[self.package_name].get_latest_version() + import pathlib # TODO: We need other paths besides this one on subprocess end, so it might be better to pass them all at once. @@ -265,31 +271,27 @@ class PACKAGE_OT_install(SubprocMixin, bpy.types.Operator): self.log.debug("Using %s as install path", install_path) import addon_utils - proc = multiprocessing.Process(target=subproc.download_and_install, - args=(self.pipe_subproc, self.package_url, install_path, addon_utils.paths())) + proc = multiprocessing.Process(target=messages.download_and_install_package, + args=(self.pipe_subproc, package, install_path)) return proc - def _subproc_progress(self, progress: subproc.Progress): + def _subproc_progress(self, progress: messages.Progress): self.log.info('Task progress at %i%%', progress.progress * 100) - def _subproc_download_error(self, error: subproc.DownloadError): + def _subproc_download_error(self, error: messages.DownloadError): self.report({'ERROR'}, 'Unable to download package: %s' % error.description) self.quit() - def _subproc_install_error(self, error: subproc.InstallError): + def _subproc_install_error(self, error: messages.InstallError): self.report({'ERROR'}, 'Unable to install package: %s' % error.message) self.quit() - def _subproc_conflict_error(self, error: subproc.FileConflictError): - self.report({'ERROR'}, 'Unable to install package: %s' % error.message) - self.quit() - - def _subproc_success(self, success: subproc.Success): + def _subproc_success(self, success: messages.Success): self.report({'INFO'}, 'Package installed successfully') bpy.ops.package.refresh_packages() self.quit() - def _subproc_aborted(self, aborted: subproc.Aborted): + def _subproc_aborted(self, aborted: messages.Aborted): self.report({'ERROR'}, 'Package installation aborted per your request') self.quit() @@ -326,8 +328,8 @@ class PACKAGE_OT_uninstall(SubprocMixin, bpy.types.Operator): import multiprocessing self.msg_handlers = { - subproc.UninstallError: self._subproc_uninstall_error, - subproc.Success: self._subproc_success, + messages.UninstallError: self._subproc_uninstall_error, + messages.Success: self._subproc_success, } import pathlib @@ -336,16 +338,16 @@ class PACKAGE_OT_uninstall(SubprocMixin, bpy.types.Operator): global _packages package = _packages[self.package_name].get_latest_version() - proc = multiprocessing.Process(target=subproc.uninstall, + proc = multiprocessing.Process(target=subproc.uninstall_package, args=(self.pipe_subproc, package, install_path)) return proc - def _subproc_uninstall_error(self, error: subproc.InstallError): + def _subproc_uninstall_error(self, error: messages.InstallError): self.report({'ERROR'}, error.message) self.quit() - def _subproc_success(self, success: subproc.Success): + def _subproc_success(self, success: messages.Success): self.report({'INFO'}, 'Package uninstalled successfully') bpy.ops.package.refresh_packages() self.quit() @@ -403,6 +405,11 @@ class PACKAGE_OT_refresh_repositories(SubprocMixin, bpy.types.Operator): _running = False def invoke(self, context, event): + self.repolist = bpy.context.user_preferences.addons[__package__].preferences.repositories + if len(self.repolist) == 0: + self.report({'ERROR'}, "No repositories to refresh") + return {'CANCELLED'} + PACKAGE_OT_refresh_repositories._running = True return super().invoke(context, event) @@ -426,38 +433,43 @@ class PACKAGE_OT_refresh_repositories(SubprocMixin, bpy.types.Operator): #TODO: make sure all possible messages are handled self.msg_handlers = { - subproc.Progress: self._subproc_progress, - subproc.SubprocError: self._subproc_error, - subproc.DownloadError: self._subproc_download_error, - subproc.Success: self._subproc_success, - subproc.RepositoryResult: self._subproc_repository_result, - subproc.Aborted: self._subproc_aborted, + messages.Progress: self._subproc_progress, + messages.SubprocError: self._subproc_error, + messages.DownloadError: self._subproc_download_error, + messages.Success: self._subproc_success, + messages.RepositoryResult: self._subproc_repository_result, + messages.BadRepositoryError: self._subproc_repository_error, + messages.Aborted: self._subproc_aborted, } import pathlib storage_path = pathlib.Path(bpy.utils.user_resource('CONFIG', 'packages', create=True)) - repository_url = bpy.context.user_preferences.addons[__package__].preferences.repositories[0].url + repository_url = self.repolist[0].url - proc = multiprocessing.Process(target=subproc.refresh, + proc = multiprocessing.Process(target=subproc.refresh_repository, args=(self.pipe_subproc, storage_path, repository_url)) return proc - def _subproc_progress(self, progress: subproc.Progress): + def _subproc_progress(self, progress: messages.Progress): self.log.info('Task progress at %i%%', progress.progress * 100) - def _subproc_error(self, error: subproc.SubprocError): + def _subproc_error(self, error: messages.SubprocError): self.report({'ERROR'}, 'Unable to refresh package list: %s' % error.message) self.quit() - def _subproc_download_error(self, error: subproc.DownloadError): - self.report({'ERROR'}, 'Unable to download package list: %s' % error.description) + def _subproc_download_error(self, error: messages.DownloadError): + self.report({'ERROR'}, 'Unable to download package list: %s' % error.message) self.quit() - def _subproc_success(self, success: subproc.Success): + def _subproc_repository_error(self, error: messages.BadRepositoryError): + self.report({'ERROR'}, str(error.message)) self.quit() - def _subproc_repository_result(self, result: subproc.RepositoryResult): + def _subproc_success(self, success: messages.Success): + self.quit() + + def _subproc_repository_result(self, result: messages.RepositoryResult): available_packages = result.repository.packages installed_packages = get_packages_from_disk(refresh=False) @@ -469,16 +481,16 @@ class PACKAGE_OT_refresh_repositories(SubprocMixin, bpy.types.Operator): _packages = build_composite_packagelist(installed_packages, available_packages) self.report({'INFO'}, 'Package list retrieved successfully') - def _subproc_aborted(self, aborted: subproc.Aborted): + def _subproc_aborted(self, aborted: messages.Aborted): self.report({'ERROR'}, 'Package list retrieval aborted per your request') self.quit() def report_process_died(self): if self.process.exitcode: - self.log.error('Process died without telling us! Exit code was %i', self.process.exitcode) + self.log.error('Refresh process died without telling us! Exit code was %i', self.process.exitcode) self.report({'ERROR'}, 'Error refreshing package lists, exit code %i' % self.process.exitcode) else: - self.log.error('Process died without telling us! Exit code was 0 though') + self.log.error('Refresh process died without telling us! Exit code was 0 though') self.report({'WARNING'}, 'Error refreshing package lists, but process finished OK. This is weird.') #TODO: @@ -494,68 +506,6 @@ class PACKAGE_OT_refresh(bpy.types.Operator): # getattr(bpy.ops, __package__).refresh_packages() return {'FINISHED'} - -class PACKAGE_OT_load_repositories(SubprocMixin, bpy.types.Operator): - bl_idname = 'package.load_repositories' - bl_label = 'Load Repositories' - bl_description = 'Load repositories from disk' - bl_options = {'REGISTER'} - - log = logging.getLogger(__name__ + '.PACKAGE_OT_load_repositories') - - def create_subprocess(self): - """ - Start the load process and register message handlers - """ - - import multiprocessing - import pathlib - - # TODO: We need other paths besides this one on subprocess end, so it might be better to pass them all at once. - # For now, just pass this one. - storage_path = pathlib.Path(bpy.utils.user_resource('CONFIG', 'packages', create=True)) - self.log.debug("Using %s as install path", install_path) - - import addon_utils - - proc = multiprocessing.Process( - target=subproc.load_repositories, - args=(self.pipe_subproc, self.storage_path) - ) - return proc - - self.msg_handlers = { - subproc.SubprocError: self._subproc_error, - subproc.RepositoryResult: self._subproc_repository_result, - subproc.Success: self._subproc_success, - subproc.Aborted: self._subproc_aborted, - } - - - def _subproc_error(self, error: subproc.SubprocError): - self.report({'ERROR'}, 'Failed to load repositories: %s' % error.message) - self.quit() - - def _subproc_repository_result(self, result: subproc.RepositoryResult): - bpy.context.user_preferences.addons[__package__].preferences['repo'] = result.repository - self.log.info("Loaded repository %s", result.repository.name) - - def _subproc_success(self, success: subproc.Success): - self.log.info("Successfully loaded repositories") - self.quit() - - def _subproc_aborted(self, aborted: subproc.Aborted): - self.report({'ERROR'}, 'Package installation aborted per your request') - self.quit() - - def report_process_died(self): - if self.process.exitcode: - self.log.error('Process died without telling us! Exit code was %i', self.process.exitcode) - self.report({'ERROR'}, 'Error downloading package, exit code %i' % self.process.exitcode) - else: - self.log.error('Process died without telling us! Exit code was 0 though') - self.report({'WARNING'}, 'Error downloading package, but process finished OK. This is weird.') - class RepositoryProperty(bpy.types.PropertyGroup): url = bpy.props.StringProperty(name="URL") status = bpy.props.EnumProperty(name="Status", items=[ @@ -844,7 +794,7 @@ class USERPREF_PT_packages(bpy.types.Panel): right.operator(PACKAGE_OT_uninstall.bl_idname, text="Uninstall").package_name=pkg.name elif pkg.user: - right.label("Installed") + right.label("Installed, but not in repo") right.scale_y = 2 right.enabled = False elif not pkg.user: @@ -854,7 +804,7 @@ class USERPREF_PT_packages(bpy.types.Panel): else: if pkg.url: right.operator(PACKAGE_OT_install.bl_idname, - text="Install").package_url=pkg.url + text="Install").package_name=pkg.name else: right.label("Not installed, but no URL?") @@ -1012,7 +962,6 @@ def register(): bpy.utils.register_class(PACKAGE_OT_refresh_repositories) bpy.utils.register_class(PACKAGE_OT_refresh_packages) bpy.utils.register_class(PACKAGE_OT_refresh) - bpy.utils.register_class(PACKAGE_OT_load_repositories) bpy.utils.register_class(USERPREF_PT_packages) bpy.utils.register_class(WM_OT_package_toggle_expand) bpy.types.WindowManager.package_search = bpy.props.StringProperty( @@ -1043,7 +992,6 @@ def unregister(): bpy.utils.unregister_class(PACKAGE_OT_refresh_repositories) bpy.utils.unregister_class(PACKAGE_OT_refresh_packages) bpy.utils.unregister_class(PACKAGE_OT_refresh) - bpy.utils.unregister_class(PACKAGE_OT_load_repositories) bpy.utils.unregister_class(USERPREF_PT_packages) bpy.utils.unregister_class(WM_OT_package_toggle_expand) del bpy.types.WindowManager.package_search diff --git a/package_manager/bpkg/exceptions.py b/package_manager/bpkg/exceptions.py index a85b378..a790148 100644 --- a/package_manager/bpkg/exceptions.py +++ b/package_manager/bpkg/exceptions.py @@ -7,5 +7,5 @@ class InstallException(BpkgException): class DownloadException(BpkgException): """Raised when there is an error downloading something""" -class BadRepository(BpkgException): - """Raised when reading a repository results in an error""" +class BadRepositoryException(BpkgException): + """Raised when there is an error while reading or manipulating a repository""" diff --git a/package_manager/bpkg/types.py b/package_manager/bpkg/types.py index 804a29a..7fd3dcd 100644 --- a/package_manager/bpkg/types.py +++ b/package_manager/bpkg/types.py @@ -42,7 +42,7 @@ class Package: if package_dict.get(attr) is not None: setattr(self, attr, package_dict[attr]) - # bl_info convenience getters + # bl_info convenience getters {{{ # required fields @property def name(self) -> str: @@ -132,6 +132,7 @@ class Package: return self.bl_info['tracker_url'] except KeyError: return None + # }}} # @classmethod # def from_dict(cls, package_dict: dict): @@ -168,6 +169,27 @@ class Package: raise BadAddon("Module does not appear to be an addon; no bl_info attribute") from err return pkg + def download(self, dest: Path, progress_callback=None) -> Path: + """Downloads package to `dest`""" + + if not self.url: + raise ValueError("Cannot download package without a URL") + + return utils.download(self.url, dest, progress_callback) + + def install(self, dest_dir: Path, cache_dir: Path, progress_callback=None): + """Downloads package to `cache_dir`, then extracts/moves package to `dest_dir`""" + + log = logging.getLogger('%s.install' % __name__) + + downloaded = self.download(cache_dir, progress_callback) + + if not downloaded: + log.debug('Download returned None, not going to install anything.') + return + + utils.install(downloaded, dest_dir) + def __repr__(self) -> str: # return self.name return "Package('name': {}, 'version': {})".format(self.name, self.version) @@ -241,7 +263,9 @@ class Repository: repodict = resp.json() except json.decoder.JSONDecodeError: self.log.exception("Failed to parse downloaded repository") - raise exceptions.DownloadException("Could not parse repository downloaded from '%s'. Are you sure this is the correct URL?" % self.url) + raise exceptions.BadRepositoryException( + "Could not parse repository downloaded from '%s'. Are you sure this is the correct URL?" % self.url + ) repodict['_headers'] = resp_headers self.set_from_dict(repodict) diff --git a/package_manager/bpkg/utils.py b/package_manager/bpkg/utils.py index e5481d8..0d83424 100644 --- a/package_manager/bpkg/utils.py +++ b/package_manager/bpkg/utils.py @@ -1,7 +1,77 @@ from pathlib import Path +from . import exceptions import shutil import logging + +def download(url: str, destination: Path, progress_callback=None) -> Path: + """ + Downloads file at the given url, and if progress_callback is specified, + repeatedly calls progress_callback with an argument between 0 and 1, or infinity. + Raises DownloadException if an error occurs with the download. + + :returns: path to the downloaded file, or None if not modified + """ + + import requests + log = logging.getLogger('%s.download' % __name__) + + if progress_callback is None: + # assing to do nothing function + progress_callback = lambda x: None + + progress_callback(0) + + + # derive filename from url if `destination` is an existing directory, otherwise use `destination` directly + if destination.is_dir(): + # TODO: get filename from Content-Disposition header, if available. + from urllib.parse import urlsplit, urlunsplit + parsed_url = urlsplit(url) + local_filename = Path(parsed_url.path).name or 'download.tmp' + local_fpath = destination / local_filename + else: + local_fpath = destination + + log.info('Downloading %s -> %s', url, local_fpath) + + try: + resp = requests.get(url, stream=True, verify=True) + except requests.exceptions.RequestException as err: + raise exceptions.DownloadException(err) from err + + try: + resp.raise_for_status() + except requests.HTTPError as err: + raise exceptions.DownloadException(resp.status_code, str(err)) from err + + if resp.status_code == requests.codes.not_modified: + log.info("Server responded 'Not Modified', not downloading") + return None + + try: + # Use float so that we can also use infinity + content_length = float(resp.headers['content-length']) + except KeyError: + log.warning('Server did not send content length, cannot report progress.') + content_length = float('inf') + + # TODO: check if there's enough disk space. + + + downloaded_length = 0 + with local_fpath.open('wb') as outfile: + for chunk in resp.iter_content(chunk_size=1024 ** 2): + if not chunk: # filter out keep-alive new chunks + continue + + outfile.write(chunk) + downloaded_length += len(chunk) + progress_callback(downloaded_length / content_length) + + return local_fpath + + def rm(path: Path): """Delete whatever is specified by `path`""" if path.is_dir(): @@ -50,3 +120,70 @@ class InplaceBackup: """Remove 'path~'""" rm(self.backup_path) + +def install(src_file: Path, dest_dir: Path): + """Extracts/moves package at `src_file` to `dest_dir`""" + + import zipfile + + log = logging.getLogger('%s.install' % __name__) + log.debug("Starting installation") + + if not src_file.is_file(): + raise exceptions.InstallException("Package isn't a file") + + if not dest_dir.is_dir(): + raise exceptions.InstallException("Destination is not a directory") + + # TODO: check to make sure addon/package isn't already installed elsewhere + + # The following is adapted from `addon_install` in bl_operators/wm.py + + # check to see if the file is in compressed format (.zip) + if zipfile.is_zipfile(str(src_file)): + log.debug("Package is zipfile") + try: + file_to_extract = zipfile.ZipFile(str(src_file), 'r') + except Exception as err: + raise exceptions.InstallException("Failed to read zip file: %s" % err) from err + + def root_files(filelist: list) -> list: + """Some string parsing to get a list of the root contents of a zip from its namelist""" + rootlist = [] + for f in filelist: + # Get all names which have no path separators (root level files) + # or have a single path separator at the end (root level directories). + if len(f.rstrip('/').split('/')) == 1: + rootlist.append(f) + return rootlist + + conflicts = [dest_dir / f for f in root_files(file_to_extract.namelist()) if (dest_dir / f).exists()] + backups = [] + for conflict in conflicts: + log.debug("Creating backup of conflict %s", conflict) + backups.append(InplaceBackup(conflict)) + + try: + file_to_extract.extractall(str(dest_dir)) + except Exception as err: + for backup in backups: + backup.restore() + raise exceptions.InstallException("Failed to extract zip file to '%s': %s" % (dest_dir, err)) from err + + for backup in backups: + backup.remove() + + else: + log.debug("Package is pyfile") + dest_file = (dest_dir / src_file.name) + + if dest_file.exists(): + backup = InplaceBackup(dest_file) + + try: + shutil.copyfile(str(src_file), str(dest_file)) + except Exception as err: + backup.restore() + raise exceptions.InstallException("Failed to copy file to '%s': %s" % (dest_dir, err)) from err + + log.debug("Installation succeeded") diff --git a/package_manager/messages.py b/package_manager/messages.py index a36928f..03ac061 100644 --- a/package_manager/messages.py +++ b/package_manager/messages.py @@ -27,38 +27,6 @@ class Progress(SubprocMessage): def __init__(self, progress: float): self.progress = progress -class SubprocError(SubprocMessage): - """Superclass for all fatal error messages sent from the subprocess.""" - - def __init__(self, message: str): - self.message = message - -class SubprocWarning(SubprocMessage): - """Superclass for all non-fatal warning messages sent from the subprocess.""" - - def __init__(self, message: str): - self.message = message - -class InstallError(SubprocError): - """Sent when there was an error installing something.""" - -class UninstallError(SubprocError): - """Sent when there was an error uninstalling something.""" - -class FileConflictError(InstallError): - """Sent when installation would overwrite existing files.""" - - def __init__(self, message: str, conflicts: list): - self.message = message - self.conflicts = conflicts - -class DownloadError(SubprocMessage): - """Sent when there was an error downloading something.""" - - def __init__(self, status_code: int, description: str): - self.status_code = status_code - self.description = description - class Success(SubprocMessage): """Sent when an operation finished sucessfully.""" @@ -71,3 +39,35 @@ class RepositoryResult(SubprocMessage): class Aborted(SubprocMessage): """Sent as response to Abort message.""" +# subproc warnings + +class SubprocWarning(SubprocMessage): + """Superclass for all non-fatal warning messages sent from the subprocess.""" + + def __init__(self, message: str): + self.message = message + +# subproc errors + +class SubprocError(SubprocMessage): + """Superclass for all fatal error messages sent from the subprocess.""" + + def __init__(self, message: str): + self.message = message + +class InstallError(SubprocError): + """Sent when there was an error installing something.""" + +class UninstallError(SubprocError): + """Sent when there was an error uninstalling something.""" + +class BadRepositoryError(SubprocError): + """Sent when a repository can't be used for some reason""" + +class DownloadError(SubprocMessage): + """Sent when there was an error downloading something.""" + + def __init__(self, message: str, status_code: int = None): + self.status_code = status_code + self.message = message + diff --git a/package_manager/subproc.py b/package_manager/subproc.py index 8e88778..e862c57 100644 --- a/package_manager/subproc.py +++ b/package_manager/subproc.py @@ -2,268 +2,73 @@ All the stuff that needs to run in a subprocess. """ -import logging -import pathlib -import shutil -import json +from pathlib import Path from . import utils from . import bpkg -from .bpkg import Package, Repository -from .messages import * -from .bpkg.exceptions import * +from . import messages +from .bpkg import exceptions as bpkg_exs +import logging -#TODO: move actual downloading code into bpkg -#functions here should only contain glue code for facilitating subprocessing of bpkg functionality -def _download(pipe_to_blender, package_url: str, download_dir: pathlib.Path) -> pathlib.Path: - """Downloads the given package - - :returns: path to the downloaded file, or None in case of error. - """ - - import requests - - log = logging.getLogger('%s.download' % __name__) - log.info('Going to download %s to %s', package_url, download_dir) - pipe_to_blender.send(Progress(0.0)) - - log.info('Downloading %s', package_url) - try: - resp = requests.get(package_url, stream=True, verify=True) - except requests.exceptions.RequestException as err: - pipe_to_blender.send(DownloadError(1, err)) - raise - - try: - resp.raise_for_status() - except requests.HTTPError as ex: - log.error('Error downloading %s: %s', package_url, ex) - pipe_to_blender.send(DownloadError(resp.status_code, str(ex))) - return None - - try: - # Use float so that we can also use infinity - content_length = float(resp.headers['content-length']) - except KeyError: - log.warning('Server did not send content length, cannot report progress.') - content_length = float('inf') - - # TODO: check if there's enough disk space. - - # TODO: get filename from Content-Disposition header, if available. - # TODO: use urllib.parse to parse the URL. - local_filename = package_url.split('/')[-1] or 'download.tmp' - local_fpath = download_dir / local_filename - - downloaded_length = 0 - with local_fpath.open('wb') as outfile: - for chunk in resp.iter_content(chunk_size=1024 ** 2): - # Handle abort messages from Blender - while pipe_to_blender.poll(): - recvd = pipe_to_blender.recv() - if isinstance(recvd, Abort): - log.warning('Aborting download of %s by request', package_url) - pipe_to_blender.send(Aborted()) - return None - log.warning('Unknown message %s received, ignoring', recvd) - - if not chunk: # filter out keep-alive new chunks - continue - - outfile.write(chunk) - - downloaded_length += len(chunk) - - # TODO: use multiplier for progress, so that we can count up to 70% and - # leave 30% "progress" for installation of the package. - pipe_to_blender.send(Progress(downloaded_length / content_length)) - - return local_fpath - -def _add_to_installed(storage_path: pathlib.Path, pkg: Package): - """Add pkg to local repository""" - repo_path = storage_path / 'local.json' - if repo_path.exists(): - repo = Repository.from_file(repo_path) - else: - repo = Repository() - repo.packages.append(pkg) - repo.to_file(repo_path) - -def _remove_from_installed(storage_path: pathlib.Path, pkg: Package): - """Remove pkg from local repository""" - repo = Repository.from_file(storage_path / 'local.json') - #TODO: this won't work, compare by name? (watch out for conflicts though) - repo.packages.remove(pkg) - -def _install(pipe_to_blender, pkgpath: pathlib.Path, dest: pathlib.Path, searchpaths: list): - """Extracts/moves package at `pkgpath` to `dest`""" - import zipfile - - log = logging.getLogger('%s.install' % __name__) - - log.debug("Starting installation") - pipe_to_blender.send(Progress(0.0)) - - if not pkgpath.is_file(): - raise InstallException("Package isn't a file") - - if not dest.is_dir(): - raise InstallException("Destination is not a directory") - - # TODO: check to make sure addon/package isn't already installed elsewhere - - # The following is adapted from `addon_install` in bl_operators/wm.py - - # check to see if the file is in compressed format (.zip) - if zipfile.is_zipfile(str(pkgpath)): - log.debug("Package is zipfile") - try: - file_to_extract = zipfile.ZipFile(str(pkgpath), 'r') - except Exception as err: - raise InstallException("Failed to read zip file: %s" % err) from err - - def root_files(filelist: list) -> list: - """Some string parsing to get a list of the root contents of a zip from its namelist""" - rootlist = [] - for f in filelist: - # Get all names which have no path separators (root level files) - # or have a single path separator at the end (root level directories). - if len(f.rstrip('/').split('/')) == 1: - rootlist.append(f) - return rootlist - - conflicts = [dest / f for f in root_files(file_to_extract.namelist()) if (dest / f).exists()] - backups = [] - for conflict in conflicts: - log.debug("Creating backup of conflict %s", conflict) - backups.append(bpkg.utils.InplaceBackup(conflict)) - - try: - file_to_extract.extractall(str(dest)) - except Exception as err: - for backup in backups: - backup.restore() - raise InstallException("Failed to extract zip file to '%s': %s" % (dest, err)) from err - - for backup in backups: - backup.remove() - - else: - log.debug("Package is pyfile") - dest_file = (dest / pkgpath.name) - - if dest_file.exists(): - backup = bpkg.utils.InplaceBackup(dest_file) - - try: - shutil.copyfile(str(pkgpath), str(dest_file)) - except Exception as err: - backup.restore() - raise InstallException("Failed to copy file to '%s': %s" % (dest, err)) from err - - try: - pkgpath.unlink() - log.debug("Removed cached package: %s", pkgpath) - except Exception as err: - pipe_to_blender.send(SubprocWarning("Install succeeded, but failed to remove package from cache: %s" % err)) - log.warning("Failed to remove package from cache: %s", err) - - pipe_to_blender.send(Progress(1.0)) - return - - -def download_and_install(pipe_to_blender, package_url: str, install_path: pathlib.Path, search_paths: list): +def download_and_install_package(pipe_to_blender, package: bpkg.Package, install_path: Path): """Downloads and installs the given package.""" + log = logging.getLogger(__name__ + '.download_and_install') + from . import cache - - log = logging.getLogger('%s.download_and_install' % __name__) - cache_dir = cache.cache_directory('downloads') - downloaded = _download(pipe_to_blender, package_url, cache_dir) - - if not downloaded: - log.debug('Download failed/aborted, not going to install anything.') - return try: - _install(pipe_to_blender, downloaded, install_path, search_paths) - pipe_to_blender.send(Success()) - except InstallException as err: - log.exception("Failed to install package: %s", err) - pipe_to_blender.send(InstallError(err)) - -def uninstall(pipe_to_blender, package: Package, install_path: pathlib.Path): - """Deletes the given package's files from the install directory""" - #TODO: move package to cache and present an "undo" button to user, to give nicer UX on misclicks - - #TODO: move this to a shared utility function - # Duplicated code with InplaceBackup class - def _rm(path: pathlib.Path): - """Just delete whatever is specified by `path`""" - if path.is_dir(): - shutil.rmtree(str(path)) - else: - path.unlink() - - for pkgfile in [install_path / pathlib.Path(p) for p in package.files]: - if not pkgfile.exists(): - pipe_to_blender.send(UninstallError("Could not find file owned by package: '%s'. Refusing to uninstall." % pkgfile)) - return None - - for pkgfile in [install_path / pathlib.Path(p) for p in package.files]: - _rm(pkgfile) + package.install(install_path, cache_dir) + except bpkg_exs.DownloadException as err: + pipe_to_blender.send(messages.DownloadError(err)) + raise + except bpkg_exs.InstallException as err: + pipe_to_blender.send(messages.InstallError(err)) + raise pipe_to_blender.send(Success()) -def _load_repo(storage_path: pathlib.Path) -> Repository: - """Reads the stored repositories""" +def uninstall_package(pipe_to_blender, package: bpkg.Package, install_path: Path): + """Deletes the given package's files from the install directory""" + #TODO: move package to cache and present an "undo" button to user, to give nicer UX on misclicks - repo_path = storage_path / 'repo.json' - return Repository.from_file(repo_path) + for pkgfile in [install_path / Path(p) for p in package.files]: + if not pkgfile.exists(): + pipe_to_blender.send(messages.UninstallError("Could not find file owned by package: '%s'. Refusing to uninstall." % pkgfile)) + return None -def refresh(pipe_to_blender, storage_path: pathlib.Path, repository_url: str): + for pkgfile in [install_path / Path(p) for p in package.files]: + bpkg.utils.rm(pkgfile) + + pipe_to_blender.send(Success()) + + +def refresh_repository(pipe_to_blender, repo_storage_path: Path, repository_url: str): """Retrieves and stores the given repository""" log = logging.getLogger(__name__ + '.refresh') repository_url = utils.add_repojson_to_url(repository_url) - repo_path = storage_path / 'repo.json' + repo_path = repo_storage_path / 'repo.json' if repo_path.exists(): - repo = Repository.from_file(repo_path) + repo = bpkg.Repository.from_file(repo_path) if repo.url != repository_url: # We're getting a new repository - repo = Repository(repository_url) + repo = bpkg.Repository(repository_url) else: - repo = Repository(repository_url) + repo = bpkg.Repository(repository_url) try: repo.refresh() - except DownloadException as err: - pipe_to_blender.send(SubprocError(err)) - return + except bpkg_exs.DownloadException as err: + pipe_to_blender.send(messages.DownloadError(err)) + raise + except bpkg_exs.BadRepositoryException as err: + pipe_to_blender.send(messages.BadRepositoryError(err)) + raise repo.to_file(repo_path) # TODO: this always writes even if repo wasn't changed - pipe_to_blender.send(RepositoryResult(repo)) - pipe_to_blender.send(Success()) + pipe_to_blender.send(messages.RepositoryResult(repo)) + pipe_to_blender.send(messages.Success()) -def load(pipe_to_blender, storage_path: pathlib.Path): - """Reads the stored repository and sends the result to blender""" - - try: - repo = _load_repo(storage_path) - pipe_to_blender.send(RepositoryResult(repo.to_dict(sort=True, ids=True))) - pipe_to_blender.send(Success()) - return repo - except BadRepository as err: - pipe_to_blender.send(SubprocError("Failed to read repository: %s" % err)) - -# def load_local(pipe_to_blender - - -def debug_hang(): - """Hangs for an hour. For testing purposes only.""" - - import time - time.sleep(3600)