Cleanup: rename package manager addon from bpkg -> package_manager
As I see it, the *package manager* is called "bpkg", while the *package manager addon* is called "package_manager". The *package manager* contains the actual package management code, and the *package manager addon* provides the interface to use it.
This commit is contained in:
Ellwood Zwovic
636
package_manager/subproc.py
Normal file
636
package_manager/subproc.py
Normal file
@@ -0,0 +1,636 @@
|
||||
"""
|
||||
All the stuff that needs to run in a subprocess.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import pathlib
|
||||
import shutil
|
||||
import json
|
||||
|
||||
|
||||
class Message:
|
||||
"""Superclass for all message sent over pipes."""
|
||||
|
||||
|
||||
class BlenderMessage(Message):
|
||||
"""Superclass for all messages sent from Blender to the subprocess."""
|
||||
|
||||
|
||||
class SubprocMessage(Message):
|
||||
"""Superclass for all messages sent from the subprocess to Blender."""
|
||||
|
||||
|
||||
class Abort(BlenderMessage):
|
||||
"""Sent when the user requests abortion of a task."""
|
||||
|
||||
|
||||
class Progress(SubprocMessage):
|
||||
"""Send from subprocess to Blender to report progress.
|
||||
|
||||
:ivar progress: the progress percentage, from 0-1.
|
||||
"""
|
||||
|
||||
def __init__(self, progress: float):
|
||||
self.progress = progress
|
||||
|
||||
class SubprocError(SubprocMessage):
|
||||
"""Superclass for all fatal error messages sent from the subprocess."""
|
||||
|
||||
def __init__(self, message: str):
|
||||
self.message = message
|
||||
|
||||
class SubprocWarning(SubprocMessage):
|
||||
"""Superclass for all non-fatal warning messages sent from the subprocess."""
|
||||
|
||||
def __init__(self, message: str):
|
||||
self.message = message
|
||||
|
||||
class InstallError(SubprocError):
|
||||
"""Sent when there was an error installing something."""
|
||||
|
||||
class UninstallError(SubprocError):
|
||||
"""Sent when there was an error uninstalling something."""
|
||||
|
||||
class FileConflictError(InstallError):
|
||||
"""Sent when installation would overwrite existing files."""
|
||||
|
||||
def __init__(self, message: str, conflicts: list):
|
||||
self.message = message
|
||||
self.conflicts = conflicts
|
||||
|
||||
class DownloadError(SubprocMessage):
|
||||
"""Sent when there was an error downloading something."""
|
||||
|
||||
def __init__(self, status_code: int, description: str):
|
||||
self.status_code = status_code
|
||||
self.description = description
|
||||
|
||||
|
||||
class Success(SubprocMessage):
|
||||
"""Sent when an operation finished sucessfully."""
|
||||
|
||||
class RepositoryResult(SubprocMessage):
|
||||
"""Sent when an operation returns a repository to be used on the parent process."""
|
||||
|
||||
def __init__(self, repository: dict):
|
||||
self.repository = repository
|
||||
|
||||
class Aborted(SubprocMessage):
|
||||
"""Sent as response to Abort message."""
|
||||
|
||||
|
||||
class InstallException(Exception):
|
||||
"""Raised when there is an error during installation"""
|
||||
|
||||
class DownloadException(Exception):
|
||||
"""Raised when there is an error downloading something"""
|
||||
|
||||
class BadRepository(Exception):
|
||||
"""Raised when reading a repository results in an error"""
|
||||
|
||||
|
||||
class InplaceBackup:
|
||||
"""Utility for moving a file out of the way by appending a '~'"""
|
||||
|
||||
log = logging.getLogger('%s.inplace-backup' % __name__)
|
||||
|
||||
def __init__(self, path: pathlib.Path):
|
||||
self.path = path
|
||||
self.backup()
|
||||
|
||||
def backup(self):
|
||||
"""Move 'path' to 'path~'"""
|
||||
if not self.path.exists():
|
||||
raise FileNotFoundError("Can't backup path which doesn't exist")
|
||||
|
||||
self.backup_path = pathlib.Path(str(self.path) + '~')
|
||||
if self.backup_path.exists():
|
||||
self.log.warning("Overwriting existing backup '{}'".format(self.backup_path))
|
||||
self._rm(self.backup_path)
|
||||
|
||||
shutil.move(str(self.path), str(self.backup_path))
|
||||
|
||||
def restore(self):
|
||||
"""Move 'path~' to 'path'"""
|
||||
try:
|
||||
getattr(self, 'backup_path')
|
||||
except AttributeError as err:
|
||||
raise RuntimeError("Can't restore file before backing it up") from err
|
||||
|
||||
if not self.backup_path.exists():
|
||||
raise FileNotFoundError("Can't restore backup which doesn't exist")
|
||||
|
||||
if self.path.exists():
|
||||
self.log.warning("Overwriting '{0}' with backup file".format(self.path))
|
||||
self._rm(self.path)
|
||||
|
||||
shutil.move(str(self.backup_path), str(self.path))
|
||||
|
||||
def remove(self):
|
||||
"""Remove 'path~'"""
|
||||
self._rm(self.backup_path)
|
||||
|
||||
def _rm(self, path: pathlib.Path):
|
||||
"""Just delete whatever is specified by `path`"""
|
||||
if path.is_dir():
|
||||
shutil.rmtree(str(path))
|
||||
else:
|
||||
path.unlink()
|
||||
|
||||
|
||||
|
||||
class Package:
|
||||
"""
|
||||
Stores package methods and metadata
|
||||
"""
|
||||
|
||||
log = logging.getLogger(__name__ + ".Package")
|
||||
|
||||
def __init__(self, package_dict:dict = None):
|
||||
self.bl_info = {}
|
||||
self.url = ""
|
||||
self.files = []
|
||||
self.set_from_dict(package_dict)
|
||||
|
||||
self.installed = False
|
||||
self.repository = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""
|
||||
Return a dict representation of the package
|
||||
"""
|
||||
return {
|
||||
'bl_info': self.bl_info,
|
||||
'url': self.url,
|
||||
'files': self.files,
|
||||
}
|
||||
|
||||
def set_from_dict(self, package_dict: dict):
|
||||
"""
|
||||
Get attributes from a dict such as produced by `to_dict`
|
||||
"""
|
||||
if package_dict is None:
|
||||
package_dict = {}
|
||||
|
||||
for attr in ('files', 'url', 'bl_info'):
|
||||
if package_dict.get(attr) is not None:
|
||||
setattr(self, attr, package_dict[attr])
|
||||
|
||||
#bl_info convenience getters
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""Get name from bl_info"""
|
||||
return self.bl_info['name']
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
"""Get description from bl_info"""
|
||||
return self.bl_info['description']
|
||||
|
||||
@property
|
||||
def version(self) -> tuple:
|
||||
"""Get version from bl_info"""
|
||||
return tuple(self.bl_info['version'])
|
||||
|
||||
# @classmethod
|
||||
# def from_dict(cls, package_dict: dict):
|
||||
# """
|
||||
# Return a Package with values from dict
|
||||
# """
|
||||
# pkg = cls()
|
||||
# pkg.set_from_dict(package_dict)
|
||||
|
||||
@classmethod
|
||||
def from_blinfo(cls, blinfo: dict):
|
||||
"""
|
||||
Return a Package with bl_info filled in
|
||||
"""
|
||||
return cls({'bl_info': blinfo})
|
||||
|
||||
@classmethod
|
||||
def from_module(cls, module):
|
||||
"""
|
||||
Return a Package object from an addon module
|
||||
"""
|
||||
from pathlib import Path
|
||||
filepath = Path(module.__file__)
|
||||
if filepath.name == '__init__.py':
|
||||
filepath = filepath.parent
|
||||
|
||||
pkg = cls()
|
||||
pkg.files = [filepath.name]
|
||||
pkg.installed_location = str(filepath)
|
||||
try:
|
||||
pkg.bl_info = module.bl_info
|
||||
except AttributeError as err:
|
||||
raise BadAddon("Module does not appear to be an addon; no bl_info attribute") from err
|
||||
return pkg
|
||||
|
||||
|
||||
class Repository:
|
||||
"""
|
||||
Stores repository metadata (including packages)
|
||||
"""
|
||||
|
||||
log = logging.getLogger(__name__ + ".Repository")
|
||||
|
||||
def __init__(self, url=None):
|
||||
if url is None:
|
||||
url = ""
|
||||
self.set_from_dict({'url': url})
|
||||
|
||||
# def cleanse_packagelist(self):
|
||||
# """Remove empty packages (no bl_info), packages with no name"""
|
||||
|
||||
def refresh(self):
|
||||
"""
|
||||
Requests repo.json from URL and embeds etag/last-modification headers
|
||||
"""
|
||||
import requests
|
||||
|
||||
if self.url is None:
|
||||
raise ValueError("Cannot refresh repository without a URL")
|
||||
|
||||
self.log.debug("Refreshing repository from %s", self.url)
|
||||
|
||||
req_headers = {}
|
||||
# Do things this way to avoid adding empty objects/None to the req_headers dict
|
||||
try:
|
||||
req_headers['If-None-Match'] = self._headers['etag']
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
req_headers['If-Modified-Since'] = self._headers['last-modified']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
try:
|
||||
resp = requests.get(self.url, headers=req_headers, timeout=60)
|
||||
except requests.exceptions.RequestException as err:
|
||||
raise DownloadException(err) from err
|
||||
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
except requests.HTTPError as err:
|
||||
self.log.error('Error downloading %s: %s', self.url, err)
|
||||
raise DownloadException(resp.status_code, resp.reason) from err
|
||||
|
||||
if resp.status_code == requests.codes.not_modified:
|
||||
self.log.debug("Packagelist not modified")
|
||||
return
|
||||
|
||||
resp_headers = {}
|
||||
try:
|
||||
resp_headers['etag'] = resp.headers['etag']
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
resp_headers['last-modified'] = resp.headers['last-modified']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
self.log.debug("Found headers: %s", resp_headers)
|
||||
|
||||
|
||||
try:
|
||||
repodict = resp.json()
|
||||
except json.decoder.JSONDecodeError:
|
||||
self.log.exception("Failed to parse downloaded repository")
|
||||
raise DownloadException("Could not parse repository downloaded from '%s'. Are you sure this is the correct URL?" % self.url)
|
||||
repodict['_headers'] = resp_headers
|
||||
|
||||
self.set_from_dict(repodict)
|
||||
|
||||
|
||||
def to_dict(self, sort=False, ids=False) -> dict:
|
||||
"""
|
||||
Return a dict representation of the repository
|
||||
"""
|
||||
packages = [p.to_dict() for p in self.packages]
|
||||
|
||||
if sort:
|
||||
packages.sort(key=lambda p: p['bl_info']['name'].lower())
|
||||
|
||||
if ids:
|
||||
for pkg in packages:
|
||||
# hash may be too big for a C int
|
||||
pkg['id'] = str(hash(pkg['url'] + pkg['bl_info']['name'] + self.name + self.url))
|
||||
|
||||
return {
|
||||
'name': self.name,
|
||||
'packages': packages,
|
||||
'url': self.url,
|
||||
'_headers': self._headers,
|
||||
}
|
||||
|
||||
def set_from_dict(self, repodict: dict):
|
||||
"""
|
||||
Get repository attributes from a dict such as produced by `to_dict`
|
||||
"""
|
||||
|
||||
def initialize(item, value):
|
||||
if item is None:
|
||||
return value
|
||||
else:
|
||||
return item
|
||||
|
||||
#Be certain to initialize everything; downloaded packagelist might contain null values
|
||||
name = initialize(repodict.get('name'), "")
|
||||
url = initialize(repodict.get('url'), "")
|
||||
packages = initialize(repodict.get('packages'), [])
|
||||
headers = initialize(repodict.get('_headers'), {})
|
||||
|
||||
self.name = name
|
||||
self.url = url
|
||||
self.packages = [Package(pkg) for pkg in packages]
|
||||
self._headers = headers
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, repodict: dict):
|
||||
"""
|
||||
Like `set_from_dict`, but immutable
|
||||
"""
|
||||
repo = cls()
|
||||
repo.set_from_dict(repodict)
|
||||
return repo
|
||||
|
||||
def to_file(self, path: pathlib.Path):
|
||||
"""
|
||||
Dump repository to a json file at `path`.
|
||||
"""
|
||||
if self.packages is None:
|
||||
self.log.warning("Writing an empty repository")
|
||||
|
||||
with path.open('w', encoding='utf-8') as repo_file:
|
||||
json.dump(self.to_dict(), repo_file, indent=4, sort_keys=True)
|
||||
self.log.debug("Repository written to %s" % path)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: pathlib.Path):
|
||||
"""
|
||||
Read repository from a json file at `path`.
|
||||
"""
|
||||
repo_file = path.open('r', encoding='utf-8')
|
||||
|
||||
with repo_file:
|
||||
try:
|
||||
repo = cls.from_dict(json.load(repo_file))
|
||||
except Exception as err:
|
||||
raise BadRepository from err
|
||||
|
||||
cls.log.debug("Repository read from %s", path)
|
||||
return repo
|
||||
|
||||
|
||||
def _download(pipe_to_blender, package_url: str, download_dir: pathlib.Path) -> pathlib.Path:
|
||||
"""Downloads the given package
|
||||
|
||||
:returns: path to the downloaded file, or None in case of error.
|
||||
"""
|
||||
|
||||
import requests
|
||||
|
||||
log = logging.getLogger('%s.download' % __name__)
|
||||
log.info('Going to download %s to %s', package_url, download_dir)
|
||||
pipe_to_blender.send(Progress(0.0))
|
||||
|
||||
log.info('Downloading %s', package_url)
|
||||
try:
|
||||
resp = requests.get(package_url, stream=True, verify=True)
|
||||
except requests.exceptions.RequestException as err:
|
||||
pipe_to_blender.send(DownloadError(1, err))
|
||||
raise
|
||||
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
except requests.HTTPError as ex:
|
||||
log.error('Error downloading %s: %s', package_url, ex)
|
||||
pipe_to_blender.send(DownloadError(resp.status_code, str(ex)))
|
||||
return None
|
||||
|
||||
try:
|
||||
# Use float so that we can also use infinity
|
||||
content_length = float(resp.headers['content-length'])
|
||||
except KeyError:
|
||||
log.warning('Server did not send content length, cannot report progress.')
|
||||
content_length = float('inf')
|
||||
|
||||
# TODO: check if there's enough disk space.
|
||||
|
||||
# TODO: get filename from Content-Disposition header, if available.
|
||||
# TODO: use urllib.parse to parse the URL.
|
||||
local_filename = package_url.split('/')[-1] or 'download.tmp'
|
||||
local_fpath = download_dir / local_filename
|
||||
|
||||
downloaded_length = 0
|
||||
with local_fpath.open('wb') as outfile:
|
||||
for chunk in resp.iter_content(chunk_size=1024 ** 2):
|
||||
# Handle abort messages from Blender
|
||||
while pipe_to_blender.poll():
|
||||
recvd = pipe_to_blender.recv()
|
||||
if isinstance(recvd, Abort):
|
||||
log.warning('Aborting download of %s by request', package_url)
|
||||
pipe_to_blender.send(Aborted())
|
||||
return None
|
||||
log.warning('Unknown message %s received, ignoring', recvd)
|
||||
|
||||
if not chunk: # filter out keep-alive new chunks
|
||||
continue
|
||||
|
||||
outfile.write(chunk)
|
||||
|
||||
downloaded_length += len(chunk)
|
||||
|
||||
# TODO: use multiplier for progress, so that we can count up to 70% and
|
||||
# leave 30% "progress" for installation of the package.
|
||||
pipe_to_blender.send(Progress(downloaded_length / content_length))
|
||||
|
||||
return local_fpath
|
||||
|
||||
def _add_to_installed(storage_path: pathlib.Path, pkg: Package):
|
||||
"""Add pkg to local repository"""
|
||||
repo_path = storage_path / 'local.json'
|
||||
if repo_path.exists():
|
||||
repo = Repository.from_file(repo_path)
|
||||
else:
|
||||
repo = Repository()
|
||||
repo.packages.append(pkg)
|
||||
repo.to_file(repo_path)
|
||||
|
||||
def _remove_from_installed(storage_path: pathlib.Path, pkg: Package):
|
||||
"""Remove pkg from local repository"""
|
||||
repo = Repository.from_file(storage_path / 'local.json')
|
||||
#TODO: this won't work, compare by name? (watch out for conflicts though)
|
||||
repo.packages.remove(pkg)
|
||||
|
||||
def _install(pipe_to_blender, pkgpath: pathlib.Path, dest: pathlib.Path, searchpaths: list):
|
||||
"""Extracts/moves package at `pkgpath` to `dest`"""
|
||||
import zipfile
|
||||
|
||||
log = logging.getLogger('%s.install' % __name__)
|
||||
|
||||
log.debug("Starting installation")
|
||||
pipe_to_blender.send(Progress(0.0))
|
||||
|
||||
if not pkgpath.is_file():
|
||||
raise InstallException("Package isn't a file")
|
||||
|
||||
if not dest.is_dir():
|
||||
raise InstallException("Destination is not a directory")
|
||||
|
||||
# TODO: check to make sure addon/package isn't already installed elsewhere
|
||||
|
||||
# The following is adapted from `addon_install` in bl_operators/wm.py
|
||||
|
||||
# check to see if the file is in compressed format (.zip)
|
||||
if zipfile.is_zipfile(pkgpath):
|
||||
log.debug("Package is zipfile")
|
||||
try:
|
||||
file_to_extract = zipfile.ZipFile(str(pkgpath), 'r')
|
||||
except Exception as err:
|
||||
raise InstallException("Failed to read zip file: %s" % err) from err
|
||||
|
||||
def root_files(filelist: list) -> list:
|
||||
"""Some string parsing to get a list of the root contents of a zip from its namelist"""
|
||||
rootlist = []
|
||||
for f in filelist:
|
||||
# Get all names which have no path separators (root level files)
|
||||
# or have a single path separator at the end (root level directories).
|
||||
if len(f.rstrip('/').split('/')) == 1:
|
||||
rootlist.append(f)
|
||||
return rootlist
|
||||
|
||||
conflicts = [dest / f for f in root_files(file_to_extract.namelist()) if (dest / f).exists()]
|
||||
backups = []
|
||||
for conflict in conflicts:
|
||||
log.debug("Creating backup of conflict %s", conflict)
|
||||
backups.append(InplaceBackup(conflict))
|
||||
|
||||
try:
|
||||
file_to_extract.extractall(str(dest))
|
||||
except Exception as err:
|
||||
for backup in backups:
|
||||
backup.restore()
|
||||
raise InstallException("Failed to extract zip file to '%s': %s" % (dest, err)) from err
|
||||
|
||||
for backup in backups:
|
||||
backup.remove()
|
||||
|
||||
else:
|
||||
log.debug("Package is pyfile")
|
||||
dest_file = (dest / pkgpath.name)
|
||||
|
||||
if dest_file.exists():
|
||||
backup = InplaceBackup(dest_file)
|
||||
|
||||
try:
|
||||
shutil.copyfile(str(pkgpath), str(dest_file))
|
||||
except Exception as err:
|
||||
backup.restore()
|
||||
raise InstallException("Failed to copy file to '%s': %s" % (dest, err)) from err
|
||||
|
||||
try:
|
||||
pkgpath.unlink()
|
||||
log.debug("Removed cached package: %s", pkgpath)
|
||||
except Exception as err:
|
||||
pipe_to_blender.send(SubprocWarning("Install succeeded, but failed to remove package from cache: %s" % err))
|
||||
log.warning("Failed to remove package from cache: %s", err)
|
||||
|
||||
pipe_to_blender.send(Progress(1.0))
|
||||
return
|
||||
|
||||
|
||||
def download_and_install(pipe_to_blender, package_url: str, install_path: pathlib.Path, search_paths: list):
|
||||
"""Downloads and installs the given package."""
|
||||
|
||||
from . import cache
|
||||
|
||||
log = logging.getLogger('%s.download_and_install' % __name__)
|
||||
|
||||
cache_dir = cache.cache_directory('downloads')
|
||||
downloaded = _download(pipe_to_blender, package_url, cache_dir)
|
||||
|
||||
if not downloaded:
|
||||
log.debug('Download failed/aborted, not going to install anything.')
|
||||
return
|
||||
|
||||
try:
|
||||
_install(pipe_to_blender, downloaded, install_path, search_paths)
|
||||
pipe_to_blender.send(Success())
|
||||
except InstallException as err:
|
||||
log.exception("Failed to install package: %s", err)
|
||||
pipe_to_blender.send(InstallError(err))
|
||||
|
||||
def uninstall(pipe_to_blender, package: Package, install_path: pathlib.Path):
|
||||
"""Deletes the given package's files from the install directory"""
|
||||
#TODO: move package to cache and present an "undo" button to user, to give nicer UX on misclicks
|
||||
|
||||
#TODO: move this to a shared utility function
|
||||
# Duplicated code with InplaceBackup class
|
||||
def _rm(path: pathlib.Path):
|
||||
"""Just delete whatever is specified by `path`"""
|
||||
if path.is_dir():
|
||||
shutil.rmtree(str(path))
|
||||
else:
|
||||
path.unlink()
|
||||
|
||||
for pkgfile in [install_path / pathlib.Path(p) for p in package.files]:
|
||||
if not pkgfile.exists():
|
||||
pipe_to_blender.send(UninstallError("Could not find file owned by package: '%s'. Refusing to uninstall." % pkgfile))
|
||||
return None
|
||||
|
||||
for pkgfile in [install_path / pathlib.Path(p) for p in package.files]:
|
||||
_rm(pkgfile)
|
||||
|
||||
pipe_to_blender.send(Success())
|
||||
|
||||
|
||||
def _load_repo(storage_path: pathlib.Path) -> Repository:
|
||||
"""Reads the stored repositories"""
|
||||
|
||||
repo_path = storage_path / 'repo.json'
|
||||
return Repository.from_file(repo_path)
|
||||
|
||||
def refresh(pipe_to_blender, storage_path: pathlib.Path, repository_url: str):
|
||||
"""Retrieves and stores the given repository"""
|
||||
|
||||
log = logging.getLogger(__name__ + '.refresh')
|
||||
|
||||
repo_path = storage_path / 'repo.json'
|
||||
if repo_path.exists():
|
||||
repo = Repository.from_file(repo_path)
|
||||
if repo.url != repository_url:
|
||||
# We're getting a new repository
|
||||
repo = Repository(repository_url)
|
||||
else:
|
||||
repo = Repository(repository_url)
|
||||
|
||||
try:
|
||||
repo.refresh()
|
||||
except DownloadException as err:
|
||||
pipe_to_blender.send(SubprocError(err))
|
||||
return
|
||||
|
||||
repo.to_file(repo_path) # TODO: this always writes even if repo wasn't changed
|
||||
pipe_to_blender.send(RepositoryResult(repo))
|
||||
pipe_to_blender.send(Success())
|
||||
|
||||
def load(pipe_to_blender, storage_path: pathlib.Path):
|
||||
"""Reads the stored repository and sends the result to blender"""
|
||||
|
||||
try:
|
||||
repo = _load_repo(storage_path)
|
||||
pipe_to_blender.send(RepositoryResult(repo.to_dict(sort=True, ids=True)))
|
||||
pipe_to_blender.send(Success())
|
||||
return repo
|
||||
except BadRepository as err:
|
||||
pipe_to_blender.send(SubprocError("Failed to read repository: %s" % err))
|
||||
|
||||
# def load_local(pipe_to_blender
|
||||
|
||||
|
||||
def debug_hang():
|
||||
"""Hangs for an hour. For testing purposes only."""
|
||||
|
||||
import time
|
||||
time.sleep(3600)
|
||||
Reference in New Issue
Block a user