archive/blender-package-manager-addon
Archived
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move package management code out of subproc.py

Browse Source
This commit is contained in:
Ellwood Zwovic
2017-07-21 00:27:16 -07:00
parent a250777d15
commit e9fbb9f6e7
7 changed files with 398 additions and 376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
package_manager/__init__.py
View File

@@ -18,10 +18,13 @@ if 'bpy' in locals():
import importlib import importlib
subproc = importlib.reload(subproc) subproc = importlib.reload(subproc)
Package = subproc.Package
bpkg = importlib.reload(bpkg)
Package = bpkg.Package
else: else:
from . import subproc from . import subproc
from .subproc import Package from . import bpkg
from .bpkg import Package
import bpy import bpy
from collections import OrderedDict from collections import OrderedDict

11
package_manager/bpkg/__init__.py Normal file
View File

@@ -0,0 +1,11 @@
from .exceptions import (
BpkgException,
InstallException,
DownloadException,
BadRepository,
)
from .types import (
Package,
Repository,
)

11
package_manager/bpkg/exceptions.py Normal file
View File

@@ -0,0 +1,11 @@
class BpkgException(Exception):
"""Superclass for all package manager exceptions"""
class InstallException(BpkgException):
"""Raised when there is an error during installation"""
class DownloadException(BpkgException):
"""Raised when there is an error downloading something"""
class BadRepository(BpkgException):
"""Raised when reading a repository results in an error"""

245
package_manager/bpkg/types.py Normal file
View File

@@ -0,0 +1,245 @@
import logging
import json
from pathlib import Path
class Package:
"""
Stores package methods and metadata
"""
log = logging.getLogger(__name__ + ".Package")
def __init__(self, package_dict:dict = None):
self.bl_info = {}
self.url = ""
self.files = []
self.set_from_dict(package_dict)
self.installed = False
self.repository = None
def to_dict(self) -> dict:
"""
Return a dict representation of the package
"""
return {
'bl_info': self.bl_info,
'url': self.url,
'files': self.files,
}
def set_from_dict(self, package_dict: dict):
"""
Get attributes from a dict such as produced by `to_dict`
"""
if package_dict is None:
package_dict = {}
for attr in ('files', 'url', 'bl_info'):
if package_dict.get(attr) is not None:
setattr(self, attr, package_dict[attr])
#bl_info convenience getters
@property
def name(self) -> str:
"""Get name from bl_info"""
return self.bl_info['name']
@property
def description(self) -> str:
"""Get description from bl_info"""
return self.bl_info['description']
@property
def version(self) -> tuple:
"""Get version from bl_info"""
return tuple(self.bl_info['version'])
# @classmethod
# def from_dict(cls, package_dict: dict):
# """
# Return a Package with values from dict
# """
# pkg = cls()
# pkg.set_from_dict(package_dict)
@classmethod
def from_blinfo(cls, blinfo: dict):
"""
Return a Package with bl_info filled in
"""
return cls({'bl_info': blinfo})
@classmethod
def from_module(cls, module):
"""
Return a Package object from an addon module
"""
from pathlib import Path
filepath = Path(module.__file__)
if filepath.name == '__init__.py':
filepath = filepath.parent
pkg = cls()
pkg.files = [filepath.name]
pkg.installed_location = str(filepath)
try:
pkg.bl_info = module.bl_info
except AttributeError as err:
raise BadAddon("Module does not appear to be an addon; no bl_info attribute") from err
return pkg
class Repository:
"""
Stores repository metadata (including packages)
"""
log = logging.getLogger(__name__ + ".Repository")
def __init__(self, url=None):
if url is None:
url = ""
self.set_from_dict({'url': url})
# def cleanse_packagelist(self):
# """Remove empty packages (no bl_info), packages with no name"""
def refresh(self):
"""
Requests repo.json from URL and embeds etag/last-modification headers
"""
import requests
if self.url is None:
raise ValueError("Cannot refresh repository without a URL")
self.log.debug("Refreshing repository from %s", self.url)
req_headers = {}
# Do things this way to avoid adding empty objects/None to the req_headers dict
try:
req_headers['If-None-Match'] = self._headers['etag']
except KeyError:
pass
try:
req_headers['If-Modified-Since'] = self._headers['last-modified']
except KeyError:
pass
try:
resp = requests.get(self.url, headers=req_headers, timeout=60)
except requests.exceptions.RequestException as err:
raise DownloadException(err) from err
try:
resp.raise_for_status()
except requests.HTTPError as err:
self.log.error('Error downloading %s: %s', self.url, err)
raise DownloadException(resp.status_code, resp.reason) from err
if resp.status_code == requests.codes.not_modified:
self.log.debug("Packagelist not modified")
return
resp_headers = {}
try:
resp_headers['etag'] = resp.headers['etag']
except KeyError:
pass
try:
resp_headers['last-modified'] = resp.headers['last-modified']
except KeyError:
pass
self.log.debug("Found headers: %s", resp_headers)
try:
repodict = resp.json()
except json.decoder.JSONDecodeError:
self.log.exception("Failed to parse downloaded repository")
raise DownloadException("Could not parse repository downloaded from '%s'. Are you sure this is the correct URL?" % self.url)
repodict['_headers'] = resp_headers
self.set_from_dict(repodict)
def to_dict(self, sort=False, ids=False) -> dict:
"""
Return a dict representation of the repository
"""
packages = [p.to_dict() for p in self.packages]
if sort:
packages.sort(key=lambda p: p['bl_info']['name'].lower())
if ids:
for pkg in packages:
# hash may be too big for a C int
pkg['id'] = str(hash(pkg['url'] + pkg['bl_info']['name'] + self.name + self.url))
return {
'name': self.name,
'packages': packages,
'url': self.url,
'_headers': self._headers,
}
def set_from_dict(self, repodict: dict):
"""
Get repository attributes from a dict such as produced by `to_dict`
"""
def initialize(item, value):
if item is None:
return value
else:
return item
#Be certain to initialize everything; downloaded packagelist might contain null values
name = initialize(repodict.get('name'), "")
url = initialize(repodict.get('url'), "")
packages = initialize(repodict.get('packages'), [])
headers = initialize(repodict.get('_headers'), {})
self.name = name
self.url = url
self.packages = [Package(pkg) for pkg in packages]
self._headers = headers
@classmethod
def from_dict(cls, repodict: dict):
"""
Like `set_from_dict`, but immutable
"""
repo = cls()
repo.set_from_dict(repodict)
return repo
def to_file(self, path: Path):
"""
Dump repository to a json file at `path`.
"""
if self.packages is None:
self.log.warning("Writing an empty repository")
with path.open('w', encoding='utf-8') as repo_file:
json.dump(self.to_dict(), repo_file, indent=4, sort_keys=True)
self.log.debug("Repository written to %s" % path)
@classmethod
def from_file(cls, path: Path):
"""
Read repository from a json file at `path`.
"""
repo_file = path.open('r', encoding='utf-8')
with repo_file:
try:
repo = cls.from_dict(json.load(repo_file))
except Exception as err:
raise BadRepository from err
cls.log.debug("Repository read from %s", path)
return repo

48
package_manager/bpkg/utils.py Normal file
View File

@@ -0,0 +1,48 @@
class InplaceBackup:
"""Utility class for moving a file out of the way by appending a '~'"""
log = logging.getLogger('%s.inplace-backup' % __name__)
def __init__(self, path: pathlib.Path):
self.path = path
self.backup()
def backup(self):
"""Move 'path' to 'path~'"""
if not self.path.exists():
raise FileNotFoundError("Can't backup path which doesn't exist")
self.backup_path = pathlib.Path(str(self.path) + '~')
if self.backup_path.exists():
self.log.warning("Overwriting existing backup '{}'".format(self.backup_path))
self._rm(self.backup_path)
shutil.move(str(self.path), str(self.backup_path))
def restore(self):
"""Move 'path~' to 'path'"""
try:
getattr(self, 'backup_path')
except AttributeError as err:
raise RuntimeError("Can't restore file before backing it up") from err
if not self.backup_path.exists():
raise FileNotFoundError("Can't restore backup which doesn't exist")
if self.path.exists():
self.log.warning("Overwriting '{0}' with backup file".format(self.path))
self._rm(self.path)
shutil.move(str(self.backup_path), str(self.path))
def remove(self):
"""Remove 'path~'"""
self._rm(self.backup_path)
def _rm(self, path: pathlib.Path):
"""Just delete whatever is specified by `path`"""
if path.is_dir():
shutil.rmtree(str(path))
else:
path.unlink()

73
package_manager/messages.py Normal file
View File

@@ -0,0 +1,73 @@
from .bpkg import Repository
class Message:
"""Superclass for all message sent over pipes."""
# Blender messages
class BlenderMessage(Message):
"""Superclass for all messages sent from Blender to the subprocess."""
class Abort(BlenderMessage):
"""Sent when the user requests abortion of a task."""
# Subproc messages
class SubprocMessage(Message):
"""Superclass for all messages sent from the subprocess to Blender."""
class Progress(SubprocMessage):
"""Send from subprocess to Blender to report progress.
:ivar progress: the progress percentage, from 0-1.
"""
def __init__(self, progress: float):
self.progress = progress
class SubprocError(SubprocMessage):
"""Superclass for all fatal error messages sent from the subprocess."""
def __init__(self, message: str):
self.message = message
class SubprocWarning(SubprocMessage):
"""Superclass for all non-fatal warning messages sent from the subprocess."""
def __init__(self, message: str):
self.message = message
class InstallError(SubprocError):
"""Sent when there was an error installing something."""
class UninstallError(SubprocError):
"""Sent when there was an error uninstalling something."""
class FileConflictError(InstallError):
"""Sent when installation would overwrite existing files."""
def __init__(self, message: str, conflicts: list):
self.message = message
self.conflicts = conflicts
class DownloadError(SubprocMessage):
"""Sent when there was an error downloading something."""
def __init__(self, status_code: int, description: str):
self.status_code = status_code
self.description = description
class Success(SubprocMessage):
"""Sent when an operation finished sucessfully."""
class RepositoryResult(SubprocMessage):
"""Sent when an operation returns a repository to be used on the parent process."""
def __init__(self, repository: Repository):
self.repository = repository
class Aborted(SubprocMessage):
"""Sent as response to Abort message."""

379
package_manager/subproc.py
View File

@@ -6,382 +6,13 @@ import logging
import pathlib import pathlib
import shutil import shutil
import json import json
from .bpkg import Package, Repository
from .messages import *
from .bpkg.exceptions import *
class Message: #TODO: move actual downloading code into bpkg
"""Superclass for all message sent over pipes.""" #functions here should only contain glue code for facilitating subprocessing of bpkg functionality
class BlenderMessage(Message):
"""Superclass for all messages sent from Blender to the subprocess."""
class SubprocMessage(Message):
"""Superclass for all messages sent from the subprocess to Blender."""
class Abort(BlenderMessage):
"""Sent when the user requests abortion of a task."""
class Progress(SubprocMessage):
"""Send from subprocess to Blender to report progress.
:ivar progress: the progress percentage, from 0-1.
"""
def __init__(self, progress: float):
self.progress = progress
class SubprocError(SubprocMessage):
"""Superclass for all fatal error messages sent from the subprocess."""
def __init__(self, message: str):
self.message = message
class SubprocWarning(SubprocMessage):
"""Superclass for all non-fatal warning messages sent from the subprocess."""
def __init__(self, message: str):
self.message = message
class InstallError(SubprocError):
"""Sent when there was an error installing something."""
class UninstallError(SubprocError):
"""Sent when there was an error uninstalling something."""
class FileConflictError(InstallError):
"""Sent when installation would overwrite existing files."""
def __init__(self, message: str, conflicts: list):
self.message = message
self.conflicts = conflicts
class DownloadError(SubprocMessage):
"""Sent when there was an error downloading something."""
def __init__(self, status_code: int, description: str):
self.status_code = status_code
self.description = description
class Success(SubprocMessage):
"""Sent when an operation finished sucessfully."""
class RepositoryResult(SubprocMessage):
"""Sent when an operation returns a repository to be used on the parent process."""
def __init__(self, repository: dict):
self.repository = repository
class Aborted(SubprocMessage):
"""Sent as response to Abort message."""
class InstallException(Exception):
"""Raised when there is an error during installation"""
class DownloadException(Exception):
"""Raised when there is an error downloading something"""
class BadRepository(Exception):
"""Raised when reading a repository results in an error"""
class InplaceBackup:
"""Utility for moving a file out of the way by appending a '~'"""
log = logging.getLogger('%s.inplace-backup' % __name__)
def __init__(self, path: pathlib.Path):
self.path = path
self.backup()
def backup(self):
"""Move 'path' to 'path~'"""
if not self.path.exists():
raise FileNotFoundError("Can't backup path which doesn't exist")
self.backup_path = pathlib.Path(str(self.path) + '~')
if self.backup_path.exists():
self.log.warning("Overwriting existing backup '{}'".format(self.backup_path))
self._rm(self.backup_path)
shutil.move(str(self.path), str(self.backup_path))
def restore(self):
"""Move 'path~' to 'path'"""
try:
getattr(self, 'backup_path')
except AttributeError as err:
raise RuntimeError("Can't restore file before backing it up") from err
if not self.backup_path.exists():
raise FileNotFoundError("Can't restore backup which doesn't exist")
if self.path.exists():
self.log.warning("Overwriting '{0}' with backup file".format(self.path))
self._rm(self.path)
shutil.move(str(self.backup_path), str(self.path))
def remove(self):
"""Remove 'path~'"""
self._rm(self.backup_path)
def _rm(self, path: pathlib.Path):
"""Just delete whatever is specified by `path`"""
if path.is_dir():
shutil.rmtree(str(path))
else:
path.unlink()
class Package:
"""
Stores package methods and metadata
"""
log = logging.getLogger(__name__ + ".Package")
def __init__(self, package_dict:dict = None):
self.bl_info = {}
self.url = ""
self.files = []
self.set_from_dict(package_dict)
self.installed = False
self.repository = None
def to_dict(self) -> dict:
"""
Return a dict representation of the package
"""
return {
'bl_info': self.bl_info,
'url': self.url,
'files': self.files,
}
def set_from_dict(self, package_dict: dict):
"""
Get attributes from a dict such as produced by `to_dict`
"""
if package_dict is None:
package_dict = {}
for attr in ('files', 'url', 'bl_info'):
if package_dict.get(attr) is not None:
setattr(self, attr, package_dict[attr])
#bl_info convenience getters
@property
def name(self) -> str:
"""Get name from bl_info"""
return self.bl_info['name']
@property
def description(self) -> str:
"""Get description from bl_info"""
return self.bl_info['description']
@property
def version(self) -> tuple:
"""Get version from bl_info"""
return tuple(self.bl_info['version'])
# @classmethod
# def from_dict(cls, package_dict: dict):
# """
# Return a Package with values from dict
# """
# pkg = cls()
# pkg.set_from_dict(package_dict)
@classmethod
def from_blinfo(cls, blinfo: dict):
"""
Return a Package with bl_info filled in
"""
return cls({'bl_info': blinfo})
@classmethod
def from_module(cls, module):
"""
Return a Package object from an addon module
"""
from pathlib import Path
filepath = Path(module.__file__)
if filepath.name == '__init__.py':
filepath = filepath.parent
pkg = cls()
pkg.files = [filepath.name]
pkg.installed_location = str(filepath)
try:
pkg.bl_info = module.bl_info
except AttributeError as err:
raise BadAddon("Module does not appear to be an addon; no bl_info attribute") from err
return pkg
class Repository:
"""
Stores repository metadata (including packages)
"""
log = logging.getLogger(__name__ + ".Repository")
def __init__(self, url=None):
if url is None:
url = ""
self.set_from_dict({'url': url})
# def cleanse_packagelist(self):
# """Remove empty packages (no bl_info), packages with no name"""
def refresh(self):
"""
Requests repo.json from URL and embeds etag/last-modification headers
"""
import requests
if self.url is None:
raise ValueError("Cannot refresh repository without a URL")
self.log.debug("Refreshing repository from %s", self.url)
req_headers = {}
# Do things this way to avoid adding empty objects/None to the req_headers dict
try:
req_headers['If-None-Match'] = self._headers['etag']
except KeyError:
pass
try:
req_headers['If-Modified-Since'] = self._headers['last-modified']
except KeyError:
pass
try:
resp = requests.get(self.url, headers=req_headers, timeout=60)
except requests.exceptions.RequestException as err:
raise DownloadException(err) from err
try:
resp.raise_for_status()
except requests.HTTPError as err:
self.log.error('Error downloading %s: %s', self.url, err)
raise DownloadException(resp.status_code, resp.reason) from err
if resp.status_code == requests.codes.not_modified:
self.log.debug("Packagelist not modified")
return
resp_headers = {}
try:
resp_headers['etag'] = resp.headers['etag']
except KeyError:
pass
try:
resp_headers['last-modified'] = resp.headers['last-modified']
except KeyError:
pass
self.log.debug("Found headers: %s", resp_headers)
try:
repodict = resp.json()
except json.decoder.JSONDecodeError:
self.log.exception("Failed to parse downloaded repository")
raise DownloadException("Could not parse repository downloaded from '%s'. Are you sure this is the correct URL?" % self.url)
repodict['_headers'] = resp_headers
self.set_from_dict(repodict)
def to_dict(self, sort=False, ids=False) -> dict:
"""
Return a dict representation of the repository
"""
packages = [p.to_dict() for p in self.packages]
if sort:
packages.sort(key=lambda p: p['bl_info']['name'].lower())
if ids:
for pkg in packages:
# hash may be too big for a C int
pkg['id'] = str(hash(pkg['url'] + pkg['bl_info']['name'] + self.name + self.url))
return {
'name': self.name,
'packages': packages,
'url': self.url,
'_headers': self._headers,
}
def set_from_dict(self, repodict: dict):
"""
Get repository attributes from a dict such as produced by `to_dict`
"""
def initialize(item, value):
if item is None:
return value
else:
return item
#Be certain to initialize everything; downloaded packagelist might contain null values
name = initialize(repodict.get('name'), "")
url = initialize(repodict.get('url'), "")
packages = initialize(repodict.get('packages'), [])
headers = initialize(repodict.get('_headers'), {})
self.name = name
self.url = url
self.packages = [Package(pkg) for pkg in packages]
self._headers = headers
@classmethod
def from_dict(cls, repodict: dict):
"""
Like `set_from_dict`, but immutable
"""
repo = cls()
repo.set_from_dict(repodict)
return repo
def to_file(self, path: pathlib.Path):
"""
Dump repository to a json file at `path`.
"""
if self.packages is None:
self.log.warning("Writing an empty repository")
with path.open('w', encoding='utf-8') as repo_file:
json.dump(self.to_dict(), repo_file, indent=4, sort_keys=True)
self.log.debug("Repository written to %s" % path)
@classmethod
def from_file(cls, path: pathlib.Path):
"""
Read repository from a json file at `path`.
"""
repo_file = path.open('r', encoding='utf-8')
with repo_file:
try:
repo = cls.from_dict(json.load(repo_file))
except Exception as err:
raise BadRepository from err
cls.log.debug("Repository read from %s", path)
return repo
def _download(pipe_to_blender, package_url: str, download_dir: pathlib.Path) -> pathlib.Path: def _download(pipe_to_blender, package_url: str, download_dir: pathlib.Path) -> pathlib.Path:
"""Downloads the given package """Downloads the given package