diff --git a/blender_cloud/blender.py b/blender_cloud/blender.py index e9f0965..b7cb433 100644 --- a/blender_cloud/blender.py +++ b/blender_cloud/blender.py @@ -406,10 +406,10 @@ class BlenderCloudPreferences(AddonPreferences): # This is only needed when the project is set up for either Attract or Flamenco. project_box.prop(self, 'cloud_project_local_path', - text='Local Cloud Project Path') + text='Local Project Path') def draw_flamenco_buttons(self, flamenco_box, bcp: flamenco.FlamencoManagerGroup, context): - from .flamenco import bam_interface + from .flamenco import bat_interface header_row = flamenco_box.row(align=True) header_row.label('Flamenco:', icon_value=icon('CLOUD')) diff --git a/blender_cloud/flamenco/__init__.py b/blender_cloud/flamenco/__init__.py index 5f492ac..0aad846 100644 --- a/blender_cloud/flamenco/__init__.py +++ b/blender_cloud/flamenco/__init__.py @@ -31,12 +31,12 @@ if "bpy" in locals(): import importlib try: - bam_interface = importlib.reload(bam_interface) + bat_interface = importlib.reload(bat_interface) sdk = importlib.reload(sdk) except NameError: - from . import bam_interface, sdk + from . import bat_interface, sdk else: - from . import bam_interface, sdk + from . import bat_interface, sdk import bpy from bpy.types import AddonPreferences, Operator, WindowManager, Scene, PropertyGroup @@ -181,8 +181,8 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin, return self.log.info('Will output render files to %s', render_output) - # BAM-pack the files to the destination directory. - outfile, missing_sources = await self.bam_pack(filepath) + # BAT-pack the files to the destination directory. + outfile, missing_sources = await self.bat_pack(filepath) if not outfile: return @@ -245,7 +245,7 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin, json.dump(job_info, outfile, sort_keys=True, indent=4) # We can now remove the local copy we made with bpy.ops.wm.save_as_mainfile(). - # Strictly speaking we can already remove it after the BAM-pack, but it may come in + # Strictly speaking we can already remove it after the BAT-pack, but it may come in # handy in case of failures. try: self.log.info('Removing temporary file %s', filepath) @@ -314,13 +314,13 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin, super().quit() bpy.context.window_manager.flamenco_status = 'IDLE' - async def bam_pack(self, filepath: Path) -> (typing.Optional[Path], typing.List[Path]): - """BAM-packs the blendfile to the destination directory. + async def bat_pack(self, filepath: Path) -> (typing.Optional[Path], typing.List[Path]): + """BAT-packs the blendfile to the destination directory. Returns the path of the destination blend file. :param filepath: the blend file to pack (i.e. the current blend file) - :returns: the destination blend file, or None if there were errors BAM-packing, + :returns: the destination blend file, or None if there were errors BAT-packing, and a list of missing paths. """ @@ -331,14 +331,12 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin, # Create a unique directory that is still more or less identifyable. # This should work better than a random ID. - # BAM doesn't like output directories that end in '.blend'. unique_dir = '%s-%s-%s' % (datetime.now().isoformat('-').replace(':', ''), self.db_user['username'], filepath.stem) outdir = Path(prefs.flamenco_job_file_path) / unique_dir - outfile = outdir / filepath.name - - exclusion_filter = prefs.flamenco_exclude_filter or None + projdir = Path(prefs.cloud_project_local_path) + exclusion_filter = (prefs.flamenco_exclude_filter or '').strip() try: outdir.mkdir(parents=True) @@ -349,10 +347,12 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin, return None, [] try: - missing_sources = await bam_interface.bam_copy(filepath, outfile, exclusion_filter) - except bam_interface.CommandExecutionError as ex: - self.log.exception('Unable to execute BAM pack') - self.report({'ERROR'}, 'Unable to execute BAM pack: %s' % ex) + outfile, missing_sources = await bat_interface.bat_copy( + filepath, projdir, outdir, exclusion_filter) + except bat_interface.FileTransferError as ex: + self.log.error('Could not transfer %d files, starting with %s', + len(ex.files_remaining), ex.files_remaining[0]) + self.report({'ERROR'}, 'Unable to transfer %d files' % len(ex.files_remaining)) self.quit() return None, [] @@ -380,7 +380,7 @@ class FLAMENCO_OT_scene_to_frame_range(FlamencoPollMixin, Operator): class FLAMENCO_OT_copy_files(Operator, FlamencoPollMixin, async_loop.AsyncModalOperatorMixin): - """Uses BAM to copy the current blendfile + dependencies to the target directory.""" + """Uses BAT to copy the current blendfile + dependencies to the target directory.""" bl_idname = 'flamenco.copy_files' bl_label = 'Copy files to target' bl_description = __doc__.rstrip('.') @@ -392,11 +392,13 @@ class FLAMENCO_OT_copy_files(Operator, from ..blender import preferences context.window_manager.flamenco_status = 'PACKING' - exclusion_filter = preferences().flamenco_exclude_filter or None + prefs = preferences() + exclusion_filter = (prefs.flamenco_exclude_filter or '').strip() - missing_sources = await bam_interface.bam_copy( + missing_sources = await bat_interface.bat_copy( Path(context.blend_data.filepath), - Path(preferences().flamenco_job_file_path), + Path(prefs.cloud_project_local_path), + Path(prefs.flamenco_job_file_path), exclusion_filter ) @@ -788,7 +790,7 @@ def register(): bpy.types.WindowManager.flamenco_status = EnumProperty( items=[ ('IDLE', 'IDLE', 'Not doing anything.'), - ('PACKING', 'PACKING', 'BAM-packing all dependencies.'), + ('PACKING', 'PACKING', 'BAT-packing all dependencies.'), ('COMMUNICATING', 'COMMUNICATING', 'Communicating with Flamenco Server.'), ], name='flamenco_status', diff --git a/blender_cloud/flamenco/bam_interface.py b/blender_cloud/flamenco/bam_interface.py deleted file mode 100644 index 160ee82..0000000 --- a/blender_cloud/flamenco/bam_interface.py +++ /dev/null @@ -1,185 +0,0 @@ -"""BAM packing interface for Flamenco.""" - -import logging -from pathlib import Path -import typing - -# Timeout of the BAM subprocess, in seconds. -SUBPROC_READLINE_TIMEOUT = 600 -log = logging.getLogger(__name__) - - -class CommandExecutionError(Exception): - """Raised when there was an error executing a BAM command.""" - pass - - -def wheel_pythonpath_278() -> str: - """Returns the value of a PYTHONPATH environment variable needed to run BAM from its wheel file. - - Workaround for Blender 2.78c not having io_blend_utils.pythonpath() - """ - - import os - from ..wheels import wheel_filename - - # Find the wheel to run. - wheelpath = wheel_filename('blender_bam') - - log.info('Using wheel %s to run BAM-Pack', wheelpath) - - # Update the PYTHONPATH to include that wheel. - existing_pypath = os.environ.get('PYTHONPATH', '') - if existing_pypath: - return os.pathsep.join((existing_pypath, wheelpath)) - - return wheelpath - - -async def bam_copy(base_blendfile: Path, target_blendfile: Path, - exclusion_filter: str) -> typing.List[Path]: - """Uses BAM to copy the given file and dependencies to the target blendfile. - - Due to the way blendfile_pack.py is programmed/structured, we cannot import it - and call a function; it has to be run in a subprocess. - - :raises: asyncio.CanceledError if the task was cancelled. - :raises: asyncio.TimeoutError if reading a line from the BAM process timed out. - :raises: CommandExecutionError if the subprocess failed or output invalid UTF-8. - :returns: a list of missing sources; hopefully empty. - """ - - import asyncio - import os - import shlex - import subprocess - - import bpy - import io_blend_utils - - args = [ - bpy.app.binary_path_python, - '-m', 'bam.pack', - '--input', str(base_blendfile), - '--output', str(target_blendfile), - '--mode', 'FILE', - ] - - if exclusion_filter: - args.extend(['--exclude', exclusion_filter]) - - cmd_to_log = ' '.join(shlex.quote(s) for s in args) - log.info('Executing %s', cmd_to_log) - - # Workaround for Blender 2.78c not having io_blend_utils.pythonpath() - if hasattr(io_blend_utils, 'pythonpath'): - pythonpath = io_blend_utils.pythonpath() - else: - pythonpath = wheel_pythonpath_278() - - env = { - 'PYTHONPATH': pythonpath, - # Needed on Windows because http://bugs.python.org/issue8557 - 'PATH': os.environ['PATH'], - } - if 'SYSTEMROOT' in os.environ: # Windows http://bugs.python.org/issue20614 - env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] - - proc = await asyncio.create_subprocess_exec( - *args, - env=env, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - - missing_sources = [] - - try: - while not proc.stdout.at_eof(): - line = await asyncio.wait_for(proc.stdout.readline(), - SUBPROC_READLINE_TIMEOUT) - if not line: - # EOF received, so let's bail. - break - - try: - line = line.decode('utf8') - except UnicodeDecodeError as ex: - raise CommandExecutionError('Command produced non-UTF8 output, ' - 'aborting: %s' % ex) - - line = line.rstrip() - if 'source missing:' in line: - path = parse_missing_source(line) - missing_sources.append(path) - log.warning('Source is missing: %s', path) - - log.info(' %s', line) - finally: - if proc.returncode is None: - # Always wait for the process, to avoid zombies. - try: - proc.kill() - except ProcessLookupError: - # The process is already stopped, so killing is impossible. That's ok. - log.debug("The process was already stopped, aborting is impossible. That's ok.") - await proc.wait() - log.info('The process stopped with status code %i', proc.returncode) - - if proc.returncode: - raise CommandExecutionError('Process stopped with status %i' % proc.returncode) - - return missing_sources - - -def parse_missing_source(line: str) -> Path: - r"""Parses a "missing source" line into a pathlib.Path. - - >>> parse_missing_source(r" source missing: b'D\xc3\xaffficult \xc3\x9cTF-8 filename'") - PosixPath('Dïfficult ÜTF-8 filename') - >>> parse_missing_source(r" source missing: b'D\xfffficult Win1252 f\xeflen\xe6me'") - PosixPath('D�fficult Win1252 f�len�me') - """ - - _, missing_source = line.split(': ', 1) - missing_source_as_bytes = parse_byte_literal(missing_source.strip()) - - # The file could originate from any platform, so UTF-8 and the current platform's - # filesystem encodings are just guesses. - try: - missing_source = missing_source_as_bytes.decode('utf8') - except UnicodeDecodeError: - import sys - try: - missing_source = missing_source_as_bytes.decode(sys.getfilesystemencoding()) - except UnicodeDecodeError: - missing_source = missing_source_as_bytes.decode('ascii', errors='replace') - - path = Path(missing_source) - - return path - - -def parse_byte_literal(bytes_literal: str) -> bytes: - r"""Parses a repr(bytes) output into a bytes object. - - >>> parse_byte_literal(r"b'D\xc3\xaffficult \xc3\x9cTF-8 filename'") - b'D\xc3\xaffficult \xc3\x9cTF-8 filename' - >>> parse_byte_literal(r"b'D\xeffficult Win1252 f\xeflen\xe6me'") - b'D\xeffficult Win1252 f\xeflen\xe6me' - """ - - # Some very basic assertions to make sure we have a proper bytes literal. - assert bytes_literal[0] == "b" - assert bytes_literal[1] in {'"', "'"} - assert bytes_literal[-1] == bytes_literal[1] - - import ast - return ast.literal_eval(bytes_literal) - - -if __name__ == '__main__': - import doctest - - doctest.testmod() diff --git a/blender_cloud/flamenco/bat_interface.py b/blender_cloud/flamenco/bat_interface.py new file mode 100644 index 0000000..4ff0b75 --- /dev/null +++ b/blender_cloud/flamenco/bat_interface.py @@ -0,0 +1,35 @@ +"""BAT🦇 packing interface for Flamenco.""" + +import asyncio +import logging +import typing +from pathlib import Path + +from blender_asset_tracer import pack +from blender_asset_tracer.pack.transfer import FileTransferError + +log = logging.getLogger(__name__) + + +async def bat_copy(base_blendfile: Path, + project: Path, + target: Path, + exclusion_filter: str) -> typing.Tuple[Path, typing.Set[Path]]: + """Use BAT🦇 to copy the given file and dependencies to the target location. + + :raises: FileTransferError if a file couldn't be transferred. + :returns: the path of the packed blend file, and a set of missing sources. + """ + + loop = asyncio.get_event_loop() + + with pack.Packer(base_blendfile, project, target) as packer: + if exclusion_filter: + packer.exclude(*exclusion_filter.split()) + log.debug('awaiting strategise') + await loop.run_in_executor(None, packer.strategise) + log.debug('awaiting execute') + await loop.run_in_executor(None, packer.execute) + log.debug('done') + + return packer.output_path, packer.missing_files diff --git a/blender_cloud/wheels/__init__.py b/blender_cloud/wheels/__init__.py index 24f01d3..77cfef1 100644 --- a/blender_cloud/wheels/__init__.py +++ b/blender_cloud/wheels/__init__.py @@ -61,6 +61,7 @@ def wheel_filename(fname_prefix: str) -> str: def load_wheels(): + load_wheel('blender_asset_tracer', 'blender_asset_tracer') load_wheel('lockfile', 'lockfile') load_wheel('cachecontrol', 'CacheControl') load_wheel('pillarsdk', 'pillarsdk') diff --git a/requirements.txt b/requirements.txt index cfa43be..99943c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ lockfile==0.12.2 pillarsdk==1.6.1 wheel==0.29.0 -blender-bam==1.1.7 +blender-asset-tracer==0.1.dev0 # Secondary requirements: asn1crypto==0.24.0 diff --git a/setup.py b/setup.py index 7b5139b..ce7e229 100755 --- a/setup.py +++ b/setup.py @@ -37,8 +37,7 @@ sys.dont_write_bytecode = True # Download wheels from pypi. The specific versions are taken from requirements.txt wheels = [ - 'lockfile', 'pillarsdk', - 'blender-bam', # for compatibility with Blender 2.78 + 'lockfile', 'pillarsdk', 'blender-asset-tracer', ]