Replaced BAM with BAT

Blender Asset Tracer, or BAT, is a newly written replacement for BAM,
with a nicer API.

blender_cloud/blender.py

@@ -406,10 +406,10 @@ class BlenderCloudPreferences(AddonPreferences):
 
         # This is only needed when the project is set up for either Attract or Flamenco.
         project_box.prop(self, 'cloud_project_local_path',
-                         text='Local Cloud Project Path')
+                         text='Local Project Path')
 
     def draw_flamenco_buttons(self, flamenco_box, bcp: flamenco.FlamencoManagerGroup, context):
-        from .flamenco import bam_interface
+        from .flamenco import bat_interface
 
         header_row = flamenco_box.row(align=True)
         header_row.label('Flamenco:', icon_value=icon('CLOUD'))

blender_cloud/flamenco/__init__.py

@@ -31,12 +31,12 @@ if "bpy" in locals():
     import importlib
 
     try:
-        bam_interface = importlib.reload(bam_interface)
+        bat_interface = importlib.reload(bat_interface)
         sdk = importlib.reload(sdk)
     except NameError:
-        from . import bam_interface, sdk
+        from . import bat_interface, sdk
 else:
-    from . import bam_interface, sdk
+    from . import bat_interface, sdk
 
 import bpy
 from bpy.types import AddonPreferences, Operator, WindowManager, Scene, PropertyGroup
@@ -181,8 +181,8 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
             return
         self.log.info('Will output render files to %s', render_output)
 
-        # BAM-pack the files to the destination directory.
+        # BAT-pack the files to the destination directory.
-        outfile, missing_sources = await self.bam_pack(filepath)
+        outfile, missing_sources = await self.bat_pack(filepath)
         if not outfile:
             return
 
@@ -245,7 +245,7 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
             json.dump(job_info, outfile, sort_keys=True, indent=4)
 
         # We can now remove the local copy we made with bpy.ops.wm.save_as_mainfile().
-        # Strictly speaking we can already remove it after the BAM-pack, but it may come in
+        # Strictly speaking we can already remove it after the BAT-pack, but it may come in
         # handy in case of failures.
         try:
             self.log.info('Removing temporary file %s', filepath)
@@ -314,13 +314,13 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
         super().quit()
         bpy.context.window_manager.flamenco_status = 'IDLE'
 
-    async def bam_pack(self, filepath: Path) -> (typing.Optional[Path], typing.List[Path]):
+    async def bat_pack(self, filepath: Path) -> (typing.Optional[Path], typing.List[Path]):
-        """BAM-packs the blendfile to the destination directory.
+        """BAT-packs the blendfile to the destination directory.
 
         Returns the path of the destination blend file.
 
         :param filepath: the blend file to pack (i.e. the current blend file)
-        :returns: the destination blend file, or None if there were errors BAM-packing,
+        :returns: the destination blend file, or None if there were errors BAT-packing,
             and a list of missing paths.
         """
 
@@ -331,14 +331,12 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
 
         # Create a unique directory that is still more or less identifyable.
         # This should work better than a random ID.
-        # BAM doesn't like output directories that end in '.blend'.
         unique_dir = '%s-%s-%s' % (datetime.now().isoformat('-').replace(':', ''),
                                    self.db_user['username'],
                                    filepath.stem)
         outdir = Path(prefs.flamenco_job_file_path) / unique_dir
-        outfile = outdir / filepath.name
+        projdir = Path(prefs.cloud_project_local_path)
-
+        exclusion_filter = (prefs.flamenco_exclude_filter or '').strip()
-        exclusion_filter = prefs.flamenco_exclude_filter or None
 
         try:
             outdir.mkdir(parents=True)
@@ -349,10 +347,12 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
             return None, []
 
         try:
-            missing_sources = await bam_interface.bam_copy(filepath, outfile, exclusion_filter)
+            outfile, missing_sources = await bat_interface.bat_copy(
-        except bam_interface.CommandExecutionError as ex:
+                filepath, projdir, outdir, exclusion_filter)
-            self.log.exception('Unable to execute BAM pack')
+        except bat_interface.FileTransferError as ex:
-            self.report({'ERROR'}, 'Unable to execute BAM pack: %s' % ex)
+            self.log.error('Could not transfer %d files, starting with %s',
+                           len(ex.files_remaining), ex.files_remaining[0])
+            self.report({'ERROR'}, 'Unable to transfer %d files' % len(ex.files_remaining))
             self.quit()
             return None, []
 
@@ -380,7 +380,7 @@ class FLAMENCO_OT_scene_to_frame_range(FlamencoPollMixin, Operator):
 class FLAMENCO_OT_copy_files(Operator,
                              FlamencoPollMixin,
                              async_loop.AsyncModalOperatorMixin):
-    """Uses BAM to copy the current blendfile + dependencies to the target directory."""
+    """Uses BAT to copy the current blendfile + dependencies to the target directory."""
     bl_idname = 'flamenco.copy_files'
     bl_label = 'Copy files to target'
     bl_description = __doc__.rstrip('.')
@@ -392,11 +392,13 @@ class FLAMENCO_OT_copy_files(Operator,
         from ..blender import preferences
 
         context.window_manager.flamenco_status = 'PACKING'
-        exclusion_filter = preferences().flamenco_exclude_filter or None
+        prefs = preferences()
+        exclusion_filter = (prefs.flamenco_exclude_filter or '').strip()
 
-        missing_sources = await bam_interface.bam_copy(
+        missing_sources = await bat_interface.bat_copy(
             Path(context.blend_data.filepath),
-            Path(preferences().flamenco_job_file_path),
+            Path(prefs.cloud_project_local_path),
+            Path(prefs.flamenco_job_file_path),
             exclusion_filter
         )
 
@@ -788,7 +790,7 @@ def register():
     bpy.types.WindowManager.flamenco_status = EnumProperty(
         items=[
             ('IDLE', 'IDLE', 'Not doing anything.'),
-            ('PACKING', 'PACKING', 'BAM-packing all dependencies.'),
+            ('PACKING', 'PACKING', 'BAT-packing all dependencies.'),
             ('COMMUNICATING', 'COMMUNICATING', 'Communicating with Flamenco Server.'),
         ],
         name='flamenco_status',

blender_cloud/flamenco/bam_interface.py

@@ -1,185 +0,0 @@
-"""BAM packing interface for Flamenco."""
-
-import logging
-from pathlib import Path
-import typing
-
-# Timeout of the BAM subprocess, in seconds.
-SUBPROC_READLINE_TIMEOUT = 600
-log = logging.getLogger(__name__)
-
-
-class CommandExecutionError(Exception):
-    """Raised when there was an error executing a BAM command."""
-    pass
-
-
-def wheel_pythonpath_278() -> str:
-    """Returns the value of a PYTHONPATH environment variable needed to run BAM from its wheel file.
-
-    Workaround for Blender 2.78c not having io_blend_utils.pythonpath()
-    """
-
-    import os
-    from ..wheels import wheel_filename
-
-    # Find the wheel to run.
-    wheelpath = wheel_filename('blender_bam')
-
-    log.info('Using wheel %s to run BAM-Pack', wheelpath)
-
-    # Update the PYTHONPATH to include that wheel.
-    existing_pypath = os.environ.get('PYTHONPATH', '')
-    if existing_pypath:
-        return os.pathsep.join((existing_pypath, wheelpath))
-
-    return wheelpath
-
-
-async def bam_copy(base_blendfile: Path, target_blendfile: Path,
-                   exclusion_filter: str) -> typing.List[Path]:
-    """Uses BAM to copy the given file and dependencies to the target blendfile.
-
-    Due to the way blendfile_pack.py is programmed/structured, we cannot import it
-    and call a function; it has to be run in a subprocess.
-
-    :raises: asyncio.CanceledError if the task was cancelled.
-    :raises: asyncio.TimeoutError if reading a line from the BAM process timed out.
-    :raises: CommandExecutionError if the subprocess failed or output invalid UTF-8.
-    :returns: a list of missing sources; hopefully empty.
-    """
-
-    import asyncio
-    import os
-    import shlex
-    import subprocess
-
-    import bpy
-    import io_blend_utils
-
-    args = [
-        bpy.app.binary_path_python,
-        '-m', 'bam.pack',
-        '--input', str(base_blendfile),
-        '--output', str(target_blendfile),
-        '--mode', 'FILE',
-    ]
-
-    if exclusion_filter:
-        args.extend(['--exclude', exclusion_filter])
-
-    cmd_to_log = ' '.join(shlex.quote(s) for s in args)
-    log.info('Executing %s', cmd_to_log)
-
-    # Workaround for Blender 2.78c not having io_blend_utils.pythonpath()
-    if hasattr(io_blend_utils, 'pythonpath'):
-        pythonpath = io_blend_utils.pythonpath()
-    else:
-        pythonpath = wheel_pythonpath_278()
-
-    env = {
-        'PYTHONPATH': pythonpath,
-        # Needed on Windows because http://bugs.python.org/issue8557
-        'PATH': os.environ['PATH'],
-    }
-    if 'SYSTEMROOT' in os.environ:  # Windows http://bugs.python.org/issue20614
-        env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
-
-    proc = await asyncio.create_subprocess_exec(
-        *args,
-        env=env,
-        stdin=subprocess.DEVNULL,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
-
-    missing_sources = []
-
-    try:
-        while not proc.stdout.at_eof():
-            line = await asyncio.wait_for(proc.stdout.readline(),
-                                          SUBPROC_READLINE_TIMEOUT)
-            if not line:
-                # EOF received, so let's bail.
-                break
-
-            try:
-                line = line.decode('utf8')
-            except UnicodeDecodeError as ex:
-                raise CommandExecutionError('Command produced non-UTF8 output, '
-                                            'aborting: %s' % ex)
-
-            line = line.rstrip()
-            if 'source missing:' in line:
-                path = parse_missing_source(line)
-                missing_sources.append(path)
-                log.warning('Source is missing: %s', path)
-
-            log.info('  %s', line)
-    finally:
-        if proc.returncode is None:
-            # Always wait for the process, to avoid zombies.
-            try:
-                proc.kill()
-            except ProcessLookupError:
-                # The process is already stopped, so killing is impossible. That's ok.
-                log.debug("The process was already stopped, aborting is impossible. That's ok.")
-            await proc.wait()
-        log.info('The process stopped with status code %i', proc.returncode)
-
-    if proc.returncode:
-        raise CommandExecutionError('Process stopped with status %i' % proc.returncode)
-
-    return missing_sources
-
-
-def parse_missing_source(line: str) -> Path:
-    r"""Parses a "missing source" line into a pathlib.Path.
-
-    >>> parse_missing_source(r"  source missing: b'D\xc3\xaffficult \xc3\x9cTF-8 filename'")
-    PosixPath('Dïfficult ÜTF-8 filename')
-    >>> parse_missing_source(r"  source missing: b'D\xfffficult Win1252 f\xeflen\xe6me'")
-    PosixPath('D<EFBFBD>fficult Win1252 f<>len<65>me')
-    """
-
-    _, missing_source = line.split(': ', 1)
-    missing_source_as_bytes = parse_byte_literal(missing_source.strip())
-
-    # The file could originate from any platform, so UTF-8 and the current platform's
-    # filesystem encodings are just guesses.
-    try:
-        missing_source = missing_source_as_bytes.decode('utf8')
-    except UnicodeDecodeError:
-        import sys
-        try:
-            missing_source = missing_source_as_bytes.decode(sys.getfilesystemencoding())
-        except UnicodeDecodeError:
-            missing_source = missing_source_as_bytes.decode('ascii', errors='replace')
-
-    path = Path(missing_source)
-
-    return path
-
-
-def parse_byte_literal(bytes_literal: str) -> bytes:
-    r"""Parses a repr(bytes) output into a bytes object.
-
-    >>> parse_byte_literal(r"b'D\xc3\xaffficult \xc3\x9cTF-8 filename'")
-    b'D\xc3\xaffficult \xc3\x9cTF-8 filename'
-    >>> parse_byte_literal(r"b'D\xeffficult Win1252 f\xeflen\xe6me'")
-    b'D\xeffficult Win1252 f\xeflen\xe6me'
-    """
-
-    # Some very basic assertions to make sure we have a proper bytes literal.
-    assert bytes_literal[0] == "b"
-    assert bytes_literal[1] in {'"', "'"}
-    assert bytes_literal[-1] == bytes_literal[1]
-
-    import ast
-    return ast.literal_eval(bytes_literal)
-
-
-if __name__ == '__main__':
-    import doctest
-
-    doctest.testmod()

blender_cloud/flamenco/bat_interface.py

@@ -0,0 +1,35 @@
+"""BAT🦇 packing interface for Flamenco."""
+
+import asyncio
+import logging
+import typing
+from pathlib import Path
+
+from blender_asset_tracer import pack
+from blender_asset_tracer.pack.transfer import FileTransferError
+
+log = logging.getLogger(__name__)
+
+
+async def bat_copy(base_blendfile: Path,
+                   project: Path,
+                   target: Path,
+                   exclusion_filter: str) -> typing.Tuple[Path, typing.Set[Path]]:
+    """Use BAT🦇 to copy the given file and dependencies to the target location.
+
+    :raises: FileTransferError if a file couldn't be transferred.
+    :returns: the path of the packed blend file, and a set of missing sources.
+    """
+
+    loop = asyncio.get_event_loop()
+
+    with pack.Packer(base_blendfile, project, target) as packer:
+        if exclusion_filter:
+            packer.exclude(*exclusion_filter.split())
+        log.debug('awaiting strategise')
+        await loop.run_in_executor(None, packer.strategise)
+        log.debug('awaiting execute')
+        await loop.run_in_executor(None, packer.execute)
+        log.debug('done')
+
+    return packer.output_path, packer.missing_files

blender_cloud/wheels/__init__.py

@@ -61,6 +61,7 @@ def wheel_filename(fname_prefix: str) -> str:
 
 
 def load_wheels():
+    load_wheel('blender_asset_tracer', 'blender_asset_tracer')
     load_wheel('lockfile', 'lockfile')
     load_wheel('cachecontrol', 'CacheControl')
     load_wheel('pillarsdk', 'pillarsdk')

requirements.txt

@@ -3,7 +3,7 @@
 lockfile==0.12.2
 pillarsdk==1.6.1
 wheel==0.29.0
-blender-bam==1.1.7
+blender-asset-tracer==0.1.dev0
 
 # Secondary requirements:
 asn1crypto==0.24.0

setup.py

@@ -37,8 +37,7 @@ sys.dont_write_bytecode = True
 
 # Download wheels from pypi. The specific versions are taken from requirements.txt
 wheels = [
-    'lockfile', 'pillarsdk',
+    'lockfile', 'pillarsdk', 'blender-asset-tracer',
-    'blender-bam',  # for compatibility with Blender 2.78
 ]