Added BAM-packing, requires version of Blender that includes BAM 1.1.1

blender_cloud/blender.py

@@ -181,7 +181,7 @@ class BlenderCloudPreferences(AddonPreferences):
     #       This assumption is true for the Blender Institute, but we should allow other setups too.
     flamenco_job_file_path = StringProperty(
         name='Job file path',
-        description='Path where to store job files',
+        description='Path where to store job files, should be accesible for Workers too',
         subtype='DIR_PATH',
         default='/render/_flamenco/storage')
 
@@ -351,14 +351,15 @@ class BlenderCloudPreferences(AddonPreferences):
         else:
             row_buttons.label('Fetching available managers.')
 
+        path_box = flamenco_box.row(align=True)
+        path_box.prop(self, 'flamenco_job_file_path')
+        path_box.operator('flamenco.open_job_file_path', text='', icon='DISK_DRIVE')
+
         # TODO: make a reusable way to select projects, and use that for Attract and Flamenco.
         note_box = flamenco_box.column(align=True)
         note_box.label('NOTE: For now, Flamenco uses the same project as Attract.')
         note_box.label('This will change in a future version of the add-on.')
 
-        flamenco_box.prop(self, 'flamenco_job_file_path')
-        note_box = flamenco_box.column(align=True)
-        note_box.label('NOTE: Flamenco assumes the workers can use this path too.')
 
 
 class PillarCredentialsUpdate(pillar.PillarOperatorMixin,

blender_cloud/flamenco/__init__.py

@@ -20,7 +20,10 @@
 
 The preferences are managed blender.py, the rest of the Flamenco-specific stuff is here.
 """
+
 import logging
+from pathlib import Path
+import typing
 
 import bpy
 from bpy.types import AddonPreferences, Operator, WindowManager, Scene, PropertyGroup
@@ -116,6 +119,7 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
     """Performs a Blender render on Flamenco."""
     bl_idname = 'flamenco.render'
     bl_label = 'Render on Flamenco'
+    bl_description = __doc__.rstrip('.')
 
     stop_upon_exception = True
     _log = logging.getLogger('bpy.ops.%s' % bl_idname)
@@ -124,36 +128,100 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
         if not await self.authenticate(context):
             return
 
-        import os.path
-        from ..blender import preferences
         from pillarsdk import exceptions as sdk_exceptions
+        from ..blender import preferences
 
+        filepath = Path(context.blend_data.filepath)
+
+        # BAM-pack the files to the destination directory.
+        outfile, missing_sources = await self.bam_pack(filepath)
+        if not outfile:
+            return
+
+        # Create the job at Flamenco Server.
         prefs = preferences()
         scene = context.scene
-
+        settings = {"blender_cmd": "{blender}",
+                    "chunk_size": scene.flamenco_render_chunk_size,
+                    "filepath": str(outfile),
+                    "frames": scene.flamenco_render_frame_range}
         try:
-            await create_job(self.user_id,
+            job_info = await create_job(self.user_id,
                                         prefs.attract_project.project,
                                         prefs.flamenco_manager.manager,
                                         'blender-render',
-                             {
+                                        settings,
-                                 "blender_cmd": "{blender}",
+                                        'Render %s' % filepath.name)
-                                 "chunk_size": scene.flamenco_render_chunk_size,
-                                 "filepath": context.blend_data.filepath,
-                                 "frames": scene.flamenco_render_frame_range
-                             },
-                             'Render %s' % os.path.basename(context.blend_data.filepath))
         except sdk_exceptions.ResourceInvalid as ex:
             self.report({'ERROR'}, 'Error creating Flamenco job: %s' % ex)
+            self.quit()
+            return
+
+        # Store the job ID in a file in the output dir.
+        with open(str(outfile.parent / 'jobinfo.json'), 'w', encoding='utf8') as outfile:
+            import json
+
+            job_info['missing_files'] = [str(mf) for mf in missing_sources]
+            json.dump(job_info, outfile, sort_keys=True, indent=4)
+
+        # Do a final report.
+        if missing_sources:
+            names = (ms.name for ms in missing_sources)
+            self.report({'WARNING'}, 'Flamenco job created with missing files: %s' %
+                        '; '.join(names))
         else:
             self.report({'INFO'}, 'Flamenco job created.')
         self.quit()
 
+    async def bam_pack(self, filepath: Path) -> (typing.Optional[Path], typing.List[Path]):
+        """BAM-packs the blendfile to the destination directory.
+
+        Returns the path of the destination blend file.
+
+        :param filepath: the blend file to pack (i.e. the current blend file)
+        :returns: the destination blend file, or None if there were errors BAM-packing,
+            and a list of missing paths.
+        """
+
+        from datetime import datetime
+        from ..blender import preferences
+        from . import bam_interface
+
+        prefs = preferences()
+
+        # Create a unique directory that is still more or less identifyable.
+        # This should work better than a random ID.
+        # BAM doesn't like output directories that end in '.blend'.
+        unique_dir = '%s-%s-%s' % (datetime.now().isoformat('-').replace(':', ''),
+                                   self.db_user['username'],
+                                   filepath.name.replace('.blend', ''))
+        outdir = Path(prefs.flamenco_job_file_path) / unique_dir
+        outfile = outdir / filepath.name
+
+        try:
+            outdir.mkdir(parents=True)
+        except Exception as ex:
+            self.log.exception('Unable to create output path %s', outdir)
+            self.report({'ERROR'}, 'Unable to create output path: %s' % ex)
+            self.quit()
+            return None, []
+
+        try:
+            missing_sources = await bam_interface.bam_copy(filepath, outfile)
+        except bam_interface.CommandExecutionError as ex:
+            self.log.exception('Unable to execute BAM pack')
+            self.report({'ERROR'}, 'Unable to execute BAM pack: %s' % ex)
+            self.quit()
+            return None, []
+
+        return outfile, missing_sources
+
 
 class FLAMENCO_OT_scene_to_frame_range(Operator):
     """Sets the scene frame range as the Flamenco render frame range."""
     bl_idname = 'flamenco.scene_to_frame_range'
     bl_label = 'Sets the scene frame range as the Flamenco render frame range'
+    bl_description = __doc__.rstrip('.')
 
     def execute(self, context):
         s = context.scene
@@ -161,6 +229,56 @@ class FLAMENCO_OT_scene_to_frame_range(Operator):
         return {'FINISHED'}
 
 
+class FLAMENCO_OT_copy_files(Operator,
+                             async_loop.AsyncModalOperatorMixin):
+    """Uses BAM to copy the current blendfile + dependencies to the target directory."""
+    bl_idname = 'flamenco.copy_files'
+    bl_label = 'Copy files to target'
+    bl_description = __doc__.rstrip('.')
+
+    stop_upon_exception = True
+
+    async def async_execute(self, context):
+        from pathlib import Path
+        from . import bam_interface
+        from ..blender import preferences
+
+        missing_sources = await bam_interface.bam_copy(
+            Path(context.blend_data.filepath),
+            Path(preferences().flamenco_job_file_path),
+        )
+
+        if missing_sources:
+            names = (ms.name for ms in missing_sources)
+            self.report({'ERROR'}, 'Missing source files: %s' % '; '.join(names))
+
+        self.quit()
+
+
+class FLAMENCO_OT_open_job_file_path(Operator):
+    """Opens the Flamenco job storage path in a file explorer."""
+    bl_idname = 'flamenco.open_job_file_path'
+    bl_label = 'Open in file explorer'
+    bl_description = __doc__.rstrip('.')
+
+    def execute(self, context):
+        import platform
+        import subprocess
+        import os
+
+        from ..blender import preferences
+
+        path = preferences().flamenco_job_file_path
+        if platform.system() == "Windows":
+            os.startfile(path)
+        elif platform.system() == "Darwin":
+            subprocess.Popen(["open", path])
+        else:
+            subprocess.Popen(["xdg-open", path])
+
+        return {'FINISHED'}
+
+
 async def create_job(user_id: str,
                      project_id: str,
                      manager_id: str,
@@ -168,8 +286,8 @@ async def create_job(user_id: str,
                      job_settings: dict,
                      job_name: str = None,
                      *,
-                     job_description: str = None) -> str:
+                     job_description: str = None) -> dict:
-    """Creates a render job at Flamenco Server, returning the job ID."""
+    """Creates a render job at Flamenco Server, returning the job object as dictionary."""
 
     import json
     from .sdk import Job
@@ -195,23 +313,38 @@ async def create_job(user_id: str,
     await pillar_call(job.create)
 
     log.info('Job created succesfully: %s', job._id)
-    return job._id
+    return job.to_dict()
 
 
-def draw_render_button(self, context):
+class FLAMENCO_PT_render(bpy.types.Panel):
+    bl_label = "Flamenco Render"
+    bl_space_type = 'PROPERTIES'
+    bl_region_type = 'WINDOW'
+    bl_context = "render"
+    bl_options = {'DEFAULT_CLOSED'}
+
+    def draw(self, context):
         layout = self.layout
 
-    from ..blender import icon
+        from ..blender import preferences
 
-    flamenco_box = layout.box()
+        layout.prop(context.scene, 'flamenco_render_chunk_size')
-    flamenco_box.label('Flamenco', icon_value=icon('CLOUD'))
-    flamenco_box.prop(context.scene, 'flamenco_render_chunk_size')
 
-    frange_row = flamenco_box.row(align=True)
+        labeled_row = layout.split(0.2, align=True)
-    frange_row.prop(context.scene, 'flamenco_render_frame_range')
+        labeled_row.label('Frame range:')
-    frange_row.operator('flamenco.scene_to_frame_range', text='', icon='ARROW_LEFTRIGHT')
+        prop_btn_row = labeled_row.row(align=True)
+        prop_btn_row.prop(context.scene, 'flamenco_render_frame_range', text='')
+        prop_btn_row.operator('flamenco.scene_to_frame_range', text='', icon='ARROW_LEFTRIGHT')
 
-    flamenco_box.operator('flamenco.render', text='Render on Flamenco', icon='RENDER_ANIMATION')
+        labeled_row = layout.split(0.2, align=True)
+        labeled_row.label('Storage:')
+        prop_btn_row = labeled_row.row(align=True)
+        prop_btn_row.label(preferences().flamenco_job_file_path)
+        prop_btn_row.operator(FLAMENCO_OT_open_job_file_path.bl_idname, text='', icon='DISK_DRIVE')
+
+        layout.operator(FLAMENCO_OT_render.bl_idname,
+                              text='Render on Flamenco',
+                              icon='RENDER_ANIMATION')
 
 
 def register():
@@ -219,6 +352,9 @@ def register():
     bpy.utils.register_class(FLAMENCO_OT_fmanagers)
     bpy.utils.register_class(FLAMENCO_OT_render)
     bpy.utils.register_class(FLAMENCO_OT_scene_to_frame_range)
+    bpy.utils.register_class(FLAMENCO_OT_copy_files)
+    bpy.utils.register_class(FLAMENCO_OT_open_job_file_path)
+    bpy.utils.register_class(FLAMENCO_PT_render)
 
     scene = bpy.types.Scene
     scene.flamenco_render_chunk_size = IntProperty(
@@ -231,11 +367,7 @@ def register():
         description='Frames to render, in "printer range" notation'
     )
 
-    bpy.types.RENDER_PT_render.append(draw_render_button)
-
-
 def unregister():
-    bpy.types.RENDER_PT_render.remove(draw_render_button)
     bpy.utils.unregister_module(__name__)
 
     try:

blender_cloud/flamenco/bam_interface.py

@@ -0,0 +1,144 @@
+"""BAM packing interface for Flamenco."""
+
+import logging
+from pathlib import Path
+import typing
+
+# Timeout of the BAM subprocess, in seconds.
+SUBPROC_READLINE_TIMEOUT = 600
+log = logging.getLogger(__name__)
+
+
+class CommandExecutionError(Exception):
+    """Raised when there was an error executing a BAM command."""
+    pass
+
+
+async def bam_copy(base_blendfile: Path, target_blendfile: Path) -> typing.List[Path]:
+    """Uses BAM to copy the given file and dependencies to the target blendfile.
+
+    Due to the way blendfile_pack.py is programmed/structured, we cannot import it
+    and call a function; it has to be run in a subprocess.
+
+    :raises: asyncio.CanceledError if the task was cancelled.
+    :raises: asyncio.TimeoutError if reading a line from the BAM process timed out.
+    :raises: CommandExecutionError if the subprocess failed or output invalid UTF-8.
+    :returns: a list of missing sources; hopefully empty.
+    """
+
+    import asyncio
+    import shlex
+    import subprocess
+
+    import bpy
+    import io_blend_utils
+
+    args = [
+        bpy.app.binary_path_python,
+        '-m', 'bam.pack',
+        '--input', str(base_blendfile),
+        '--output', str(target_blendfile),
+        '--mode', 'FILE',
+    ]
+
+    cmd_to_log = ' '.join(shlex.quote(s) for s in args)
+    log.info('Executing %s', cmd_to_log)
+
+    proc = await asyncio.create_subprocess_exec(
+        *args,
+        env={'PYTHONPATH': io_blend_utils.pythonpath()},
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+
+    missing_sources = []
+
+    try:
+        while not proc.stdout.at_eof():
+            line = await asyncio.wait_for(proc.stdout.readline(),
+                                          SUBPROC_READLINE_TIMEOUT)
+            if not line:
+                # EOF received, so let's bail.
+                break
+
+            try:
+                line = line.decode('utf8')
+            except UnicodeDecodeError as ex:
+                raise CommandExecutionError('Command produced non-UTF8 output, '
+                                            'aborting: %s' % ex)
+
+            line = line.rstrip()
+            if 'source missing:' in line:
+                path = parse_missing_source(line)
+                missing_sources.append(path)
+                log.warning('Source is missing: %s', path)
+
+            log.info('  %s', line)
+    finally:
+        if proc.returncode is None:
+            # Always wait for the process, to avoid zombies.
+            try:
+                proc.kill()
+            except ProcessLookupError:
+                # The process is already stopped, so killing is impossible. That's ok.
+                log.debug("The process was already stopped, aborting is impossible. That's ok.")
+            await proc.wait()
+        log.info('The process stopped with status code %i', proc.returncode)
+
+    if proc.returncode:
+        raise CommandExecutionError('Process stopped with status %i' % proc.returncode)
+
+    return missing_sources
+
+
+def parse_missing_source(line: str) -> Path:
+    r"""Parses a "missing source" line into a pathlib.Path.
+
+    >>> parse_missing_source(r"  source missing: b'D\xc3\xaffficult \xc3\x9cTF-8 filename'")
+    PosixPath('Dïfficult ÜTF-8 filename')
+    >>> parse_missing_source(r"  source missing: b'D\xfffficult Win1252 f\xeflen\xe6me'")
+    PosixPath('D<EFBFBD>fficult Win1252 f<>len<65>me')
+    """
+
+    _, missing_source = line.split(': ', 1)
+    missing_source_as_bytes = parse_byte_literal(missing_source.strip())
+
+    # The file could originate from any platform, so UTF-8 and the current platform's
+    # filesystem encodings are just guesses.
+    try:
+        missing_source = missing_source_as_bytes.decode('utf8')
+    except UnicodeDecodeError:
+        import sys
+        try:
+            missing_source = missing_source_as_bytes.decode(sys.getfilesystemencoding())
+        except UnicodeDecodeError:
+            missing_source = missing_source_as_bytes.decode('ascii', errors='replace')
+
+    path = Path(missing_source)
+
+    return path
+
+
+def parse_byte_literal(bytes_literal: str) -> bytes:
+    r"""Parses a repr(bytes) output into a bytes object.
+
+    >>> parse_byte_literal(r"b'D\xc3\xaffficult \xc3\x9cTF-8 filename'")
+    b'D\xc3\xaffficult \xc3\x9cTF-8 filename'
+    >>> parse_byte_literal(r"b'D\xeffficult Win1252 f\xeflen\xe6me'")
+    b'D\xeffficult Win1252 f\xeflen\xe6me'
+    """
+
+    # Some very basic assertions to make sure we have a proper bytes literal.
+    assert bytes_literal[0] == "b"
+    assert bytes_literal[1] in {'"', "'"}
+    assert bytes_literal[-1] == bytes_literal[1]
+
+    import ast
+    return ast.literal_eval(bytes_literal)
+
+
+if __name__ == '__main__':
+    import doctest
+
+    doctest.testmod()