Flamenco: determine render output path using add-on prefs and filename.

README-flamenco.md

@@ -0,0 +1,57 @@
+# Flamenco
+
+The Blender Cloud add-on has preliminary support for [Flamenco](https://flamenco.io/).
+It requires a project on the [Blender Cloud](https://cloud.blender.org/) that is set up for
+Flamenco, and it requires you to be logged in as a user with rights to use Flamenco.
+
+
+## Quirks
+
+Flamenco support is unpolished, so it has some quirks.
+
+- Project selection happens through the Attract project selector. As a result, you can only
+  select Attract-enabled projects (even when they are not set up for Flamenco). Be careful
+  which project you select.
+- The top level directory of the project is also set through the Attract properties.
+- Defaults are heavily biased for our use in the Blender Institute.
+- Settings that should be project-specific are not, i.e. are regular add-on preferences.
+- Sending a project to Flamenco will check the "File Extensions" setting in the Output panel,
+  and save the blend file to the current filename.
+
+## Render job file locations
+
+Rendering via Flamenco roughly comprises of two steps:
+
+1. Packing the file to render with its dependencies, and placing them in the "job file path".
+2. Rendering, and placing the output files in the "job output path".
+
+### Job file path
+
+The "job file path" consists of the following path components:
+
+1. The add-on preference "job file path", e.g. `/render/_flamenco/storage`
+2. The current date and time, your Blender Cloud username, and the name of the current blend file.
+3. The name of the current blend file.
+
+For example:
+
+`/render/_flamenco/storage/2017-01-18-104841.931387-sybren-03_02_A.layout/03_02_A.layout.blend`
+
+### Job output path
+
+The file path of output files consists of the following path components:
+
+1. The add-on preference "job file path", e.g. `/render/agent327/frames`
+2. The path of the current blend file, relative to the project directory. The first N components
+   of this path can be stripped; when N=1 it turns `scenes/03-searching/03_02_A-snooping/` into
+   `03-searching/03_02_A-snooping/`.
+3. The name of the current blend file, without `.blend`.
+4. The file name depends on the type of output:
+   - When rendering to image files: A 5-digit frame number with the required file extension.
+   - When rendering to a video file: The frame range with the required file extension.
+
+For example:
+
+`/render/agent327/frames/03-searching/03_02_A-snooping/03_02_A.layout/00441.exr`
+
+`/render/agent327/frames/03-searching/03_02_A-snooping/03_02_A.layout/14-51,60-133.mkv`

blender_cloud/blender.py

@@ -26,7 +26,7 @@ import os.path
 
 import bpy
 from bpy.types import AddonPreferences, Operator, WindowManager, Scene, PropertyGroup
-from bpy.props import StringProperty, EnumProperty, PointerProperty, BoolProperty
+from bpy.props import StringProperty, EnumProperty, PointerProperty, BoolProperty, IntProperty
 import rna_prop_ui
 
 from . import pillar, async_loop, flamenco
@@ -178,13 +178,29 @@ class BlenderCloudPreferences(AddonPreferences):
     flamenco_manager = PointerProperty(type=flamenco.FlamencoManagerGroup)
     # TODO: before making Flamenco public, change the defaults to something less Institute-specific.
     # NOTE: The assumption is that the workers can also find the files in the same path.
-    #       This assumption is true for the Blender Institute, but we should allow other setups too.
+    #       This assumption is true for the Blender Institute.
     flamenco_job_file_path = StringProperty(
         name='Job file path',
         description='Path where to store job files, should be accesible for Workers too',
         subtype='DIR_PATH',
         default='/render/_flamenco/storage')
 
+    # TODO: before making Flamenco public, change the defaults to something less Institute-specific.
+    flamenco_job_output_path = StringProperty(
+        name='Job output path',
+        description='Path where to store output files, should be accessible for Workers',
+        subtype='DIR_PATH',
+        default='/render/_flamenco/output')
+    flamenco_job_output_strip_components = IntProperty(
+        name='Job output path strip components',
+        description='The final output path comprises of the job output path, and the blend file '
+                    'path relative to the project with this many path components stripped off '
+                    'the front',
+        min=0,
+        default=0,
+        soft_max=4,
+    )
+
     def draw(self, context):
         import textwrap
 
@@ -271,7 +287,7 @@ class BlenderCloudPreferences(AddonPreferences):
 
         # Flamenco stuff
         flamenco_box = layout.box()
-        self.draw_flamenco_buttons(flamenco_box, self.flamenco_manager)
+        self.draw_flamenco_buttons(flamenco_box, self.flamenco_manager, context)
 
     def draw_subscribe_button(self, layout):
         layout.operator('pillar.subscribe', icon='WORLD')
@@ -331,7 +347,7 @@ class BlenderCloudPreferences(AddonPreferences):
 
         attract_box.prop(self, 'attract_project_local_path')
 
-    def draw_flamenco_buttons(self, flamenco_box, bcp: flamenco.FlamencoManagerGroup):
+    def draw_flamenco_buttons(self, flamenco_box, bcp: flamenco.FlamencoManagerGroup, context):
         flamenco_row = flamenco_box.row(align=True)
         flamenco_row.label('Flamenco', icon_value=icon('CLOUD'))
 
@@ -353,7 +369,25 @@ class BlenderCloudPreferences(AddonPreferences):
 
         path_box = flamenco_box.row(align=True)
         path_box.prop(self, 'flamenco_job_file_path')
-        path_box.operator('flamenco.open_job_file_path', text='', icon='DISK_DRIVE')
+        props = path_box.operator('flamenco.explore_file_path', text='', icon='DISK_DRIVE')
+        props.path = self.flamenco_job_file_path
+
+        job_output_box = flamenco_box.column(align=True)
+        path_box = job_output_box.row(align=True)
+        path_box.prop(self, 'flamenco_job_output_path')
+        props = path_box.operator('flamenco.explore_file_path', text='', icon='DISK_DRIVE')
+        props.path = self.flamenco_job_output_path
+
+        job_output_box.prop(self, 'flamenco_job_output_strip_components',
+                            text='Strip components')
+
+        from .flamenco import render_output_path
+
+        path_box = job_output_box.row(align=True)
+        output_path = render_output_path(context)
+        path_box.label(str(output_path))
+        props = path_box.operator('flamenco.explore_file_path', text='', icon='DISK_DRIVE')
+        props.path = str(output_path.parent)
 
         # TODO: make a reusable way to select projects, and use that for Attract and Flamenco.
         note_box = flamenco_box.column(align=True)
@@ -361,7 +395,6 @@ class BlenderCloudPreferences(AddonPreferences):
         note_box.label('This will change in a future version of the add-on.')
 
 
-
 class PillarCredentialsUpdate(pillar.PillarOperatorMixin,
                               Operator):
     """Updates the Pillar URL and tests the new URL."""

blender_cloud/flamenco/__init__.py

@@ -20,9 +20,9 @@
 
 The preferences are managed blender.py, the rest of the Flamenco-specific stuff is here.
 """
-
+import functools
 import logging
-from pathlib import Path
+from pathlib import Path, PurePath
 import typing
 
 import bpy
@@ -135,6 +135,20 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
         from ..blender import preferences
 
         filepath = Path(context.blend_data.filepath)
+        scene = context.scene
+
+        # The file extension should be determined by the render settings, not necessarily
+        # by the setttings in the output panel.
+        scene.render.use_file_extension = True
+        bpy.ops.wm.save_mainfile()
+
+        # Determine where the render output will be stored.
+        render_output = render_output_path(context)
+        if render_output is None:
+            self.report({'ERROR'}, 'Current file is outside of project path.')
+            self.quit()
+            return
+        self.log.info('Will output render files to %s', render_output)
 
         # BAM-pack the files to the destination directory.
         outfile, missing_sources = await self.bam_pack(filepath)
@@ -145,11 +159,12 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
 
         # Create the job at Flamenco Server.
         prefs = preferences()
-        scene = context.scene
+
         settings = {'blender_cmd': '{blender}',
                     'chunk_size': scene.flamenco_render_chunk_size,
                     'filepath': str(outfile),
                     'frames': scene.flamenco_render_frame_range,
+                    'render_output': str(render_output),
                     }
         try:
             job_info = await create_job(self.user_id,
@@ -205,7 +220,7 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
         # BAM doesn't like output directories that end in '.blend'.
         unique_dir = '%s-%s-%s' % (datetime.now().isoformat('-').replace(':', ''),
                                    self.db_user['username'],
-                                   filepath.name.replace('.blend', ''))
+                                   filepath.stem)
         outdir = Path(prefs.flamenco_job_file_path) / unique_dir
         outfile = outdir / filepath.name
 
@@ -266,26 +281,25 @@ class FLAMENCO_OT_copy_files(Operator,
         self.quit()
 
 
-class FLAMENCO_OT_open_job_file_path(Operator):
+class FLAMENCO_OT_explore_file_path(Operator):
     """Opens the Flamenco job storage path in a file explorer."""
-    bl_idname = 'flamenco.open_job_file_path'
+    bl_idname = 'flamenco.explore_file_path'
     bl_label = 'Open in file explorer'
     bl_description = __doc__.rstrip('.')
 
+    path = StringProperty(name='Path', description='Path to explore', subtype='DIR_PATH')
+
     def execute(self, context):
         import platform
         import subprocess
         import os
 
-        from ..blender import preferences
-
-        path = preferences().flamenco_job_file_path
         if platform.system() == "Windows":
-            os.startfile(path)
+            os.startfile(self.path)
         elif platform.system() == "Darwin":
-            subprocess.Popen(["open", path])
+            subprocess.Popen(["open", self.path])
         else:
-            subprocess.Popen(["xdg-open", path])
+            subprocess.Popen(["xdg-open", self.path])
 
         return {'FINISHED'}
 
@@ -328,6 +342,69 @@ async def create_job(user_id: str,
     return job.to_dict()
 
 
+def is_image_type(render_output_type: str) -> bool:
+    """Determines whether the render output type is an image (True) or video (False)."""
+
+    # This list is taken from rna_scene.c:273, rna_enum_image_type_items.
+    video_types = {'AVI_JPEG', 'AVI_RAW', 'FRAMESERVER', 'FFMPEG', 'QUICKTIME'}
+    return render_output_type not in video_types
+
+
+@functools.lru_cache(1)
+def _render_output_path(
+        local_project_path: str,
+        blend_filepath: str,
+        flamenco_job_output_strip_components: int,
+        flamenco_job_output_path: str,
+        render_image_format: str,
+        flamenco_render_frame_range: str,
+) -> typing.Optional[PurePath]:
+    """Cached version of render_output_path()
+
+    This ensures that redraws of the Flamenco Render and Add-on preferences panels
+    is fast.
+    """
+
+    project_path = Path(bpy.path.abspath(local_project_path)).resolve()
+    blendfile = Path(blend_filepath)
+
+    try:
+        proj_rel = blendfile.parent.relative_to(project_path)
+    except ValueError:
+        log.exception('Current file is outside of project path %s', project_path)
+        return None
+
+    rel_parts = proj_rel.parts[flamenco_job_output_strip_components:]
+    output_top = Path(flamenco_job_output_path)
+    dir_components = output_top.joinpath(*rel_parts) / blendfile.stem
+
+    # Blender will have to append the file extensions by itself.
+    if is_image_type(render_image_format):
+        return dir_components / '#####'
+    return dir_components / flamenco_render_frame_range
+
+
+def render_output_path(context) -> typing.Optional[PurePath]:
+    """Returns the render output path to be sent to Flamenco.
+
+    Returns None when the current blend file is outside the project path.
+    """
+
+    from ..blender import preferences
+
+    scene = context.scene
+    prefs = preferences()
+
+    return _render_output_path(
+        prefs.attract_project_local_path,
+        context.blend_data.filepath,
+        prefs.flamenco_job_output_strip_components,
+        prefs.flamenco_job_output_path,
+        scene.render.image_settings.file_format,
+        scene.flamenco_render_frame_range,
+    )
+
+
 class FLAMENCO_PT_render(bpy.types.Panel):
     bl_label = "Flamenco Render"
     bl_space_type = 'PROPERTIES'
@@ -340,6 +417,8 @@ class FLAMENCO_PT_render(bpy.types.Panel):
 
         from ..blender import preferences
 
+        prefs = preferences()
+
         layout.prop(context.scene, 'flamenco_render_job_priority')
         layout.prop(context.scene, 'flamenco_render_chunk_size')
 
@@ -353,11 +432,23 @@ class FLAMENCO_PT_render(bpy.types.Panel):
         prop_btn_row.prop(context.scene, 'flamenco_render_frame_range', text='')
         prop_btn_row.operator('flamenco.scene_to_frame_range', text='', icon='ARROW_LEFTRIGHT')
 
-        labeled_row = layout.split(0.2, align=True)
+        readonly_stuff = layout.column(align=True)
+        labeled_row = readonly_stuff.split(0.2, align=True)
         labeled_row.label('Storage:')
         prop_btn_row = labeled_row.row(align=True)
-        prop_btn_row.label(preferences().flamenco_job_file_path)
+        prop_btn_row.label(prefs.flamenco_job_file_path)
-        prop_btn_row.operator(FLAMENCO_OT_open_job_file_path.bl_idname, text='', icon='DISK_DRIVE')
+        props = prop_btn_row.operator(FLAMENCO_OT_explore_file_path.bl_idname,
+                                      text='', icon='DISK_DRIVE')
+        props.path = prefs.flamenco_job_file_path
+
+        labeled_row = readonly_stuff.split(0.2, align=True)
+        labeled_row.label('Output:')
+        prop_btn_row = labeled_row.row(align=True)
+        render_output = render_output_path(context)
+        prop_btn_row.label(str(render_output))
+        props = prop_btn_row.operator(FLAMENCO_OT_explore_file_path.bl_idname,
+                                      text='', icon='DISK_DRIVE')
+        props.path = str(render_output.parent)
 
         layout.operator(FLAMENCO_OT_render.bl_idname,
                         text='Render on Flamenco',
@@ -370,7 +461,7 @@ def register():
     bpy.utils.register_class(FLAMENCO_OT_render)
     bpy.utils.register_class(FLAMENCO_OT_scene_to_frame_range)
     bpy.utils.register_class(FLAMENCO_OT_copy_files)
-    bpy.utils.register_class(FLAMENCO_OT_open_job_file_path)
+    bpy.utils.register_class(FLAMENCO_OT_explore_file_path)
     bpy.utils.register_class(FLAMENCO_PT_render)
 
     scene = bpy.types.Scene