Bumped version to 1.7.3

The sys.platform string is 'win32' even on 64-bit Windows. Furthermore,
we expect 'windows', not 'win32'. platform.system().lower() gives us this.

CHANGELOG.md

@@ -0,0 +1,63 @@
+# Blender Cloud changelog
+
+## Version 1.7.3 (in development)
+
+- Default to scene frame range when no frame range is given.
+- Refuse to render on Flamenco before blend file is saved at least once.
+- Fixed some Windows-specific issues.
+
+
+## Version 1.7.2 (2017-06-22)
+
+- Fixed compatibility with Blender 2.78c.
+
+
+## Version 1.7.1 (2017-06-13)
+
+- Fixed asyncio issues on Windows
+
+
+## Version 1.7.0 (2017-06-09)
+
+- Fixed reloading after upgrading from 1.4.4 (our last public release).
+- Fixed bug handling a symlinked project path.
+- Added support for Manager-defined path replacement variables.
+
+
+## Version 1.6.4 (2017-04-21)
+
+- Added file exclusion filter for Flamenco. A filter like `*.abc;*.mkv;*.mov` can be
+  used to prevent certain files from being copied to the job storage directory.
+  Requires a Blender that is bundled with BAM 1.1.7 or newer.
+
+
+## Version 1.6.3 (2017-03-21)
+
+- Fixed bug where local project path wasn't shown for projects only set up for Flamenco
+  (and not Attract).
+- Added this CHANGELOG.md file, which will contain user-relevant changes.
+
+
+## Version 1.6.2 (2017-03-17)
+
+- Flamenco: when opening non-existing file path, open parent instead
+- Fix T50954: Improve Blender Cloud add-on project selector
+
+
+## Version 1.6.1 (2017-03-07)
+
+- Show error in GUI when Blender Cloud is unreachable
+- Fixed sample count when using branched path tracing
+
+
+## Version 1.6.0 (2017-02-14)
+
+- Default to frame chunk size of 1 (instead of 10).
+- Turn off "use overwrite" and "use placeholder" for Flamenco blend files.
+- Fixed bugs when blendfile is outside the project directory
+
+
+## Older versions
+
+For the history of older versions, please refer to the
+[Git history](https://developer.blender.org/diffusion/BCA/)

blender_cloud/__init__.py

@@ -21,7 +21,7 @@
 bl_info = {
     'name': 'Blender Cloud',
     "author": "Sybren A. Stüvel, Francesco Siddi, Inês Almeida, Antony Riakiotakis",
-    'version': (1, 5, 9999),
+    'version': (1, 7, 3),
     'blender': (2, 77, 0),
     'location': 'Addon Preferences panel, and Ctrl+Shift+Alt+A anywhere for texture browser',
     'description': 'Texture library browser and Blender Sync. Requires the Blender ID addon '
@@ -65,21 +65,28 @@ def register():
 
         def reload_mod(name):
             modname = '%s.%s' % (__name__, name)
-            module = importlib.reload(sys.modules[modname])
-            sys.modules[modname] = module
-            return module
+            try:
+                old_module = sys.modules[modname]
+            except KeyError:
+                # Wasn't loaded before -- can happen after an upgrade.
+                new_module = importlib.import_module(modname)
+            else:
+                new_module = importlib.reload(old_module)
+
+            sys.modules[modname] = new_module
+            return new_module
 
         reload_mod('blendfile')
         reload_mod('home_project')
         reload_mod('utils')
 
-        blender = reload_mod('blender')
         async_loop = reload_mod('async_loop')
+        flamenco = reload_mod('flamenco')
+        attract = reload_mod('attract')
         texture_browser = reload_mod('texture_browser')
         settings_sync = reload_mod('settings_sync')
         image_sharing = reload_mod('image_sharing')
-        attract = reload_mod('attract')
-        flamenco = reload_mod('flamenco')
+        blender = reload_mod('blender')
     else:
         from . import (blender, texture_browser, async_loop, settings_sync, blendfile, home_project,
                        image_sharing, attract, flamenco)
@@ -88,11 +95,13 @@ def register():
     async_loop.register()
 
     flamenco.register()
+    attract.register()
     texture_browser.register()
-    blender.register()
     settings_sync.register()
     image_sharing.register()
-    attract.register()
+    blender.register()
+
+    blender.handle_project_update()
 
 
 def _monkey_patch_requests():

blender_cloud/async_loop.py

@@ -33,17 +33,12 @@ _loop_kicking_operator_running = False
 
 
 def setup_asyncio_executor():
-    """Sets up AsyncIO to run on a single thread.
+    """Sets up AsyncIO to run properly on each platform."""
 
-    This ensures that only one Pillar HTTP call is performed at the same time. Other
-    calls that could be performed in parallel are queued, and thus we can
-    reliably cancel them.
-    """
     import sys
 
-    executor = concurrent.futures.ThreadPoolExecutor()
-
     if sys.platform == 'win32':
+        asyncio.get_event_loop().close()
         # On Windows, the default event loop is SelectorEventLoop, which does
         # not support subprocesses. ProactorEventLoop should be used instead.
         # Source: https://docs.python.org/3/library/asyncio-subprocess.html
@@ -51,9 +46,15 @@ def setup_asyncio_executor():
         asyncio.set_event_loop(loop)
     else:
         loop = asyncio.get_event_loop()
+
+    executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
     loop.set_default_executor(executor)
     # loop.set_debug(True)
 
+    from . import pillar
+    # No more than this many Pillar calls should be made simultaneously
+    pillar.pillar_semaphore = asyncio.Semaphore(3, loop=loop)
+
 
 def kick_async_loop(*args) -> bool:
     """Performs a single iteration of the asyncio event loop.

blender_cloud/attract/__init__.py

@@ -60,6 +60,9 @@ from bpy.types import Operator, Panel, AddonPreferences
 
 log = logging.getLogger(__name__)
 
+# Global flag used to determine whether panels etc. can be drawn.
+attract_is_active = False
+
 
 def active_strip(context):
     try:
@@ -139,6 +142,9 @@ def shot_id_use(strips):
 def compute_strip_conflicts(scene):
     """Sets the strip property atc_object_id_conflict for each strip."""
 
+    if not attract_is_active:
+        return
+
     if not scene or not scene.sequence_editor or not scene.sequence_editor.sequences_all:
         return
 
@@ -161,7 +167,13 @@ def scene_update_post_handler(scene):
     compute_strip_conflicts(scene)
 
 
-class ToolsPanel(Panel):
+class AttractPollMixin:
+    @classmethod
+    def poll(cls, context):
+        return attract_is_active
+
+
+class ToolsPanel(AttractPollMixin, Panel):
     bl_label = 'Attract'
     bl_space_type = 'SEQUENCE_EDITOR'
     bl_region_type = 'UI'
@@ -238,7 +250,7 @@ class ToolsPanel(Panel):
             layout.operator(ATTRACT_OT_submit_all.bl_idname)
 
 
-class AttractOperatorMixin:
+class AttractOperatorMixin(AttractPollMixin):
     """Mix-in class for all Attract operators."""
 
     def _project_needs_setup_error(self):
@@ -261,7 +273,7 @@ class AttractOperatorMixin:
         from .. import pillar, blender
 
         prefs = blender.preferences()
-        project = self.find_project(prefs.attract_project.project)
+        project = self.find_project(prefs.project.project)
 
         # FIXME: Eve doesn't seem to handle the $elemMatch projection properly,
         # even though it works fine in MongoDB itself. As a result, we have to
@@ -295,7 +307,7 @@ class AttractOperatorMixin:
                                'cut_in_timeline_in_frames': strip.frame_final_start},
                 'order': 0,
                 'node_type': 'attract_shot',
-                'project': blender.preferences().attract_project.project,
+                'project': blender.preferences().project.project,
                 'user': user_uuid}
 
         # Create a Node item with the attract API
@@ -373,7 +385,7 @@ class AttractShotFetchUpdate(AttractOperatorMixin, Operator):
 
     @classmethod
     def poll(cls, context):
-        return any(selected_shots(context))
+        return AttractOperatorMixin.poll(context) and any(selected_shots(context))
 
     def execute(self, context):
         for strip in selected_shots(context):
@@ -393,6 +405,9 @@ class AttractShotRelink(AttractShotFetchUpdate):
 
     @classmethod
     def poll(cls, context):
+        if not AttractOperatorMixin.poll(context):
+            return False
+
         strip = active_strip(context)
         return strip is not None and not getattr(strip, 'atc_object_id', None)
 
@@ -433,7 +448,8 @@ class ATTRACT_OT_shot_open_in_browser(AttractOperatorMixin, Operator):
 
     @classmethod
     def poll(cls, context):
-        return bool(context.selected_sequences and active_strip(context))
+        return AttractOperatorMixin.poll(context) and \
+               bool(context.selected_sequences and active_strip(context))
 
     def execute(self, context):
         from ..blender import PILLAR_WEB_SERVER_URL
@@ -459,7 +475,8 @@ class AttractShotDelete(AttractOperatorMixin, Operator):
 
     @classmethod
     def poll(cls, context):
-        return bool(context.selected_sequences)
+        return AttractOperatorMixin.poll(context) and \
+               bool(context.selected_sequences)
 
     def execute(self, context):
         from .. import pillar
@@ -511,7 +528,8 @@ class AttractStripUnlink(AttractOperatorMixin, Operator):
 
     @classmethod
     def poll(cls, context):
-        return bool(context.selected_sequences)
+        return AttractOperatorMixin.poll(context) and \
+               bool(context.selected_sequences)
 
     def execute(self, context):
         unlinked_ids = set()
@@ -554,7 +572,8 @@ class AttractShotSubmitSelected(AttractOperatorMixin, Operator):
 
     @classmethod
     def poll(cls, context):
-        return bool(context.selected_sequences)
+        return AttractOperatorMixin.poll(context) and \
+               bool(context.selected_sequences)
 
     def execute(self, context):
         # Check that the project is set up for Attract.
@@ -610,7 +629,8 @@ class ATTRACT_OT_open_meta_blendfile(AttractOperatorMixin, Operator):
 
     @classmethod
     def poll(cls, context):
-        return bool(any(cls.filename_from_metadata(s) for s in context.selected_sequences))
+        return AttractOperatorMixin.poll(context) and \
+               bool(any(cls.filename_from_metadata(s) for s in context.selected_sequences))
 
     @staticmethod
     def filename_from_metadata(strip):
@@ -677,7 +697,7 @@ class ATTRACT_OT_make_shot_thumbnail(AttractOperatorMixin,
 
     @classmethod
     def poll(cls, context):
-        return bool(context.selected_sequences)
+        return AttractOperatorMixin.poll(context) and bool(context.selected_sequences)
 
     @contextlib.contextmanager
     def thumbnail_render_settings(self, context, thumbnail_width=512):
@@ -843,7 +863,7 @@ class ATTRACT_OT_make_shot_thumbnail(AttractOperatorMixin,
         from .. import blender
 
         prefs = blender.preferences()
-        project = self.find_project(prefs.attract_project.project)
+        project = self.find_project(prefs.project.project)
 
         self.log.info('Uploading file %s', filename)
         resp = await pillar.pillar_call(
@@ -873,7 +893,8 @@ class ATTRACT_OT_copy_id_to_clipboard(AttractOperatorMixin, Operator):
 
     @classmethod
     def poll(cls, context):
-        return bool(context.selected_sequences and active_strip(context))
+        return AttractOperatorMixin.poll(context) and \
+               bool(context.selected_sequences and active_strip(context))
 
     def execute(self, context):
         strip = active_strip(context)
@@ -905,6 +926,29 @@ def draw_strip_movie_meta(self, context):
     box.label('Original Frame Range: %s-%s' % (sfra, efra))
 
 
+def activate():
+    global attract_is_active
+
+    log.info('Activating Attract')
+    attract_is_active = True
+    bpy.app.handlers.scene_update_post.append(scene_update_post_handler)
+    draw.callback_enable()
+
+
+def deactivate():
+    global attract_is_active
+
+    log.info('Deactivating Attract')
+    attract_is_active = False
+    draw.callback_disable()
+
+    try:
+        bpy.app.handlers.scene_update_post.remove(scene_update_post_handler)
+    except ValueError:
+        # This is thrown when scene_update_post_handler does not exist in the handler list.
+        pass
+
+
 def register():
     bpy.types.Sequence.atc_is_synced = bpy.props.BoolProperty(name="Is Synced")
     bpy.types.Sequence.atc_object_id = bpy.props.StringProperty(name="Attract Object ID")
@@ -942,13 +986,9 @@ def register():
     bpy.utils.register_class(ATTRACT_OT_make_shot_thumbnail)
     bpy.utils.register_class(ATTRACT_OT_copy_id_to_clipboard)
 
-    bpy.app.handlers.scene_update_post.append(scene_update_post_handler)
-    draw.callback_enable()
-
 
 def unregister():
-    draw.callback_disable()
-    bpy.app.handlers.scene_update_post.remove(scene_update_post_handler)
+    deactivate()
     bpy.utils.unregister_module(__name__)
     del bpy.types.Sequence.atc_is_synced
     del bpy.types.Sequence.atc_object_id

blender_cloud/attract/draw.py

@@ -172,6 +172,11 @@ def callback_disable():
     if not cb_handle:
         return
 
-    bpy.types.SpaceSequenceEditor.draw_handler_remove(cb_handle[0], 'WINDOW')
+    try:
+        bpy.types.SpaceSequenceEditor.draw_handler_remove(cb_handle[0], 'WINDOW')
+    except ValueError:
+        # Thrown when already removed.
+        pass
+    cb_handle.clear()
 
     tag_redraw_all_sequencer_editors()

blender_cloud/blender.py

@@ -20,7 +20,7 @@
 
 Separated from __init__.py so that we can import & run from non-Blender environments.
 """
-
+import functools
 import logging
 import os.path
 
@@ -32,8 +32,7 @@ import rna_prop_ui
 from . import pillar, async_loop, flamenco
 from .utils import pyside_cache, redraw
 
-PILLAR_WEB_SERVER_URL = 'https://cloud.blender.org/'
-# PILLAR_WEB_SERVER_URL = 'http://pillar-web:5001/'
+PILLAR_WEB_SERVER_URL = os.environ.get('BCLOUD_SERVER', 'https://cloud.blender.org/')
 PILLAR_SERVER_URL = '%sapi/' % PILLAR_WEB_SERVER_URL
 
 ADDON_NAME = 'blender_cloud'
@@ -109,13 +108,61 @@ class SyncStatusProperties(PropertyGroup):
 def bcloud_available_projects(self, context):
     """Returns the list of items used by BlenderCloudProjectGroup.project EnumProperty."""
 
-    attr_proj = preferences().attract_project
-    projs = attr_proj.available_projects
+    projs = preferences().project.available_projects
     if not projs:
         return [('', 'No projects available in your Blender Cloud', '')]
     return [(p['_id'], p['name'], '') for p in projs]
 
 
+@functools.lru_cache(1)
+def project_extensions(project_id) -> set:
+    """Returns the extensions the project is enabled for.
+
+    At the moment of writing these are 'attract' and 'flamenco'.
+    """
+
+    log.debug('Finding extensions for project %s', project_id)
+
+    # We can't use our @property, since the preferences may be loaded from a
+    # preferences blend file, in which case it is not constructed from Python code.
+    available_projects = preferences().project.get('available_projects', [])
+    if not available_projects:
+        log.debug('No projects available.')
+        return set()
+
+    proj = next((p for p in available_projects
+                 if p['_id'] == project_id), None)
+    if proj is None:
+        log.debug('Project %s not found in available projects.', project_id)
+        return set()
+
+    return set(proj.get('enabled_for', ()))
+
+
+def handle_project_update(_=None, _2=None):
+    """Handles changing projects, which may cause extensions to be disabled/enabled.
+
+    Ignores arguments so that it can be used as property update callback.
+    """
+
+    project_id = preferences().project.project
+    log.info('Updating internal state to reflect extensions enabled on current project %s.',
+             project_id)
+
+    project_extensions.cache_clear()
+
+    from blender_cloud import attract, flamenco
+    attract.deactivate()
+    flamenco.deactivate()
+
+    enabled_for = project_extensions(project_id)
+    log.info('Project extensions: %s', enabled_for)
+    if 'attract' in enabled_for:
+        attract.activate()
+    if 'flamenco' in enabled_for:
+        flamenco.activate()
+
+
 class BlenderCloudProjectGroup(PropertyGroup):
     status = EnumProperty(
         items=[
@@ -129,7 +176,9 @@ class BlenderCloudProjectGroup(PropertyGroup):
     project = EnumProperty(
         items=bcloud_available_projects,
         name='Cloud project',
-        description='Which Blender Cloud project to work with')
+        description='Which Blender Cloud project to work with',
+        update=handle_project_update
+    )
 
     # List of projects is stored in 'available_projects' ID property,
     # because I don't know how to store a variable list of strings in a proper RNA property.
@@ -140,6 +189,7 @@ class BlenderCloudProjectGroup(PropertyGroup):
     @available_projects.setter
     def available_projects(self, new_projects):
         self['available_projects'] = new_projects
+        handle_project_update()
 
 
 class BlenderCloudPreferences(AddonPreferences):
@@ -155,44 +205,50 @@ class BlenderCloudPreferences(AddonPreferences):
     )
 
     local_texture_dir = StringProperty(
-        name='Default Blender Cloud texture storage directory',
+        name='Default Blender Cloud Texture Storage Directory',
         subtype='DIR_PATH',
         default='//textures')
 
     open_browser_after_share = BoolProperty(
-        name='Open browser after sharing file',
+        name='Open Browser after Sharing File',
         description='When enabled, Blender will open a webbrowser',
         default=True
     )
 
-    # TODO: store local path with the Attract project, so that people
-    # can switch projects and the local path switches with it.
-    attract_project = PointerProperty(type=BlenderCloudProjectGroup)
-    attract_project_local_path = StringProperty(
-        name='Local project path',
+    # TODO: store project-dependent properties with the project, so that people
+    # can switch projects and the Attract and Flamenco properties switch with it.
+    project = PointerProperty(type=BlenderCloudProjectGroup)
+
+    cloud_project_local_path = StringProperty(
+        name='Local Project Path',
         description='Local path of your Attract project, used to search for blend files; '
                     'usually best to set to an absolute path',
         subtype='DIR_PATH',
         default='//../')
 
     flamenco_manager = PointerProperty(type=flamenco.FlamencoManagerGroup)
+    flamenco_exclude_filter = StringProperty(
+        name='File Exclude Filter',
+        description='Filter like "*.abc;*.mkv" to prevent certain files to be packed '
+                    'into the output directory',
+        default='')
     # TODO: before making Flamenco public, change the defaults to something less Institute-specific.
     # NOTE: The assumption is that the workers can also find the files in the same path.
     #       This assumption is true for the Blender Institute.
     flamenco_job_file_path = StringProperty(
-        name='Job file path',
+        name='Job File Path',
         description='Path where to store job files, should be accesible for Workers too',
         subtype='DIR_PATH',
         default='/render/_flamenco/storage')
 
     # TODO: before making Flamenco public, change the defaults to something less Institute-specific.
     flamenco_job_output_path = StringProperty(
-        name='Job output path',
+        name='Job Output Path',
         description='Path where to store output files, should be accessible for Workers',
         subtype='DIR_PATH',
         default='/render/_flamenco/output')
     flamenco_job_output_strip_components = IntProperty(
-        name='Job output path strip components',
+        name='Job Output Path Strip Components',
         description='The final output path comprises of the job output path, and the blend file '
                     'path relative to the project with this many path components stripped off '
                     'the front',
@@ -200,6 +256,11 @@ class BlenderCloudPreferences(AddonPreferences):
         default=0,
         soft_max=4,
     )
+    flamenco_open_browser_after_submit = BoolProperty(
+        name='Open Browser after Submitting Job',
+        description='When enabled, Blender will open a webbrowser',
+        default=True
+    )
 
     def draw(self, context):
         import textwrap
@@ -214,8 +275,8 @@ class BlenderCloudPreferences(AddonPreferences):
             blender_id_profile = None
         else:
             blender_id_profile = blender_id.get_active_profile()
-
         if blender_id is None:
+
             msg_icon = 'ERROR'
             text = 'This add-on requires Blender ID'
             help_text = 'Make sure that the Blender ID add-on is installed and activated'
@@ -281,13 +342,17 @@ class BlenderCloudPreferences(AddonPreferences):
         share_box.label('Image Sharing on Blender Cloud', icon_value=icon('CLOUD'))
         share_box.prop(self, 'open_browser_after_share')
 
-        # Attract stuff
-        attract_box = layout.box()
-        self.draw_attract_buttons(attract_box, self.attract_project)
+        # Project selector
+        project_box = layout.box()
+        project_box.enabled = self.project.status in {'NONE', 'IDLE'}
+
+        self.draw_project_selector(project_box, self.project)
+        extensions = project_extensions(self.project.project)
 
         # Flamenco stuff
-        flamenco_box = layout.box()
-        self.draw_flamenco_buttons(flamenco_box, self.flamenco_manager, context)
+        if 'flamenco' in extensions:
+            flamenco_box = project_box.column()
+            self.draw_flamenco_buttons(flamenco_box, self.flamenco_manager, context)
 
     def draw_subscribe_button(self, layout):
         layout.operator('pillar.subscribe', icon='WORLD')
@@ -323,12 +388,11 @@ class BlenderCloudPreferences(AddonPreferences):
         else:
             row_pull.label('Cloud Sync is running.')
 
-    def draw_attract_buttons(self, attract_box, bcp: BlenderCloudProjectGroup):
-        attract_row = attract_box.row(align=True)
-        attract_row.label('Attract', icon_value=icon('CLOUD'))
+    def draw_project_selector(self, project_box, bcp: BlenderCloudProjectGroup):
+        project_row = project_box.row(align=True)
+        project_row.label('Project settings', icon_value=icon('CLOUD'))
 
-        attract_row.enabled = bcp.status in {'NONE', 'IDLE'}
-        row_buttons = attract_row.row(align=True)
+        row_buttons = project_row.row(align=True)
 
         projects = bcp.available_projects
         project = bcp.project
@@ -345,54 +409,77 @@ class BlenderCloudPreferences(AddonPreferences):
         else:
             row_buttons.label('Fetching available projects.')
 
-        attract_box.prop(self, 'attract_project_local_path')
+        enabled_for = project_extensions(project)
+        if not project:
+            return
+
+        if not enabled_for:
+            project_box.label('This project is not set up for Attract or Flamenco')
+            return
+
+        project_box.label('This project is set up for: %s' %
+                          ', '.join(sorted(enabled_for)))
+
+        # This is only needed when the project is set up for either Attract or Flamenco.
+        project_box.prop(self, 'cloud_project_local_path',
+                         text='Local Cloud Project Path')
 
     def draw_flamenco_buttons(self, flamenco_box, bcp: flamenco.FlamencoManagerGroup, context):
-        flamenco_row = flamenco_box.row(align=True)
-        flamenco_row.label('Flamenco', icon_value=icon('CLOUD'))
+        from .flamenco import bam_interface
 
-        flamenco_row.enabled = bcp.status in {'NONE', 'IDLE'}
-        row_buttons = flamenco_row.row(align=True)
+        header_row = flamenco_box.row(align=True)
+        header_row.label('Flamenco:', icon_value=icon('CLOUD'))
+
+        manager_split = flamenco_box.split(0.32, align=True)
+        manager_split.label('Manager:')
+        manager_box = manager_split.row(align=True)
 
         if bcp.status in {'NONE', 'IDLE'}:
             if not bcp.available_managers or not bcp.manager:
-                row_buttons.operator('flamenco.managers',
+                manager_box.operator('flamenco.managers',
                                      text='Find Flamenco Managers',
                                      icon='FILE_REFRESH')
             else:
-                row_buttons.prop(bcp, 'manager', text='Manager')
-                row_buttons.operator('flamenco.managers',
+                manager_box.prop(bcp, 'manager', text='')
+                manager_box.operator('flamenco.managers',
                                      text='',
                                      icon='FILE_REFRESH')
         else:
-            row_buttons.label('Fetching available managers.')
+            manager_box.label('Fetching available managers.')
 
-        path_box = flamenco_box.row(align=True)
-        path_box.prop(self, 'flamenco_job_file_path')
+        path_split = flamenco_box.split(0.32, align=True)
+        path_split.label(text='Job File Path:')
+        path_box = path_split.row(align=True)
+        path_box.prop(self, 'flamenco_job_file_path', text='')
         props = path_box.operator('flamenco.explore_file_path', text='', icon='DISK_DRIVE')
         props.path = self.flamenco_job_file_path
 
         job_output_box = flamenco_box.column(align=True)
-        path_box = job_output_box.row(align=True)
-        path_box.prop(self, 'flamenco_job_output_path')
+        path_split = job_output_box.split(0.32, align=True)
+        path_split.label(text='Job Output Path:')
+        path_box = path_split.row(align=True)
+        path_box.prop(self, 'flamenco_job_output_path', text='')
         props = path_box.operator('flamenco.explore_file_path', text='', icon='DISK_DRIVE')
         props.path = self.flamenco_job_output_path
+        job_output_box.prop(self, 'flamenco_exclude_filter')
 
-        job_output_box.prop(self, 'flamenco_job_output_strip_components',
-                            text='Strip components')
+        prop_split = job_output_box.split(0.32, align=True)
+        prop_split.label('Strip Components:')
+        prop_split.prop(self, 'flamenco_job_output_strip_components', text='')
 
         from .flamenco import render_output_path
 
         path_box = job_output_box.row(align=True)
         output_path = render_output_path(context)
-        path_box.label(str(output_path))
-        props = path_box.operator('flamenco.explore_file_path', text='', icon='DISK_DRIVE')
-        props.path = str(output_path.parent)
+        if output_path:
+            path_box.label(str(output_path))
+            props = path_box.operator('flamenco.explore_file_path', text='', icon='DISK_DRIVE')
+            props.path = str(output_path.parent)
+        else:
+            path_box.label('Blend file is not in your project path, '
+                           'unable to give output path example.')
 
-        # TODO: make a reusable way to select projects, and use that for Attract and Flamenco.
-        note_box = flamenco_box.column(align=True)
-        note_box.label('NOTE: For now, Flamenco uses the same project as Attract.')
-        note_box.label('This will change in a future version of the add-on.')
+        flamenco_box.prop(self, 'flamenco_open_browser_after_submit')
 
 
 class PillarCredentialsUpdate(pillar.PillarOperatorMixin,
@@ -477,7 +564,7 @@ class PILLAR_OT_projects(async_loop.AsyncModalOperatorMixin,
 
         self.log.info('Going to fetch projects for user %s', self.user_id)
 
-        preferences().attract_project.status = 'FETCHING'
+        preferences().project.status = 'FETCHING'
 
         # Get all projects, except the home project.
         projects_user = await pillar_call(
@@ -486,7 +573,8 @@ class PILLAR_OT_projects(async_loop.AsyncModalOperatorMixin,
                        'category': {'$ne': 'home'}},
              'sort': '-_created',
              'projection': {'_id': True,
-                            'name': True},
+                            'name': True,
+                            'extension_props': True},
              })
 
         projects_shared = await pillar_call(
@@ -495,20 +583,34 @@ class PILLAR_OT_projects(async_loop.AsyncModalOperatorMixin,
                        'permissions.groups.group': {'$in': self.db_user.groups}},
              'sort': '-_created',
              'projection': {'_id': True,
-                            'name': True},
+                            'name': True,
+                            'extension_props': True},
              })
 
         # We need to convert to regular dicts before storing in ID properties.
         # Also don't store more properties than we need.
-        projects = [{'_id': p['_id'], 'name': p['name']} for p in projects_user['_items']] + \
-                   [{'_id': p['_id'], 'name': p['name']} for p in projects_shared['_items']]
+        def reduce_properties(project_list):
+            for p in project_list:
+                p = p.to_dict()
+                extension_props = p.get('extension_props', {})
+                enabled_for = list(extension_props.keys())
 
-        preferences().attract_project.available_projects = projects
+                self._log.debug('Project %r is enabled for %s', p['name'], enabled_for)
+                yield {
+                    '_id': p['_id'],
+                    'name': p['name'],
+                    'enabled_for': enabled_for,
+                }
+
+        projects = list(reduce_properties(projects_user['_items'])) + \
+                   list(reduce_properties(projects_shared['_items']))
+
+        preferences().project.available_projects = projects
 
         self.quit()
 
     def quit(self):
-        preferences().attract_project.status = 'IDLE'
+        preferences().project.status = 'IDLE'
         super().quit()
 
 

blender_cloud/flamenco/__init__.py

@@ -20,11 +20,26 @@
 
 The preferences are managed blender.py, the rest of the Flamenco-specific stuff is here.
 """
+
 import functools
 import logging
+import os
 from pathlib import Path, PurePath
 import typing
 
+
+if "bpy" in locals():
+    import importlib
+
+    try:
+        bam_interface = importlib.reload(bam_interface)
+        sdk = importlib.reload(sdk)
+    except NameError:
+        from . import bam_interface, sdk
+else:
+    from . import bam_interface, sdk
+
+
 import bpy
 from bpy.types import AddonPreferences, Operator, WindowManager, Scene, PropertyGroup
 from bpy.props import StringProperty, EnumProperty, PointerProperty, BoolProperty, IntProperty
@@ -32,8 +47,12 @@ from bpy.props import StringProperty, EnumProperty, PointerProperty, BoolPropert
 from .. import async_loop, pillar
 from ..utils import pyside_cache, redraw
 
+
 log = logging.getLogger(__name__)
 
+# Global flag used to determine whether panels etc. can be drawn.
+flamenco_is_active = False
+
 
 @pyside_cache('manager')
 def available_managers(self, context):
@@ -73,8 +92,15 @@ class FlamencoManagerGroup(PropertyGroup):
         self['available_managers'] = new_managers
 
 
+class FlamencoPollMixin:
+    @classmethod
+    def poll(cls, context):
+        return flamenco_is_active
+
+
 class FLAMENCO_OT_fmanagers(async_loop.AsyncModalOperatorMixin,
                             pillar.AuthenticatedPillarOperatorMixin,
+                            FlamencoPollMixin,
                             Operator):
     """Fetches the Flamenco Managers available to the user"""
     bl_idname = 'flamenco.managers'
@@ -115,6 +141,7 @@ class FLAMENCO_OT_fmanagers(async_loop.AsyncModalOperatorMixin,
 
 class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
                          pillar.AuthenticatedPillarOperatorMixin,
+                         FlamencoPollMixin,
                          Operator):
     """Performs a Blender render on Flamenco."""
     bl_idname = 'flamenco.render'
@@ -125,25 +152,30 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
     log = logging.getLogger('%s.FLAMENCO_OT_render' % __name__)
 
     async def async_execute(self, context):
+        # Refuse to start if the file hasn't been saved. It's okay if
+        # it's dirty, but we do need a filename and a location.
+        if not os.path.exists(context.blend_data.filepath):
+            self.report({'ERROR'}, 'Please save your Blend file before using '
+                                   'the Blender Cloud addon.')
+            self.quit()
+            return
+
         if not await self.authenticate(context):
             return
 
-        context.window_manager.progress_begin(0, 4)
-        context.window_manager.progress_update(1)
-
-        from pillarsdk import exceptions as sdk_exceptions
+        import pillarsdk.exceptions
+        from .sdk import Manager
+        from ..pillar import pillar_call
         from ..blender import preferences
 
-        filepath = Path(context.blend_data.filepath)
         scene = context.scene
 
-        # The file extension should be determined by the render settings, not necessarily
-        # by the setttings in the output panel.
-        scene.render.use_file_extension = True
-        bpy.ops.wm.save_mainfile()
+        # Save to a different file, specifically for Flamenco.
+        context.window_manager.flamenco_status = 'PACKING'
+        filepath = await self._save_blendfile(context)
 
         # Determine where the render output will be stored.
-        render_output = render_output_path(context)
+        render_output = render_output_path(context, filepath)
         if render_output is None:
             self.report({'ERROR'}, 'Current file is outside of project path.')
             self.quit()
@@ -155,26 +187,53 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
         if not outfile:
             return
 
-        context.window_manager.progress_update(3)
-
-        # Create the job at Flamenco Server.
+        # Fetch Manager for doing path replacement.
+        self.log.info('Going to fetch manager %s', self.user_id)
         prefs = preferences()
 
+        manager_id = prefs.flamenco_manager.manager
+        try:
+            manager = await pillar_call(Manager.find, manager_id)
+        except pillarsdk.exceptions.ResourceNotFound:
+            self.report({'ERROR'}, 'Manager %s not found, refresh your managers in '
+                                   'the Blender Cloud add-on settings.' % manager_id)
+            self.quit()
+            return
+
+        # Create the job at Flamenco Server.
+        context.window_manager.flamenco_status = 'COMMUNICATING'
+
+        frame_range = scene.flamenco_render_frame_range.strip() or scene_frame_range(context)
         settings = {'blender_cmd': '{blender}',
-                    'chunk_size': scene.flamenco_render_chunk_size,
-                    'filepath': str(outfile),
-                    'frames': scene.flamenco_render_frame_range,
-                    'render_output': str(render_output),
+                    'chunk_size': scene.flamenco_render_fchunk_size,
+                    'filepath': manager.replace_path(outfile),
+                    'frames': frame_range,
+                    'render_output': manager.replace_path(render_output),
                     }
+
+        # Add extra settings specific to the job type
+        if scene.flamenco_render_job_type == 'blender-render-progressive':
+            if scene.cycles.progressive == 'BRANCHED_PATH':
+                samples = scene.cycles.aa_samples
+            else:
+                samples = scene.cycles.samples
+
+            if scene.cycles.use_square_samples:
+                samples **= 2
+
+            settings['cycles_num_chunks'] = scene.flamenco_render_schunk_count
+            settings['cycles_sample_count'] = samples
+            settings['format'] = 'EXR'
+
         try:
             job_info = await create_job(self.user_id,
-                                        prefs.attract_project.project,
-                                        prefs.flamenco_manager.manager,
+                                        prefs.project.project,
+                                        manager_id,
                                         scene.flamenco_render_job_type,
                                         settings,
                                         'Render %s' % filepath.name,
                                         priority=scene.flamenco_render_job_priority)
-        except sdk_exceptions.ResourceInvalid as ex:
+        except Exception as ex:
             self.report({'ERROR'}, 'Error creating Flamenco job: %s' % ex)
             self.quit()
             return
@@ -186,6 +245,25 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
             job_info['missing_files'] = [str(mf) for mf in missing_sources]
             json.dump(job_info, outfile, sort_keys=True, indent=4)
 
+        # We can now remove the local copy we made with bpy.ops.wm.save_as_mainfile().
+        # Strictly speaking we can already remove it after the BAM-pack, but it may come in
+        # handy in case of failures.
+        try:
+            self.log.info('Removing temporary file %s', filepath)
+            filepath.unlink()
+        except Exception as ex:
+            self.report({'ERROR'}, 'Unable to remove file: %s' % ex)
+            self.quit()
+            return
+
+        if prefs.flamenco_open_browser_after_submit:
+            import webbrowser
+            from urllib.parse import urljoin
+            from ..blender import PILLAR_WEB_SERVER_URL
+
+            url = urljoin(PILLAR_WEB_SERVER_URL, '/flamenco/jobs/%s/redir' % job_info['_id'])
+            webbrowser.open_new_tab(url)
+
         # Do a final report.
         if missing_sources:
             names = (ms.name for ms in missing_sources)
@@ -193,11 +271,49 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
                         '; '.join(names))
         else:
             self.report({'INFO'}, 'Flamenco job created.')
+
         self.quit()
 
+    async def _save_blendfile(self, context):
+        """Save to a different file, specifically for Flamenco.
+
+        We shouldn't overwrite the artist's file.
+        We can compress, since this file won't be managed by SVN and doesn't need diffability.
+        """
+
+        render = context.scene.render
+
+        # Remember settings we need to restore after saving.
+        old_use_file_extension = render.use_file_extension
+        old_use_overwrite = render.use_overwrite
+        old_use_placeholder = render.use_placeholder
+
+        try:
+
+            # The file extension should be determined by the render settings, not necessarily
+            # by the setttings in the output panel.
+            render.use_file_extension = True
+
+            # Rescheduling should not overwrite existing frames.
+            render.use_overwrite = False
+            render.use_placeholder = False
+
+            filepath = Path(context.blend_data.filepath).with_suffix('.flamenco.blend')
+            self.log.info('Saving copy to temporary file %s', filepath)
+            bpy.ops.wm.save_as_mainfile(filepath=str(filepath),
+                                        compress=True,
+                                        copy=True)
+        finally:
+            # Restore the settings we changed, even after an exception.
+            render.use_file_extension = old_use_file_extension
+            render.use_overwrite = old_use_overwrite
+            render.use_placeholder = old_use_placeholder
+
+        return filepath
+
     def quit(self):
         super().quit()
-        bpy.context.window_manager.progress_end()
+        bpy.context.window_manager.flamenco_status = 'IDLE'
 
     async def bam_pack(self, filepath: Path) -> (typing.Optional[Path], typing.List[Path]):
         """BAM-packs the blendfile to the destination directory.
@@ -211,7 +327,6 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
 
         from datetime import datetime
         from ..blender import preferences
-        from . import bam_interface
 
         prefs = preferences()
 
@@ -224,6 +339,8 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
         outdir = Path(prefs.flamenco_job_file_path) / unique_dir
         outfile = outdir / filepath.name
 
+        exclusion_filter = prefs.flamenco_exclude_filter or None
+
         try:
             outdir.mkdir(parents=True)
         except Exception as ex:
@@ -233,7 +350,7 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
             return None, []
 
         try:
-            missing_sources = await bam_interface.bam_copy(filepath, outfile)
+            missing_sources = await bam_interface.bam_copy(filepath, outfile, exclusion_filter)
         except bam_interface.CommandExecutionError as ex:
             self.log.exception('Unable to execute BAM pack')
             self.report({'ERROR'}, 'Unable to execute BAM pack: %s' % ex)
@@ -243,19 +360,26 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
         return outfile, missing_sources
 
 
-class FLAMENCO_OT_scene_to_frame_range(Operator):
+def scene_frame_range(context) -> str:
+    """Returns the frame range string for the current scene."""
+
+    s = context.scene
+    return '%i-%i' % (s.frame_start, s.frame_end)
+
+
+class FLAMENCO_OT_scene_to_frame_range(FlamencoPollMixin, Operator):
     """Sets the scene frame range as the Flamenco render frame range."""
     bl_idname = 'flamenco.scene_to_frame_range'
     bl_label = 'Sets the scene frame range as the Flamenco render frame range'
     bl_description = __doc__.rstrip('.')
 
     def execute(self, context):
-        s = context.scene
-        s.flamenco_render_frame_range = '%i-%i' % (s.frame_start, s.frame_end)
+        context.scene.flamenco_render_frame_range = scene_frame_range(context)
         return {'FINISHED'}
 
 
 class FLAMENCO_OT_copy_files(Operator,
+                             FlamencoPollMixin,
                              async_loop.AsyncModalOperatorMixin):
     """Uses BAM to copy the current blendfile + dependencies to the target directory."""
     bl_idname = 'flamenco.copy_files'
@@ -266,12 +390,15 @@ class FLAMENCO_OT_copy_files(Operator,
 
     async def async_execute(self, context):
         from pathlib import Path
-        from . import bam_interface
         from ..blender import preferences
 
+        context.window_manager.flamenco_status = 'PACKING'
+        exclusion_filter = preferences().flamenco_exclude_filter or None
+
         missing_sources = await bam_interface.bam_copy(
             Path(context.blend_data.filepath),
             Path(preferences().flamenco_job_file_path),
+            exclusion_filter
         )
 
         if missing_sources:
@@ -280,9 +407,18 @@ class FLAMENCO_OT_copy_files(Operator,
 
         self.quit()
 
+    def quit(self):
+        super().quit()
+        bpy.context.window_manager.flamenco_status = 'IDLE'
+
+
+class FLAMENCO_OT_explore_file_path(FlamencoPollMixin,
+                                    Operator):
+    """Opens the Flamenco job storage path in a file explorer.
+
+    If the path cannot be found, this operator tries to open its parent.
+    """
 
-class FLAMENCO_OT_explore_file_path(Operator):
-    """Opens the Flamenco job storage path in a file explorer."""
     bl_idname = 'flamenco.explore_file_path'
     bl_label = 'Open in file explorer'
     bl_description = __doc__.rstrip('.')
@@ -291,15 +427,30 @@ class FLAMENCO_OT_explore_file_path(Operator):
 
     def execute(self, context):
         import platform
-        import subprocess
-        import os
+        import pathlib
+
+        # Possibly open a parent of the path
+        to_open = pathlib.Path(self.path)
+        while to_open.parent != to_open:  # while we're not at the root
+            if to_open.exists():
+                break
+            to_open = to_open.parent
+        else:
+            self.report({'ERROR'}, 'Unable to open %s or any of its parents.' % self.path)
+            return {'CANCELLED'}
+        to_open = str(to_open)
 
         if platform.system() == "Windows":
-            os.startfile(self.path)
+            import os
+            os.startfile(to_open)
+
         elif platform.system() == "Darwin":
-            subprocess.Popen(["open", self.path])
+            import subprocess
+            subprocess.Popen(["open", to_open])
+
         else:
-            subprocess.Popen(["xdg-open", self.path])
+            import subprocess
+            subprocess.Popen(["xdg-open", to_open])
 
         return {'FINISHED'}
 
@@ -353,7 +504,7 @@ def is_image_type(render_output_type: str) -> bool:
 @functools.lru_cache(1)
 def _render_output_path(
         local_project_path: str,
-        blend_filepath: str,
+        blend_filepath: Path,
         flamenco_job_output_strip_components: int,
         flamenco_job_output_path: str,
         render_image_format: str,
@@ -365,27 +516,46 @@ def _render_output_path(
     is fast.
     """
 
-    project_path = Path(bpy.path.abspath(local_project_path)).resolve()
-    blendfile = Path(blend_filepath)
+    try:
+        project_path = Path(bpy.path.abspath(local_project_path)).resolve()
+    except FileNotFoundError:
+        # Path.resolve() will raise a FileNotFoundError if the project path doesn't exist.
+        return None
 
     try:
-        proj_rel = blendfile.parent.relative_to(project_path)
+        blend_abspath = blend_filepath.resolve().absolute()
+    except FileNotFoundError:
+        # Path.resolve() will raise a FileNotFoundError if the path doesn't exist.
+        return None
+
+    try:
+        proj_rel = blend_abspath.parent.relative_to(project_path)
     except ValueError:
         return None
 
     rel_parts = proj_rel.parts[flamenco_job_output_strip_components:]
     output_top = Path(flamenco_job_output_path)
-    dir_components = output_top.joinpath(*rel_parts) / blendfile.stem
+
+    # Strip off '.flamenco' too; we use 'xxx.flamenco.blend' as job file, but
+    # don't want to have all the output paths ending in '.flamenco'.
+    stem = blend_filepath.stem
+    if stem.endswith('.flamenco'):
+        stem = stem[:-9]
+
+    dir_components = output_top.joinpath(*rel_parts) / stem
 
     # Blender will have to append the file extensions by itself.
     if is_image_type(render_image_format):
-        return dir_components / '#####'
+        return dir_components / '######'
     return dir_components / flamenco_render_frame_range
 
 
-def render_output_path(context) -> typing.Optional[PurePath]:
+def render_output_path(context, filepath: Path = None) -> typing.Optional[PurePath]:
     """Returns the render output path to be sent to Flamenco.
 
+    :param context: the Blender context (used to find Flamenco preferences etc.)
+    :param filepath: the Path of the blend file to render, or None for the current file.
+
     Returns None when the current blend file is outside the project path.
     """
 
@@ -394,9 +564,12 @@ def render_output_path(context) -> typing.Optional[PurePath]:
     scene = context.scene
     prefs = preferences()
 
+    if filepath is None:
+        filepath = Path(context.blend_data.filepath)
+
     return _render_output_path(
-        prefs.attract_project_local_path,
-        context.blend_data.filepath,
+        prefs.cloud_project_local_path,
+        filepath,
         prefs.flamenco_job_output_strip_components,
         prefs.flamenco_job_output_path,
         scene.render.image_settings.file_format,
@@ -404,7 +577,7 @@ def render_output_path(context) -> typing.Optional[PurePath]:
     )
 
 
-class FLAMENCO_PT_render(bpy.types.Panel):
+class FLAMENCO_PT_render(bpy.types.Panel, FlamencoPollMixin):
     bl_label = "Flamenco Render"
     bl_space_type = 'PROPERTIES'
     bl_region_type = 'WINDOW'
@@ -418,21 +591,24 @@ class FLAMENCO_PT_render(bpy.types.Panel):
 
         prefs = preferences()
 
-        layout.prop(context.scene, 'flamenco_render_job_priority')
-        layout.prop(context.scene, 'flamenco_render_chunk_size')
-
-        labeled_row = layout.split(0.2, align=True)
-        labeled_row.label('Job type:')
+        labeled_row = layout.split(0.25, align=True)
+        labeled_row.label('Job Type:')
         labeled_row.prop(context.scene, 'flamenco_render_job_type', text='')
 
-        labeled_row = layout.split(0.2, align=True)
-        labeled_row.label('Frame range:')
+        labeled_row = layout.split(0.25, align=True)
+        labeled_row.label('Frame Range:')
         prop_btn_row = labeled_row.row(align=True)
         prop_btn_row.prop(context.scene, 'flamenco_render_frame_range', text='')
         prop_btn_row.operator('flamenco.scene_to_frame_range', text='', icon='ARROW_LEFTRIGHT')
 
+        layout.prop(context.scene, 'flamenco_render_job_priority')
+        layout.prop(context.scene, 'flamenco_render_fchunk_size')
+
+        if getattr(context.scene, 'flamenco_render_job_type', None) == 'blender-render-progressive':
+            layout.prop(context.scene, 'flamenco_render_schunk_count')
+
         readonly_stuff = layout.column(align=True)
-        labeled_row = readonly_stuff.split(0.2, align=True)
+        labeled_row = readonly_stuff.split(0.25, align=True)
         labeled_row.label('Storage:')
         prop_btn_row = labeled_row.row(align=True)
         prop_btn_row.label(prefs.flamenco_job_file_path)
@@ -440,10 +616,11 @@ class FLAMENCO_PT_render(bpy.types.Panel):
                                       text='', icon='DISK_DRIVE')
         props.path = prefs.flamenco_job_file_path
 
-        labeled_row = readonly_stuff.split(0.2, align=True)
+        labeled_row = readonly_stuff.split(0.25, align=True)
         labeled_row.label('Output:')
         prop_btn_row = labeled_row.row(align=True)
         render_output = render_output_path(context)
+
         if render_output is None:
             prop_btn_row.label('Unable to render with Flamenco, outside of project directory.')
         else:
@@ -452,17 +629,40 @@ class FLAMENCO_PT_render(bpy.types.Panel):
                                           text='', icon='DISK_DRIVE')
             props.path = str(render_output.parent)
 
-            layout.operator(FLAMENCO_OT_render.bl_idname,
-                            text='Render on Flamenco',
-                            icon='RENDER_ANIMATION')
+            flamenco_status = context.window_manager.flamenco_status
+            if flamenco_status == 'IDLE':
+                layout.operator(FLAMENCO_OT_render.bl_idname,
+                                text='Render on Flamenco',
+                                icon='RENDER_ANIMATION')
+            elif flamenco_status == 'PACKING':
+                layout.label('Flamenco is packing your file + dependencies')
+            elif flamenco_status == 'COMMUNICATING':
+                layout.label('Communicating with Flamenco Server')
+            else:
+                layout.label('Unknown Flamenco status %s' % flamenco_status)
 
-        if not context.scene.render.use_overwrite:
-            warnbox = layout.box().column(align=True)
-            warnbox.label('Please enable "Overwrite" in the Output panel,')
-            warnbox.label('or re-queueing this job might not do anything.')
+
+def activate():
+    """Activates draw callbacks, menu items etc. for Flamenco."""
+
+    global flamenco_is_active
+    log.info('Activating Flamenco')
+    flamenco_is_active = True
+    _render_output_path.cache_clear()
+
+
+def deactivate():
+    """Deactivates draw callbacks, menu items etc. for Flamenco."""
+
+    global flamenco_is_active
+    log.info('Deactivating Flamenco')
+    flamenco_is_active = False
+    _render_output_path.cache_clear()
 
 
 def register():
+    from ..utils import redraw
+
     bpy.utils.register_class(FlamencoManagerGroup)
     bpy.utils.register_class(FLAMENCO_OT_fmanagers)
     bpy.utils.register_class(FLAMENCO_OT_render)
@@ -472,36 +672,62 @@ def register():
     bpy.utils.register_class(FLAMENCO_PT_render)
 
     scene = bpy.types.Scene
-    scene.flamenco_render_chunk_size = IntProperty(
-        name='Chunk size',
+    scene.flamenco_render_fchunk_size = IntProperty(
+        name='Frame Chunk Size',
         description='Maximum number of frames to render per task',
-        default=10,
+        min=1,
+        default=1,
+    )
+    scene.flamenco_render_schunk_count = IntProperty(
+        name='Number of Sample Chunks',
+        description='Number of Cycles samples chunks to use per frame',
+        min=2,
+        default=3,
+        soft_max=10,
     )
     scene.flamenco_render_frame_range = StringProperty(
-        name='Frame range',
+        name='Frame Range',
         description='Frames to render, in "printer range" notation'
     )
     scene.flamenco_render_job_type = EnumProperty(
-        name='Job type',
+        name='Job Type',
         items=[
-            ('blender-render', 'Simple Blender render', 'Not tiled, not resumable, just render'),
-        ],
-        description='Flamenco render job type',
+            ('blender-render', 'Simple Render', 'Simple frame-by-frame render'),
+            ('blender-render-progressive', 'Progressive Render',
+             'Each frame is rendered multiple times with different Cycles sample chunks, then combined'),
+        ]
     )
+
     scene.flamenco_render_job_priority = IntProperty(
-        name='Job priority',
+        name='Job Priority',
         min=0,
         default=50,
         max=100,
         description='Higher numbers mean higher priority'
     )
 
+    bpy.types.WindowManager.flamenco_status = EnumProperty(
+        items=[
+            ('IDLE', 'IDLE', 'Not doing anything.'),
+            ('PACKING', 'PACKING', 'BAM-packing all dependencies.'),
+            ('COMMUNICATING', 'COMMUNICATING', 'Communicating with Flamenco Server.'),
+        ],
+        name='flamenco_status',
+        default='IDLE',
+        description='Current status of the Flamenco add-on',
+        update=redraw)
+
 
 def unregister():
+    deactivate()
     bpy.utils.unregister_module(__name__)
 
     try:
-        del bpy.types.Scene.flamenco_render_chunk_size
+        del bpy.types.Scene.flamenco_render_fchunk_size
+    except AttributeError:
+        pass
+    try:
+        del bpy.types.Scene.flamenco_render_schunk_count
     except AttributeError:
         pass
     try:
@@ -516,3 +742,7 @@ def unregister():
         del bpy.types.Scene.flamenco_render_job_priority
     except AttributeError:
         pass
+    try:
+        del bpy.types.WindowManager.flamenco_status
+    except AttributeError:
+        pass

blender_cloud/flamenco/bam_interface.py

@@ -14,7 +14,30 @@ class CommandExecutionError(Exception):
     pass
 
 
-async def bam_copy(base_blendfile: Path, target_blendfile: Path) -> typing.List[Path]:
+def wheel_pythonpath_278() -> str:
+    """Returns the value of a PYTHONPATH environment variable needed to run BAM from its wheel file.
+
+    Workaround for Blender 2.78c not having io_blend_utils.pythonpath()
+    """
+
+    import os
+    from ..wheels import wheel_filename
+
+    # Find the wheel to run.
+    wheelpath = wheel_filename('blender_bam')
+
+    log.info('Using wheel %s to run BAM-Pack', wheelpath)
+
+    # Update the PYTHONPATH to include that wheel.
+    existing_pypath = os.environ.get('PYTHONPATH', '')
+    if existing_pypath:
+        return os.pathsep.join((existing_pypath, wheelpath))
+
+    return wheelpath
+
+
+async def bam_copy(base_blendfile: Path, target_blendfile: Path,
+                   exclusion_filter: str) -> typing.List[Path]:
     """Uses BAM to copy the given file and dependencies to the target blendfile.
 
     Due to the way blendfile_pack.py is programmed/structured, we cannot import it
@@ -27,6 +50,7 @@ async def bam_copy(base_blendfile: Path, target_blendfile: Path) -> typing.List[
     """
 
     import asyncio
+    import os
     import shlex
     import subprocess
 
@@ -41,12 +65,29 @@ async def bam_copy(base_blendfile: Path, target_blendfile: Path) -> typing.List[
         '--mode', 'FILE',
     ]
 
+    if exclusion_filter:
+        args.extend(['--exclude', exclusion_filter])
+
     cmd_to_log = ' '.join(shlex.quote(s) for s in args)
     log.info('Executing %s', cmd_to_log)
 
+    # Workaround for Blender 2.78c not having io_blend_utils.pythonpath()
+    if hasattr(io_blend_utils, 'pythonpath'):
+        pythonpath = io_blend_utils.pythonpath()
+    else:
+        pythonpath = wheel_pythonpath_278()
+
+    env = {
+        'PYTHONPATH': pythonpath,
+        # Needed on Windows because http://bugs.python.org/issue8557
+        'PATH': os.environ['PATH'],
+    }
+    if 'SYSTEMROOT' in os.environ:  # Windows http://bugs.python.org/issue20614
+        env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
+
     proc = await asyncio.create_subprocess_exec(
         *args,
-        env={'PYTHONPATH': io_blend_utils.pythonpath()},
+        env=env,
         stdin=subprocess.DEVNULL,
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,

blender_cloud/flamenco/sdk.py

@@ -1,9 +1,50 @@
+import functools
+import pathlib
+
 from pillarsdk.resource import List, Find, Create
 
 
 class Manager(List, Find):
     """Manager class wrapping the REST nodes endpoint"""
     path = 'flamenco/managers'
+    PurePlatformPath = pathlib.PurePath
+
+    @functools.lru_cache()
+    def _sorted_path_replacements(self) -> list:
+        import platform
+
+        if self.path_replacement is None:
+            return []
+
+        items = self.path_replacement.to_dict().items()
+
+        def by_length(item):
+            return -len(item[0]), item[0]
+
+        this_platform = platform.system().lower()
+        return [(varname, platform_replacements[this_platform])
+                for varname, platform_replacements in sorted(items, key=by_length)
+                if this_platform in platform_replacements]
+
+    def replace_path(self, some_path: pathlib.PurePath) -> str:
+        """Performs path variable replacement.
+
+        Tries to find platform-specific path prefixes, and replaces them with
+        variables.
+        """
+
+        for varname, path in self._sorted_path_replacements():
+            replacement = self.PurePlatformPath(path)
+            try:
+                relpath = some_path.relative_to(replacement)
+            except ValueError:
+                # Not relative to each other, so no replacement possible
+                continue
+
+            replacement_root = self.PurePlatformPath('{%s}' % varname)
+            return (replacement_root / relpath).as_posix()
+
+        return some_path.as_posix()
 
 
 class Job(List, Find, Create):

blender_cloud/pillar.py

@@ -115,6 +115,12 @@ def with_existing_dir(filename: str, open_mode: str, encoding=None):
         yield file_object
 
 
+def _shorten(somestr: str, maxlen=40) -> str:
+    """Shortens strings for logging"""
+
+    return (somestr[:maxlen - 3] + '...') if len(somestr) > maxlen else somestr
+
+
 def save_as_json(pillar_resource, json_filename):
     with with_existing_dir(json_filename, 'w') as outfile:
         log.debug('Saving metadata to %r' % json_filename)
@@ -200,8 +206,11 @@ def pillar_api(pillar_endpoint: str = None, caching=True) -> pillarsdk.Api:
     return _pillar_api[caching]
 
 
-# No more than this many Pillar calls should be made simultaneously
-pillar_semaphore = asyncio.Semaphore(3)
+# This is an asyncio.Semaphore object, which is late-instantiated to be sure
+# the asyncio loop has been created properly. On Windows we create a new one,
+# which can cause this semaphore to still be linked against the old default
+# loop.
+pillar_semaphore = None
 
 
 async def pillar_call(pillar_func, *args, caching=True, **kwargs):
@@ -214,8 +223,21 @@ async def pillar_call(pillar_func, *args, caching=True, **kwargs):
     partial = functools.partial(pillar_func, *args, api=pillar_api(caching=caching), **kwargs)
     loop = asyncio.get_event_loop()
 
-    async with pillar_semaphore:
+    # Use explicit calls to acquire() and release() so that we have more control over
+    # how long we wait and how we handle timeouts.
+    try:
+        await asyncio.wait_for(pillar_semaphore.acquire(), timeout=10, loop=loop)
+    except asyncio.TimeoutError:
+        log.info('Waiting for semaphore to call %s', pillar_func.__name__)
+        try:
+            await asyncio.wait_for(pillar_semaphore.acquire(), timeout=50, loop=loop)
+        except asyncio.TimeoutError:
+            raise RuntimeError('Timeout waiting for Pillar Semaphore!')
+
+    try:
         return await loop.run_in_executor(None, partial)
+    finally:
+        pillar_semaphore.release()
 
 
 def sync_call(pillar_func, *args, caching=True, **kwargs):
@@ -441,9 +463,9 @@ async def download_to_file(url, filename, *,
         log.debug('Downloading was cancelled before doing the GET')
         raise asyncio.CancelledError('Downloading was cancelled')
 
-    log.debug('Performing GET %s', url)
+    log.debug('Performing GET %s', _shorten(url))
     response = await loop.run_in_executor(None, perform_get_request)
-    log.debug('Status %i from GET %s', response.status_code, url)
+    log.debug('Status %i from GET %s', response.status_code, _shorten(url))
     response.raise_for_status()
 
     if response.status_code == 304:
@@ -457,9 +479,9 @@ async def download_to_file(url, filename, *,
         log.debug('Downloading was cancelled before downloading the GET response')
         raise asyncio.CancelledError('Downloading was cancelled')
 
-    log.debug('Downloading response of GET %s', url)
+    log.debug('Downloading response of GET %s', _shorten(url))
     await loop.run_in_executor(None, download_loop)
-    log.debug('Done downloading response of GET %s', url)
+    log.debug('Done downloading response of GET %s', _shorten(url))
 
     # We're done downloading, now we have something cached we can use.
     log.debug('Saving header cache to %s', header_store)
@@ -534,7 +556,8 @@ async def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str,
              for texture_node in texture_nodes)
 
     # raises any exception from failed handle_texture_node() calls.
-    await asyncio.gather(*coros)
+    loop = asyncio.get_event_loop()
+    await asyncio.gather(*coros, loop=loop)
 
     log.info('fetch_texture_thumbs: Done downloading texture thumbnails')
 
@@ -747,7 +770,8 @@ async def download_texture(texture_node,
                                     future=future)
         downloaders.append(dlr)
 
-    return await asyncio.gather(*downloaders, return_exceptions=True)
+    loop = asyncio.get_event_loop()
+    return await asyncio.gather(*downloaders, return_exceptions=True, loop=loop)
 
 
 async def upload_file(project_id: str, file_path: pathlib.Path, *,
@@ -773,9 +797,9 @@ async def upload_file(project_id: str, file_path: pathlib.Path, *,
         log.debug('Uploading was cancelled before doing the POST')
         raise asyncio.CancelledError('Uploading was cancelled')
 
-    log.debug('Performing POST %s', url)
+    log.debug('Performing POST %s', _shorten(url))
     response = await loop.run_in_executor(None, upload)
-    log.debug('Status %i from POST %s', response.status_code, url)
+    log.debug('Status %i from POST %s', response.status_code, _shorten(url))
     response.raise_for_status()
 
     resp = response.json()
@@ -860,6 +884,13 @@ class AuthenticatedPillarOperatorMixin(PillarOperatorMixin):
             self.report({'ERROR'}, 'Please log in on Blender ID first.')
             self.quit()
             return False
+        except requests.exceptions.ConnectionError:
+            self.log.exception('Error checking pillar credentials.')
+            self.report({'ERROR'}, 'Unable to connect to Blender Cloud, '
+                                   'check your internet connection.')
+            self.quit()
+            return False
+
 
         self.user_id = self.db_user['_id']
         return True

blender_cloud/texture_browser.py

@@ -457,7 +457,7 @@ class BlenderCloudBrowser(pillar.PillarOperatorMixin,
 
         return menu_item
 
-    def update_menu_item(self, node, *args) -> MenuItem:
+    def update_menu_item(self, node, *args):
         node_uuid = node['_id']
 
         # Just make this thread-safe to be on the safe side.
@@ -538,6 +538,7 @@ class BlenderCloudBrowser(pillar.PillarOperatorMixin,
             self.add_menu_item(node, None, 'SPINNER', texture_node['name'])
 
         def thumbnail_loaded(node, file_desc, thumb_path):
+            self.log.debug('Node %s thumbnail loaded', node['_id'])
             self.update_menu_item(node, file_desc, thumb_path)
 
         await pillar.fetch_texture_thumbs(node_uuid, 's', directory,

blender_cloud/wheels/__init__.py

@@ -44,6 +44,12 @@ def load_wheel(module_name, fname_prefix):
                   module_name, module.__file__, fname_prefix)
         return
 
+    sys.path.append(wheel_filename(fname_prefix))
+    module = __import__(module_name)
+    log.debug('Loaded %s from %s', module_name, module.__file__)
+
+
+def wheel_filename(fname_prefix: str) -> str:
     path_pattern = os.path.join(my_dir, '%s*.whl' % fname_prefix)
     wheels = glob.glob(path_pattern)
     if not wheels:
@@ -51,9 +57,7 @@ def load_wheel(module_name, fname_prefix):
 
     # If there are multiple wheels that match, load the latest one.
     wheels.sort()
-    sys.path.append(wheels[-1])
-    module = __import__(module_name)
-    log.debug('Loaded %s from %s', module_name, module.__file__)
+    return wheels[-1]
 
 
 def load_wheels():

requirements.txt

@@ -3,6 +3,7 @@
 lockfile==0.12.2
 pillarsdk==1.6.1
 wheel==0.29.0
+blender-bam==1.1.7
 
 # Secondary requirements:
 cffi==1.6.0

setup.py

@@ -101,6 +101,11 @@ class BuildWheels(Command):
             log.info('Downloading Pillar Python SDK wheel')
             self.download_wheel(requirements['pillarsdk'])
 
+        # Download BAM from pypi. This is required for compatibility with Blender 2.78.
+        if not list(self.wheels_path.glob('blender_bam*.whl')):
+            log.info('Downloading BAM wheel')
+            self.download_wheel(requirements['blender-bam'])
+
         # Build CacheControl.
         if not list(self.wheels_path.glob('CacheControl*.whl')):
             log.info('Building CacheControl in %s', self.cachecontrol_path)
@@ -227,11 +232,11 @@ setup(
               'wheels': BuildWheels},
     name='blender_cloud',
     description='The Blender Cloud addon allows browsing the Blender Cloud from Blender.',
-    version='1.5.9999',
+    version='1.7.3',
     author='Sybren A. Stüvel',
     author_email='sybren@stuvel.eu',
     packages=find_packages('.'),
-    data_files=[('blender_cloud', ['README.md', 'README-flamenco.md']),
+    data_files=[('blender_cloud', ['README.md', 'README-flamenco.md', 'CHANGELOG.md']),
                 ('blender_cloud/icons', glob.glob('blender_cloud/icons/*'))],
     scripts=[],
     url='https://developer.blender.org/diffusion/BCA/',

tests/test_path_replacement.py

@@ -0,0 +1,90 @@
+"""Unittests for blender_cloud.utils.
+
+This unittest requires bpy to be importable, so build Blender as a module and install it
+into your virtualenv. See https://stuvel.eu/files/bconf2016/#/10 for notes how.
+"""
+
+import datetime
+import pathlib
+import unittest.mock
+
+import pillarsdk.utils
+
+from blender_cloud.flamenco import sdk
+
+
+class PathReplacementTest(unittest.TestCase):
+    def setUp(self):
+        self.test_manager = sdk.Manager({
+            '_created': datetime.datetime(2017, 5, 31, 15, 12, 32, tzinfo=pillarsdk.utils.utc),
+            '_etag': 'c39942ee4bcc4658adcc21e4bcdfb0ae',
+            '_id': '592edd609837732a2a272c62',
+            '_updated': datetime.datetime(2017, 6, 8, 14, 51, 3, tzinfo=pillarsdk.utils.utc),
+            'description': 'Manager formerly known as "testman"',
+            'job_types': {'sleep': {'vars': {}}},
+            'name': '<script>alert("this is a manager")</script>',
+            'owner': '592edd609837732a2a272c63',
+            'path_replacement': {'job_storage': {'darwin': '/Volume/shared',
+                                                 'linux': '/shared',
+                                                 'windows': 's:/'},
+                                 'render': {'darwin': '/Volume/render/',
+                                            'linux': '/render/',
+                                            'windows': 'r:/'},
+                                 'longrender': {'darwin': '/Volume/render/long',
+                                                'linux': '/render/long',
+                                                'windows': 'r:/long'},
+                                 },
+            'projects': ['58cbdd5698377322d95eb55e'],
+            'service_account': '592edd609837732a2a272c60',
+            'stats': {'nr_of_workers': 3},
+            'url': 'http://192.168.3.101:8083/',
+            'user_groups': ['58cbdd5698377322d95eb55f'],
+            'variables': {'blender': {'darwin': '/opt/myblenderbuild/blender',
+                                      'linux': '/home/sybren/workspace/build_linux/bin/blender '
+                                               '--enable-new-depsgraph --factory-startup',
+                                      'windows': 'c:/temp/blender.exe'}}}
+        )
+
+    def test_linux(self):
+        # (expected result, input)
+        test_paths = [
+            ('/doesnotexistreally', '/doesnotexistreally'),
+            ('{render}/agent327/scenes/A_01_03_B', '/render/agent327/scenes/A_01_03_B'),
+            ('{job_storage}/render/agent327/scenes', '/shared/render/agent327/scenes'),
+            ('{longrender}/agent327/scenes', '/render/long/agent327/scenes'),
+        ]
+
+        self._do_test(test_paths, 'linux', pathlib.PurePosixPath)
+
+    def test_windows(self):
+        # (expected result, input)
+        test_paths = [
+            ('c:/doesnotexistreally', 'c:/doesnotexistreally'),
+            ('c:/some/path', r'c:\some\path'),
+            ('{render}/agent327/scenes/A_01_03_B', r'R:\agent327\scenes\A_01_03_B'),
+            ('{render}/agent327/scenes/A_01_03_B', r'r:\agent327\scenes\A_01_03_B'),
+            ('{render}/agent327/scenes/A_01_03_B', r'r:/agent327/scenes/A_01_03_B'),
+            ('{job_storage}/render/agent327/scenes', 's:/render/agent327/scenes'),
+            ('{longrender}/agent327/scenes', 'r:/long/agent327/scenes'),
+        ]
+
+        self._do_test(test_paths, 'windows', pathlib.PureWindowsPath)
+
+    def test_darwin(self):
+        # (expected result, input)
+        test_paths = [
+            ('/Volume/doesnotexistreally', '/Volume/doesnotexistreally'),
+            ('{render}/agent327/scenes/A_01_03_B', r'/Volume/render/agent327/scenes/A_01_03_B'),
+            ('{job_storage}/render/agent327/scenes', '/Volume/shared/render/agent327/scenes'),
+            ('{longrender}/agent327/scenes', '/Volume/render/long/agent327/scenes'),
+        ]
+
+        self._do_test(test_paths, 'darwin', pathlib.PurePosixPath)
+
+    def _do_test(self, test_paths, platform, pathclass):
+        self.test_manager.PurePlatformPath = pathclass
+        with unittest.mock.patch('sys.platform', platform):
+            for expected_result, input_path in test_paths:
+                self.assertEqual(expected_result,
+                                 self.test_manager.replace_path(pathclass(input_path)),
+                                 'for input %s on platform %s' % (input_path, platform))

update_version.sh

@@ -1,15 +1,19 @@
 #!/bin/bash
 
-if [ -z "$1" ]; then
+VERSION="${1/version-}"
+
+if [ -z "$VERSION" ]; then
     echo "Usage: $0 new-version" >&2
     exit 1
 fi
 
-BL_INFO_VER=$(echo "$1" | sed 's/\./, /g')
+BL_INFO_VER=$(echo "$VERSION" | sed 's/\./, /g')
 
-sed "s/version='[^']*'/version='$1'/" -i setup.py
+sed "s/version='[^']*'/version='$VERSION'/" -i setup.py
 sed "s/'version': ([^)]*)/'version': ($BL_INFO_VER)/" -i blender_cloud/__init__.py
 
 git diff
 echo
-echo "Don't forget to commit!"
+echo "Don't forget to commit and tag:"
+echo git commit -m \'Bumped version to $VERSION\' setup.py blender_cloud/__init__.py
+echo git tag -a version-$VERSION -m \'Tagged version $VERSION\'