Released 1.7.1

The biggest issue was the construction of an asyncio.Semaphore() while the
default loop is alive, and then creating a new loop on win32.

I've also taken the opportunity to explicitly pass our loop to some calls,
rather than expecting them to use the correct one automagically, and added
some more explicit timeout handling to the semaphore usage.

CHANGELOG.md

@@ -1,9 +1,21 @@
 # Blender Cloud changelog
 
 
+## Version 1.7.1 (2017-06-13)
+
+- Fixed asyncio issues on Windows
+
+
+## Version 1.7.0 (2017-06-09)
+
+- Fixed reloading after upgrading from 1.4.4 (our last public release).
+- Fixed bug handling a symlinked project path.
+- Added support for Manager-defined path replacement variables.
+
+
 ## Version 1.6.4 (2017-04-21)
 
-- Added file exclusion filter for Flamenco. A filter like "*.abc;*.mkv;*.mov" can be
+- Added file exclusion filter for Flamenco. A filter like `*.abc;*.mkv;*.mov` can be
   used to prevent certain files from being copied to the job storage directory.
   Requires a Blender that is bundled with BAM 1.1.7 or newer.
 

blender_cloud/__init__.py

@@ -21,7 +21,7 @@
 bl_info = {
     'name': 'Blender Cloud',
     "author": "Sybren A. Stüvel, Francesco Siddi, Inês Almeida, Antony Riakiotakis",
-    'version': (1, 6, 4),
+    'version': (1, 7, 1),
     'blender': (2, 77, 0),
     'location': 'Addon Preferences panel, and Ctrl+Shift+Alt+A anywhere for texture browser',
     'description': 'Texture library browser and Blender Sync. Requires the Blender ID addon '
@@ -65,21 +65,28 @@ def register():
 
         def reload_mod(name):
             modname = '%s.%s' % (__name__, name)
-            module = importlib.reload(sys.modules[modname])
-            sys.modules[modname] = module
-            return module
+            try:
+                old_module = sys.modules[modname]
+            except KeyError:
+                # Wasn't loaded before -- can happen after an upgrade.
+                new_module = importlib.import_module(modname)
+            else:
+                new_module = importlib.reload(old_module)
+
+            sys.modules[modname] = new_module
+            return new_module
 
         reload_mod('blendfile')
         reload_mod('home_project')
         reload_mod('utils')
 
-        blender = reload_mod('blender')
         async_loop = reload_mod('async_loop')
+        flamenco = reload_mod('flamenco')
+        attract = reload_mod('attract')
         texture_browser = reload_mod('texture_browser')
         settings_sync = reload_mod('settings_sync')
         image_sharing = reload_mod('image_sharing')
-        attract = reload_mod('attract')
-        flamenco = reload_mod('flamenco')
+        blender = reload_mod('blender')
     else:
         from . import (blender, texture_browser, async_loop, settings_sync, blendfile, home_project,
                        image_sharing, attract, flamenco)
@@ -88,11 +95,11 @@ def register():
     async_loop.register()
 
     flamenco.register()
+    attract.register()
     texture_browser.register()
-    blender.register()
     settings_sync.register()
     image_sharing.register()
-    attract.register()
+    blender.register()
 
     blender.handle_project_update()
 

blender_cloud/async_loop.py

@@ -33,17 +33,12 @@ _loop_kicking_operator_running = False
 
 
 def setup_asyncio_executor():
-    """Sets up AsyncIO to run on a single thread.
+    """Sets up AsyncIO to run properly on each platform."""
 
-    This ensures that only one Pillar HTTP call is performed at the same time. Other
-    calls that could be performed in parallel are queued, and thus we can
-    reliably cancel them.
-    """
     import sys
 
-    executor = concurrent.futures.ThreadPoolExecutor()
-
     if sys.platform == 'win32':
+        asyncio.get_event_loop().close()
         # On Windows, the default event loop is SelectorEventLoop, which does
         # not support subprocesses. ProactorEventLoop should be used instead.
         # Source: https://docs.python.org/3/library/asyncio-subprocess.html
@@ -51,9 +46,15 @@ def setup_asyncio_executor():
         asyncio.set_event_loop(loop)
     else:
         loop = asyncio.get_event_loop()
+
+    executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
     loop.set_default_executor(executor)
     # loop.set_debug(True)
 
+    from . import pillar
+    # No more than this many Pillar calls should be made simultaneously
+    pillar.pillar_semaphore = asyncio.Semaphore(3, loop=loop)
+
 
 def kick_async_loop(*args) -> bool:
     """Performs a single iteration of the asyncio event loop.

blender_cloud/blender.py

@@ -426,6 +426,8 @@ class BlenderCloudPreferences(AddonPreferences):
                          text='Local Cloud Project Path')
 
     def draw_flamenco_buttons(self, flamenco_box, bcp: flamenco.FlamencoManagerGroup, context):
+        from .flamenco import bam_interface
+
         header_row = flamenco_box.row(align=True)
         header_row.label('Flamenco:', icon_value=icon('CLOUD'))
 
@@ -461,7 +463,15 @@ class BlenderCloudPreferences(AddonPreferences):
         props = path_box.operator('flamenco.explore_file_path', text='', icon='DISK_DRIVE')
         props.path = self.flamenco_job_output_path
 
-        job_output_box.prop(self, 'flamenco_exclude_filter')
+        show_warning = bool(self.flamenco_exclude_filter and
+                            not bam_interface.bam_supports_exclude_option())
+        job_output_box.alert = show_warning
+        job_output_box.prop(self, 'flamenco_exclude_filter',
+                            icon='ERROR' if show_warning else 'NONE')
+        if show_warning:
+            job_output_box.label(
+                text='Warning, the exclusion filter requires a newer version of Blender!')
+        job_output_box.alert = False
 
         prop_split = job_output_box.split(0.32, align=True)
         prop_split.label('Strip Components:')

blender_cloud/flamenco/__init__.py

@@ -139,6 +139,9 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
         if not await self.authenticate(context):
             return
 
+        import pillarsdk.exceptions
+        from .sdk import Manager
+        from ..pillar import pillar_call
         from ..blender import preferences
 
         scene = context.scene
@@ -160,15 +163,26 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
         if not outfile:
             return
 
-        # Create the job at Flamenco Server.
+        # Fetch Manager for doing path replacement.
+        self.log.info('Going to fetch manager %s', self.user_id)
         prefs = preferences()
 
+        manager_id = prefs.flamenco_manager.manager
+        try:
+            manager = await pillar_call(Manager.find, manager_id)
+        except pillarsdk.exceptions.ResourceNotFound:
+            self.report({'ERROR'}, 'Manager %s not found, refresh your managers in '
+                                   'the Blender Cloud add-on settings.' % manager_id)
+            self.quit()
+            return
+
+        # Create the job at Flamenco Server.
         context.window_manager.flamenco_status = 'COMMUNICATING'
         settings = {'blender_cmd': '{blender}',
                     'chunk_size': scene.flamenco_render_fchunk_size,
-                    'filepath': str(outfile),
+                    'filepath': manager.replace_path(outfile),
                     'frames': scene.flamenco_render_frame_range,
-                    'render_output': str(render_output),
+                    'render_output': manager.replace_path(render_output),
                     }
 
         # Add extra settings specific to the job type
@@ -188,7 +202,7 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
         try:
             job_info = await create_job(self.user_id,
                                         prefs.project.project,
-                                        prefs.flamenco_manager.manager,
+                                        manager_id,
                                         scene.flamenco_render_job_type,
                                         settings,
                                         'Render %s' % filepath.name,
@@ -477,7 +491,13 @@ def _render_output_path(
         return None
 
     try:
-        proj_rel = blend_filepath.parent.relative_to(project_path)
+        blend_abspath = blend_filepath.resolve().absolute()
+    except FileNotFoundError:
+        # Path.resolve() will raise a FileNotFoundError if the path doesn't exist.
+        return None
+
+    try:
+        proj_rel = blend_abspath.parent.relative_to(project_path)
     except ValueError:
         return None
 
@@ -596,6 +616,7 @@ def activate():
     global flamenco_is_active
     log.info('Activating Flamenco')
     flamenco_is_active = True
+    _render_output_path.cache_clear()
 
 
 def deactivate():
@@ -604,6 +625,7 @@ def deactivate():
     global flamenco_is_active
     log.info('Deactivating Flamenco')
     flamenco_is_active = False
+    _render_output_path.cache_clear()
 
 
 def register():

blender_cloud/flamenco/bam_interface.py

@@ -1,5 +1,6 @@
 """BAM packing interface for Flamenco."""
 
+import functools
 import logging
 from pathlib import Path
 import typing
@@ -14,6 +15,28 @@ class CommandExecutionError(Exception):
     pass
 
 
+if 'bam_supports_exclude_option' in locals():
+    locals()['bam_supports_exclude_option'].cache_clear()
+
+
+@functools.lru_cache(maxsize=1)
+def bam_supports_exclude_option() -> bool:
+    """Returns True if the version of BAM bundled with Blender supports --exclude.
+
+    This feature was added to BAM 1.1.7, so we can do a simple version check.
+    """
+
+    try:
+        import io_blend_utils
+    except ImportError:
+        # If this happens, BAM won't work at all. However, this function can be called from
+        # the GUI; by being a bit careful while importing, we avoid breaking Blender's GUI.
+        log.exception('Error importing io_blend_utils module.')
+        return False
+
+    return io_blend_utils.bl_info['version'] >= (1, 1, 7)
+
+
 async def bam_copy(base_blendfile: Path, target_blendfile: Path,
                    exclusion_filter: str) -> typing.List[Path]:
     """Uses BAM to copy the given file and dependencies to the target blendfile.
@@ -43,7 +66,11 @@ async def bam_copy(base_blendfile: Path, target_blendfile: Path,
     ]
 
     if exclusion_filter:
-        args.extend(['--exclude', exclusion_filter])
+        if bam_supports_exclude_option():
+            args.extend(['--exclude', exclusion_filter])
+        else:
+            log.warning('Your version of Blender does not support the exclusion filter, '
+                        'copying all files.')
 
     cmd_to_log = ' '.join(shlex.quote(s) for s in args)
     log.info('Executing %s', cmd_to_log)

blender_cloud/flamenco/sdk.py

@@ -1,9 +1,51 @@
+import functools
+import pathlib
+
 from pillarsdk.resource import List, Find, Create
 
 
 class Manager(List, Find):
     """Manager class wrapping the REST nodes endpoint"""
     path = 'flamenco/managers'
+    PurePlatformPath = pathlib.PurePath
+
+    @functools.lru_cache()
+    def _sorted_path_replacements(self) -> list:
+        import sys
+
+        if self.path_replacement is None:
+            return []
+
+        print('SORTING PATH REPLACEMENTS')
+
+        items = self.path_replacement.to_dict().items()
+
+        def by_length(item):
+            return -len(item[0]), item[0]
+
+        platform = sys.platform
+        return [(varname, platform_replacements[platform])
+                for varname, platform_replacements in sorted(items, key=by_length)]
+
+    def replace_path(self, some_path: pathlib.PurePath) -> str:
+        """Performs path variable replacement.
+
+        Tries to find platform-specific path prefixes, and replaces them with
+        variables.
+        """
+
+        for varname, path in self._sorted_path_replacements():
+            replacement = self.PurePlatformPath(path)
+            try:
+                relpath = some_path.relative_to(replacement)
+            except ValueError:
+                # Not relative to each other, so no replacement possible
+                continue
+
+            replacement_root = self.PurePlatformPath('{%s}' % varname)
+            return (replacement_root / relpath).as_posix()
+
+        return some_path.as_posix()
 
 
 class Job(List, Find, Create):

blender_cloud/pillar.py

@@ -115,6 +115,12 @@ def with_existing_dir(filename: str, open_mode: str, encoding=None):
         yield file_object
 
 
+def _shorten(somestr: str, maxlen=40) -> str:
+    """Shortens strings for logging"""
+
+    return (somestr[:maxlen - 3] + '...') if len(somestr) > maxlen else somestr
+
+
 def save_as_json(pillar_resource, json_filename):
     with with_existing_dir(json_filename, 'w') as outfile:
         log.debug('Saving metadata to %r' % json_filename)
@@ -200,8 +206,11 @@ def pillar_api(pillar_endpoint: str = None, caching=True) -> pillarsdk.Api:
     return _pillar_api[caching]
 
 
-# No more than this many Pillar calls should be made simultaneously
-pillar_semaphore = asyncio.Semaphore(3)
+# This is an asyncio.Semaphore object, which is late-instantiated to be sure
+# the asyncio loop has been created properly. On Windows we create a new one,
+# which can cause this semaphore to still be linked against the old default
+# loop.
+pillar_semaphore = None
 
 
 async def pillar_call(pillar_func, *args, caching=True, **kwargs):
@@ -214,8 +223,21 @@ async def pillar_call(pillar_func, *args, caching=True, **kwargs):
     partial = functools.partial(pillar_func, *args, api=pillar_api(caching=caching), **kwargs)
     loop = asyncio.get_event_loop()
 
-    async with pillar_semaphore:
+    # Use explicit calls to acquire() and release() so that we have more control over
+    # how long we wait and how we handle timeouts.
+    try:
+        await asyncio.wait_for(pillar_semaphore.acquire(), timeout=10, loop=loop)
+    except asyncio.TimeoutError:
+        log.info('Waiting for semaphore to call %s', pillar_func.__name__)
+        try:
+            await asyncio.wait_for(pillar_semaphore.acquire(), timeout=50, loop=loop)
+        except asyncio.TimeoutError:
+            raise RuntimeError('Timeout waiting for Pillar Semaphore!')
+
+    try:
         return await loop.run_in_executor(None, partial)
+    finally:
+        pillar_semaphore.release()
 
 
 def sync_call(pillar_func, *args, caching=True, **kwargs):
@@ -441,9 +463,9 @@ async def download_to_file(url, filename, *,
         log.debug('Downloading was cancelled before doing the GET')
         raise asyncio.CancelledError('Downloading was cancelled')
 
-    log.debug('Performing GET %s', url)
+    log.debug('Performing GET %s', _shorten(url))
     response = await loop.run_in_executor(None, perform_get_request)
-    log.debug('Status %i from GET %s', response.status_code, url)
+    log.debug('Status %i from GET %s', response.status_code, _shorten(url))
     response.raise_for_status()
 
     if response.status_code == 304:
@@ -457,9 +479,9 @@ async def download_to_file(url, filename, *,
         log.debug('Downloading was cancelled before downloading the GET response')
         raise asyncio.CancelledError('Downloading was cancelled')
 
-    log.debug('Downloading response of GET %s', url)
+    log.debug('Downloading response of GET %s', _shorten(url))
     await loop.run_in_executor(None, download_loop)
-    log.debug('Done downloading response of GET %s', url)
+    log.debug('Done downloading response of GET %s', _shorten(url))
 
     # We're done downloading, now we have something cached we can use.
     log.debug('Saving header cache to %s', header_store)
@@ -534,7 +556,8 @@ async def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str,
              for texture_node in texture_nodes)
 
     # raises any exception from failed handle_texture_node() calls.
-    await asyncio.gather(*coros)
+    loop = asyncio.get_event_loop()
+    await asyncio.gather(*coros, loop=loop)
 
     log.info('fetch_texture_thumbs: Done downloading texture thumbnails')
 
@@ -747,7 +770,8 @@ async def download_texture(texture_node,
                                     future=future)
         downloaders.append(dlr)
 
-    return await asyncio.gather(*downloaders, return_exceptions=True)
+    loop = asyncio.get_event_loop()
+    return await asyncio.gather(*downloaders, return_exceptions=True, loop=loop)
 
 
 async def upload_file(project_id: str, file_path: pathlib.Path, *,
@@ -773,9 +797,9 @@ async def upload_file(project_id: str, file_path: pathlib.Path, *,
         log.debug('Uploading was cancelled before doing the POST')
         raise asyncio.CancelledError('Uploading was cancelled')
 
-    log.debug('Performing POST %s', url)
+    log.debug('Performing POST %s', _shorten(url))
     response = await loop.run_in_executor(None, upload)
-    log.debug('Status %i from POST %s', response.status_code, url)
+    log.debug('Status %i from POST %s', response.status_code, _shorten(url))
     response.raise_for_status()
 
     resp = response.json()

blender_cloud/texture_browser.py

@@ -457,7 +457,7 @@ class BlenderCloudBrowser(pillar.PillarOperatorMixin,
 
         return menu_item
 
-    def update_menu_item(self, node, *args) -> MenuItem:
+    def update_menu_item(self, node, *args):
         node_uuid = node['_id']
 
         # Just make this thread-safe to be on the safe side.
@@ -538,6 +538,7 @@ class BlenderCloudBrowser(pillar.PillarOperatorMixin,
             self.add_menu_item(node, None, 'SPINNER', texture_node['name'])
 
         def thumbnail_loaded(node, file_desc, thumb_path):
+            self.log.debug('Node %s thumbnail loaded', node['_id'])
             self.update_menu_item(node, file_desc, thumb_path)
 
         await pillar.fetch_texture_thumbs(node_uuid, 's', directory,

setup.py

@@ -227,7 +227,7 @@ setup(
               'wheels': BuildWheels},
     name='blender_cloud',
     description='The Blender Cloud addon allows browsing the Blender Cloud from Blender.',
-    version='1.6.4',
+    version='1.7.1',
     author='Sybren A. Stüvel',
     author_email='sybren@stuvel.eu',
     packages=find_packages('.'),

tests/test_path_replacement.py

@@ -0,0 +1,90 @@
+"""Unittests for blender_cloud.utils.
+
+This unittest requires bpy to be importable, so build Blender as a module and install it
+into your virtualenv. See https://stuvel.eu/files/bconf2016/#/10 for notes how.
+"""
+
+import datetime
+import pathlib
+import unittest.mock
+
+import pillarsdk.utils
+
+from blender_cloud.flamenco import sdk
+
+
+class PathReplacementTest(unittest.TestCase):
+    def setUp(self):
+        self.test_manager = sdk.Manager({
+            '_created': datetime.datetime(2017, 5, 31, 15, 12, 32, tzinfo=pillarsdk.utils.utc),
+            '_etag': 'c39942ee4bcc4658adcc21e4bcdfb0ae',
+            '_id': '592edd609837732a2a272c62',
+            '_updated': datetime.datetime(2017, 6, 8, 14, 51, 3, tzinfo=pillarsdk.utils.utc),
+            'description': 'Manager formerly known as "testman"',
+            'job_types': {'sleep': {'vars': {}}},
+            'name': '<script>alert("this is a manager")</script>',
+            'owner': '592edd609837732a2a272c63',
+            'path_replacement': {'job_storage': {'darwin': '/Volume/shared',
+                                                 'linux': '/shared',
+                                                 'windows': 's:/'},
+                                 'render': {'darwin': '/Volume/render/',
+                                            'linux': '/render/',
+                                            'windows': 'r:/'},
+                                 'longrender': {'darwin': '/Volume/render/long',
+                                                'linux': '/render/long',
+                                                'windows': 'r:/long'},
+                                 },
+            'projects': ['58cbdd5698377322d95eb55e'],
+            'service_account': '592edd609837732a2a272c60',
+            'stats': {'nr_of_workers': 3},
+            'url': 'http://192.168.3.101:8083/',
+            'user_groups': ['58cbdd5698377322d95eb55f'],
+            'variables': {'blender': {'darwin': '/opt/myblenderbuild/blender',
+                                      'linux': '/home/sybren/workspace/build_linux/bin/blender '
+                                               '--enable-new-depsgraph --factory-startup',
+                                      'windows': 'c:/temp/blender.exe'}}}
+        )
+
+    def test_linux(self):
+        # (expected result, input)
+        test_paths = [
+            ('/doesnotexistreally', '/doesnotexistreally'),
+            ('{render}/agent327/scenes/A_01_03_B', '/render/agent327/scenes/A_01_03_B'),
+            ('{job_storage}/render/agent327/scenes', '/shared/render/agent327/scenes'),
+            ('{longrender}/agent327/scenes', '/render/long/agent327/scenes'),
+        ]
+
+        self._do_test(test_paths, 'linux', pathlib.PurePosixPath)
+
+    def test_windows(self):
+        # (expected result, input)
+        test_paths = [
+            ('c:/doesnotexistreally', 'c:/doesnotexistreally'),
+            ('c:/some/path', r'c:\some\path'),
+            ('{render}/agent327/scenes/A_01_03_B', r'R:\agent327\scenes\A_01_03_B'),
+            ('{render}/agent327/scenes/A_01_03_B', r'r:\agent327\scenes\A_01_03_B'),
+            ('{render}/agent327/scenes/A_01_03_B', r'r:/agent327/scenes/A_01_03_B'),
+            ('{job_storage}/render/agent327/scenes', 's:/render/agent327/scenes'),
+            ('{longrender}/agent327/scenes', 'r:/long/agent327/scenes'),
+        ]
+
+        self._do_test(test_paths, 'windows', pathlib.PureWindowsPath)
+
+    def test_darwin(self):
+        # (expected result, input)
+        test_paths = [
+            ('/Volume/doesnotexistreally', '/Volume/doesnotexistreally'),
+            ('{render}/agent327/scenes/A_01_03_B', r'/Volume/render/agent327/scenes/A_01_03_B'),
+            ('{job_storage}/render/agent327/scenes', '/Volume/shared/render/agent327/scenes'),
+            ('{longrender}/agent327/scenes', '/Volume/render/long/agent327/scenes'),
+        ]
+
+        self._do_test(test_paths, 'darwin', pathlib.PurePosixPath)
+
+    def _do_test(self, test_paths, platform, pathclass):
+        self.test_manager.PurePlatformPath = pathclass
+        with unittest.mock.patch('sys.platform', platform):
+            for expected_result, input_path in test_paths:
+                self.assertEqual(expected_result,
+                                 self.test_manager.replace_path(pathclass(input_path)),
+                                 'for input %s on platform %s' % (input_path, platform))