Transitioned from synchronous to asynchronous

Using the new 'async def' functionality in Python 3.5!
Still crappy GUI, we're going to replace that.

blender_cloud/__init__.py

@@ -34,14 +34,16 @@ bl_info = {
 
 import os.path
 import typing
+import asyncio
 
 # Support reloading
 if 'pillar' in locals():
     import importlib
 
     pillar = importlib.reload(pillar)
+    async_loop = importlib.reload(async_loop)
 else:
-    from . import pillar
+    from . import pillar, async_loop
 
 import bpy
 import bpy.utils.previews
@@ -151,31 +153,73 @@ def enum_previews_from_directory_items(self, context) -> typing.List[typing.AnyS
         return pcoll.previews
 
     print('Loading previews for project {!r} node {!r}'.format(project_uuid, node_uuid))
-    enum_items = []
 
+    if pcoll.async_task is not None and not pcoll.async_task.done():
+        # We're still asynchronously downloading, but the UUIDs changed.
+        print('Cancelling running async download task {}'.format(pcoll.async_task))
+        pcoll.async_task.cancel()
+
+    # Download the previews asynchronously.
+    pcoll.previews = []
+    pcoll.project_uuid = project_uuid
+    pcoll.node_uuid = node_uuid
+    pcoll.context = context
+    pcoll.async_task = asyncio.ensure_future(async_download_previews(wm.thumbnails_cache, pcoll))
+
+    # Start the async manager so everything happens.
+    async_loop.ensure_async_loop()
+
+    return pcoll.previews
+
+
+async def async_download_previews(thumbnails_directory, pcoll):
     # If we have a node UUID, we fetch the textures
     # FIXME: support mixture of sub-nodes and textures under one node.
-    if node_uuid:
+    enum_items = pcoll.previews
-        # Make sure we can go up again.
-        parent = pillar.parent_node_uuid(node_uuid)
-        enum_items.append(('node-{}'.format(parent), 'up', 'up',
-                           'FILE_FOLDER',
-                           len(enum_items)))
 
-        directory = os.path.join(wm.thumbnails_cache, project_uuid, node_uuid)
+    node_uuid = pcoll.node_uuid
-        os.makedirs(directory, exist_ok=True)
+    project_uuid = pcoll.project_uuid
 
-        for file_desc, thumb_path in pillar.fetch_texture_thumbs(node_uuid, 's', directory):
+    def thumbnail_loading(file_desc):
+        # TODO: trigger re-draw
+        pass
+
+    try:
+        region = pcoll.context.window.screen.regions[0]
+        print('We have a region: {}'.format(region))
+    except IndexError:
+        region = None
+
+    def thumbnail_loaded(file_desc, thumb_path):
         thumb = pcoll.get(thumb_path)
         if thumb is None:
             thumb = pcoll.load(thumb_path, thumb_path, 'IMAGE')
         enum_items.append(('thumb-{}'.format(thumb_path), file_desc['filename'],
                            thumb_path,
-                               # TODO: get something here that allows downloading the texture
                            thumb.icon_id,
                            len(enum_items)))
+        # TODO: trigger re-draw
+        if region is not None:
+            try:
+                region.tag_redraw()
+            except Exception as e:
+                print('WE DIE! ', e)
+
+    if node_uuid:
+        # Make sure we can go up again.
+        parent = await pillar.parent_node_uuid(node_uuid)
+        enum_items.append(('node-{}'.format(parent), 'up', 'up',
+                           'FILE_FOLDER',
+                           len(enum_items)))
+
+        directory = os.path.join(thumbnails_directory, project_uuid, node_uuid)
+        os.makedirs(directory, exist_ok=True)
+
+        await pillar.fetch_texture_thumbs(node_uuid, 's', directory,
+                                          thumbnail_loading=thumbnail_loading,
+                                          thumbnail_loaded=thumbnail_loaded)
     elif project_uuid:
-        children = pillar.get_nodes(project_uuid, '')
+        children = await pillar.get_nodes(project_uuid, '')
 
         for child in children:
             print('  - %(_id)s = %(name)s' % child)
@@ -183,11 +227,15 @@ def enum_previews_from_directory_items(self, context) -> typing.List[typing.AnyS
                                'description',
                                'FILE_FOLDER',
                                len(enum_items)))
-
+            # TODO: trigger re-draw
-    pcoll.previews = enum_items
+            if region is not None:
-    pcoll.project_uuid = project_uuid
+                try:
-    pcoll.node_uuid = node_uuid
+                    region.tag_redraw()
-    return pcoll.previews
+                except Exception as e:
+                    print('WE DIE! ', e)
+    else:
+        # TODO: add "nothing here" icon and trigger re-draw
+        pass
 
 
 def enum_previews_from_directory_update(self, context):
@@ -224,7 +272,38 @@ class PreviewsExamplePanel(bpy.types.Panel):
         row.prop(wm, "blender_cloud_project")
         row.prop(wm, "blender_cloud_node")
         row.template_icon_view(wm, "blender_cloud_thumbnails", show_labels=True)
-        row.prop(wm, "blender_cloud_thumbnails")
+        # row.prop(wm, "blender_cloud_thumbnails")
+
+
+class AsyncOperator(Operator):
+    bl_idname = 'async.action'
+    bl_label = 'Asynchronous action'
+    bl_description = ''
+
+    def execute(self, context):
+        print('{}: executing'.format(self))
+
+        asyncio.ensure_future(do_async_stuff(context))
+        async_loop.ensure_async_loop()
+        print('{}: done'.format(self))
+
+        return {'FINISHED'}
+
+
+async def do_async_stuff(context):
+    print('do_async_stuff(): starting')
+
+    wm = context.window_manager
+    project_uuid = wm.blender_cloud_project
+    print('Loading nodes for project {!r}'.format(project_uuid))
+
+    children = await pillar.get_nodes(project_uuid, '')
+
+    for child in children:
+        print('  - %(_id)s = %(name)s' % child)
+        await asyncio.sleep(0.5)
+
+    print('do_async_stuff(): done')
 
 
 def register():
@@ -260,6 +339,8 @@ def register():
     pcoll.previews = ()
     pcoll.project_uuid = ''
     pcoll.node_uuid = ''
+    pcoll.async_task = None
+    pcoll.context = None
 
     preview_collections["blender_cloud"] = pcoll
 

blender_cloud/async_loop.py

@@ -0,0 +1,49 @@
+"""Manages the asyncio loop."""
+
+import asyncio
+import bpy
+
+
+def kick_async_loop(*args):
+    loop = asyncio.get_event_loop()
+
+    if loop.is_closed():
+        print('{}: loop closed, stopping'.format(__name__))
+        stop_async_loop()
+        return
+
+    if not asyncio.Task.all_tasks():
+        print('{}: no more scheduled tasks, stopping'.format(__name__))
+        stop_async_loop()
+        return
+
+    # Perform a single async loop step
+    async def do_nothing():
+        pass
+
+    loop.run_until_complete(do_nothing())
+
+
+def async_loop_handler() -> callable:
+    name = kick_async_loop.__name__
+    for handler in bpy.app.handlers.scene_update_pre:
+        if getattr(handler, '__name__', '') == name:
+            return handler
+    return None
+
+
+def ensure_async_loop():
+    if async_loop_handler() is not None:
+        return
+    bpy.app.handlers.scene_update_pre.append(kick_async_loop)
+
+
+def stop_async_loop():
+    handler = async_loop_handler()
+    if handler is None:
+        return
+    bpy.app.handlers.scene_update_pre.remove(handler)
+
+    # Optional: cancel all pending tasks.
+    # for task in asyncio.Task.all_tasks():
+    #     task.cancel()

blender_cloud/pillar.py

@@ -1,3 +1,4 @@
+import asyncio
 import sys
 import os
 import concurrent.futures
@@ -14,6 +15,7 @@ if not any('pillar_sdk' in path for path in sys.path):
 
 import pillarsdk
 import pillarsdk.exceptions
+import pillarsdk.utils
 
 _pillar_api = None  # will become a pillarsdk.Api object.
 
@@ -62,14 +64,17 @@ def pillar_api() -> pillarsdk.Api:
     return _pillar_api
 
 
-def get_project_uuid(project_url: str) -> str:
+async def get_project_uuid(project_url: str) -> str:
     """Returns the UUID for the project, given its '/p/<project_url>' string."""
 
-    try:
+    find_one = functools.partial(pillarsdk.Project.find_one, {
-        project = pillarsdk.Project.find_one({
         'where': {'url': project_url},
         'projection': {'permissions': 1},
     }, api=pillar_api())
+
+    loop = asyncio.get_event_loop()
+    try:
+        project = await loop.run_in_executor(None, find_one)
     except pillarsdk.exceptions.ResourceNotFound:
         print('Project with URL %r does not exist' % project_url)
         return None
@@ -78,7 +83,7 @@ def get_project_uuid(project_url: str) -> str:
     return project['_id']
 
 
-def get_nodes(project_uuid: str = None, parent_node_uuid: str = None) -> list:
+async def get_nodes(project_uuid: str = None, parent_node_uuid: str = None) -> list:
     """Gets nodes for either a project or given a parent node.
 
     @param project_uuid: the UUID of the project, or None if only querying by parent_node_uuid.
@@ -102,73 +107,112 @@ def get_nodes(project_uuid: str = None, parent_node_uuid: str = None) -> list:
     if project_uuid:
         where['project'] = project_uuid
 
-    children = pillarsdk.Node.all({
+    node_all = functools.partial(pillarsdk.Node.all, {
         'projection': {'name': 1, 'parent': 1, 'node_type': 1,
                        'properties.order': 1, 'properties.status': 1,
-                       'properties.content_type': 1, 'picture': 1,
+                       'properties.content_type': 1, 'picture': 1},
-                       'permissions': 1, 'project': 1,  # for permission checking
-                       },
         'where': where,
-        'sort': 'properties.order'},
+        'sort': 'properties.order'}, api=pillar_api())
-        api=pillar_api())
+
+    loop = asyncio.get_event_loop()
+    children = await loop.run_in_executor(None, node_all)
 
     return children['_items']
 
 
-def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str, thumbnail_directory: str):
+async def download_to_file(url, filename, chunk_size=10 * 1024):
+    """Downloads a file via HTTP(S) directly to the filesystem."""
+
+    loop = asyncio.get_event_loop()
+    await loop.run_in_executor(None, pillarsdk.utils.download_to_file, url, filename, chunk_size)
+
+
+async def stream_thumb_to_file(file: pillarsdk.File, directory: str, desired_size: str):
+    """Streams a thumbnail to a file.
+
+    @param file: the pillar File object that represents the image whose thumbnail to download.
+    @param directory: the directory to save the file to.
+    @param desired_size: thumbnail size
+    @return: the absolute path of the downloaded file.
+    """
+
+    api = pillar_api()
+
+    loop = asyncio.get_event_loop()
+    thumb_link = await loop.run_in_executor(None, functools.partial(
+        file.thumbnail_file, desired_size, api=api))
+
+    if thumb_link is None:
+        raise ValueError("File {} has no thumbnail of size {}"
+                         .format(file['_id'], desired_size))
+
+    root, ext = os.path.splitext(file['file_path'])
+    thumb_fname = "{0}-{1}.jpg".format(root, desired_size)
+    thumb_path = os.path.abspath(os.path.join(directory, thumb_fname))
+
+    await download_to_file(thumb_link, thumb_path)
+
+    return thumb_path
+
+
+async def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str,
+                               thumbnail_directory: str,
+                               *,
+                               thumbnail_loading: callable,
+                               thumbnail_loaded: callable):
     """Generator, fetches all texture thumbnails in a certain parent node.
 
     @param parent_node_uuid: the UUID of the parent node. All sub-nodes will be downloaded.
     @param desired_size: size indicator, from 'sbtmlh'.
     @param thumbnail_directory: directory in which to store the downloaded thumbnails.
-    @returns: generator that yields (pillarsdk.File object, thumbnail path) tuples
+    @param thumbnail_loading: callback function that takes (pillarsdk.File object)
+        parameter, which is called before a thumbnail will be downloaded. This allows you to
+        show a "downloading" indicator.
+    @param thumbnail_loaded: callback function that takes (pillarsdk.File object, thumbnail path)
+        parameters, which is called for every thumbnail after it's been downloaded.
     """
 
     api = pillar_api()
+    loop = asyncio.get_event_loop()
 
-    def fetch_thumbnail_from_node(texture_node: pillarsdk.Node):
+    file_find = functools.partial(pillarsdk.File.find, params={
-        # Fetch the File description JSON
-        pic_uuid = texture_node['picture']
-        file_desc = pillarsdk.File.find(pic_uuid, {
         'projection': {'filename': 1, 'variations': 1, 'width': 1, 'height': 1},
     }, api=api)
 
+    # TODO: Still single-threaded, could branch out to a few threads here to download in parallel.
+    texture_nodes = await get_nodes(parent_node_uuid=parent_node_uuid)
+    for texture_node in texture_nodes:
+        # Skip non-texture nodes, as we can't thumbnail them anyway.
+        if texture_node['node_type'] != 'texture':
+            continue
+
+        # Find the File that belongs to this texture node
+        pic_uuid = texture_node['picture']
+        file_desc = await loop.run_in_executor(None, file_find, pic_uuid)
+        loop.call_soon_threadsafe(functools.partial(thumbnail_loading, file_desc))
+
         if file_desc is None:
-            print('Unable to find picture {}'.format(pic_uuid))
+            print('Unable to find file for texture node {}'.format(pic_uuid))
-            return None, None
+            thumb_path = None
-
+        else:
             # Save the thumbnail
-        thumb_path = file_desc.stream_thumb_to_file(thumbnail_directory, desired_size, api=api)
+            thumb_path = await stream_thumb_to_file(file_desc, thumbnail_directory, desired_size)
+            # print('Texture node {} has file {}'.format(texture_node['_id'], thumb_path))
 
-        return file_desc, thumb_path
+        loop.call_soon_threadsafe(functools.partial(thumbnail_loaded, file_desc, thumb_path))
-
-    texture_nodes = (node for node in get_nodes(parent_node_uuid=parent_node_uuid)
-                     if node['node_type'] == 'texture')
-
-    # # Single-threaded, not maintained:
-    # for node in texture_nodes:
-    #     node, file = fetch_thumbnail_from_node(node)
-    #     print('Node {} has picture {}'.format(node, file))
-
-    # Multi-threaded:
-    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
-        # Queue up fetching of thumbnails
-        futures = [executor.submit(fetch_thumbnail_from_node, node)
-                   for node in texture_nodes]
-
-        for future in futures:
-            file_desc, thumb_path = future.result()
-            yield file_desc, thumb_path
 
     print('Done downloading texture thumbnails')
 
 
-@functools.lru_cache(128)
+async def parent_node_uuid(node_uuid: str) -> str:
-def parent_node_uuid(node_uuid: str) -> str:
     """Returns the UUID of the node's parent node, or an empty string if this is the top level."""
 
     api = pillar_api()
-    node = pillarsdk.Node.find(node_uuid, {'projection': {'parent': 1}}, api=api)
+    loop = asyncio.get_event_loop()
+
+    find_node = functools.partial(pillarsdk.Node.find, node_uuid,
+                                  {'projection': {'parent': 1}}, api=api)
+    node = await loop.run_in_executor(None, find_node)
     if node is None:
         return ''