Added downloading of all files belonging to one texture.

blender_cloud/gui.py

@@ -107,7 +107,9 @@ class MenuItem:
     def update(self, node, file_desc, thumb_path: str, label_text):
         # We can get updated information about our Node, but a MenuItem should
         # always represent one node, and it shouldn't be shared between nodes.
-        assert self.node_uuid == node['_id'], "Don't change the node ID this MenuItem reflects, just create a new one."
+        if self.node_uuid != node['_id']:
+            raise ValueError("Don't change the node ID this MenuItem reflects, "
+                             "just create a new one.")
         self.node = node
         self.file_desc = file_desc  # pillarsdk.File object, or None if a 'folder' node.
         self.thumb_path = thumb_path
@@ -427,14 +429,14 @@ class BlenderCloudBrowser(bpy.types.Operator):
         self.log.debug('Browsing assets at project %r node %r', self.project_uuid, self.node_uuid)
         self._new_async_task(self.async_download_previews(self.thumbnails_cache))
 
-    def _new_async_task(self, async_task: asyncio.coroutine):
+    def _new_async_task(self, async_task: asyncio.coroutine, future: asyncio.Future=None):
         """Stops the currently running async task, and starts another one."""
 
         self.log.debug('Setting up a new task %r, so any existing task must be stopped', async_task)
         self._stop_async_task()
 
         # Download the previews asynchronously.
-        self.signalling_future = asyncio.Future()
+        self.signalling_future = future or asyncio.Future()
         self.async_task = asyncio.ensure_future(async_task)
         self.log.debug('Created new task %r', self.async_task)
 
@@ -548,19 +550,39 @@ class BlenderCloudBrowser(bpy.types.Operator):
     def handle_item_selection(self, context, item: MenuItem):
         """Called when the user clicks on a menu item that doesn't represent a folder."""
 
-        # FIXME: Download all files from the texture node, instead of just one.
+        self.clear_images()
-        # FIXME: Properly set up header store.
         self._state = 'DOWNLOADING_TEXTURE'
-        url = item.file_desc.link
+
-        local_path = os.path.join(context.scene.blender_cloud_dir, item.file_desc.filename)
+        node_path_components = [node['name'] for node in self.path_stack if node is not None]
-        local_path = bpy.path.abspath(local_path)
+        local_path_components = [self.project_uuid] + node_path_components + [self.node['name']]
-        self.log.info('Downloading %s to %s', url, local_path)
+
+        top_texture_directory = bpy.path.abspath(context.scene.blender_cloud_dir)
+        local_path = os.path.join(top_texture_directory, *local_path_components)
+        meta_path = os.path.join(top_texture_directory, '.blender_cloud')
+
+        self.log.info('Downloading texture %r to %s', item.node_uuid, local_path)
+        self.log.debug('Metadata will be stored at %s', meta_path)
+
+        file_paths = []
+
+        def texture_downloading(file_path, file_desc, *args):
+            self.log.info('Texture downloading to %s', file_path)
+
+        def texture_downloaded(file_path, file_desc, *args):
+            self.log.info('Texture downloaded to %r.', file_path)
+            bpy.data.images.load(filepath=file_path)
+            file_paths.append(file_path)
 
         def texture_download_completed(_):
-            self.log.info('Texture download complete, inspect %r.', local_path)
+            self.log.info('Texture download complete, inspect:\n%s', '\n'.join(file_paths))
             self._state = 'QUIT'
 
-        self._new_async_task(pillar.download_to_file(url, local_path, header_store='/dev/null'))
+        signalling_future = asyncio.Future()
+        self._new_async_task(pillar.download_texture(item.node, local_path,
+                                                     metadata_directory=meta_path,
+                                                     texture_loading=texture_downloading,
+                                                     texture_loaded=texture_downloaded,
+                                                     future=signalling_future))
         self.async_task.add_done_callback(texture_download_completed)
 
 

blender_cloud/pillar.py

@@ -3,7 +3,8 @@ import json
 import os
 import functools
 import logging
-from contextlib import closing
+from contextlib import closing, contextmanager
+import pprint
 
 import requests
 import requests.structures
@@ -19,6 +20,7 @@ uncached_session = requests.session()
 _testing_blender_id_profile = None  # Just for testing, overrides what is returned by blender_id_profile.
 _downloaded_urls = set()  # URLs we've downloaded this Blender session.
 
+
 class UserNotLoggedInError(RuntimeError):
     """Raised when the user should be logged in on Blender ID, but isn't.
 
@@ -26,6 +28,33 @@ class UserNotLoggedInError(RuntimeError):
     """
 
 
+class PillarError(RuntimeError):
+    """Raised when there is some issue with the communication with Pillar.
+
+    This is only raised for logical errors (for example nodes that should
+    exist but still can't be found). HTTP communication errors are signalled
+    with other exceptions.
+    """
+
+
+@contextmanager
+def with_existing_dir(filename: str, open_mode: str, encoding=None):
+    """Opens a file, ensuring its directory exists."""
+
+    directory = os.path.dirname(filename)
+    if not os.path.exists(directory):
+        log.debug('Creating directory %s', directory)
+        os.makedirs(directory, mode=0o700, exist_ok=True)
+    with open(filename, open_mode, encoding=encoding) as file_object:
+        yield file_object
+
+
+def save_as_json(pillar_resource, json_filename):
+    with with_existing_dir(json_filename, 'w') as outfile:
+        log.debug('Saving metadata to %r' % json_filename)
+        json.dump(pillar_resource, outfile, sort_keys=True, cls=pillarsdk.utils.PillarJSONEncoder)
+
+
 def blender_id_profile() -> dict:
     """Returns the Blender ID profile of the currently logged in user."""
 
@@ -188,13 +217,13 @@ async def download_to_file(url, filename, *,
         if is_cancelled(future):
             log.debug('Downloading was cancelled before doing the GET.')
             raise asyncio.CancelledError('Downloading was cancelled')
+        log.debug('Performing GET request, waiting for response.')
         return uncached_session.get(url, headers=headers, stream=True, verify=True)
 
     # Download the file in a different thread.
     def download_loop():
-        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        with with_existing_dir(filename, 'wb') as outfile:
-
+            with closing(response):
-        with closing(response), open(filename, 'wb') as outfile:
                 for block in response.iter_content(chunk_size=chunk_size):
                     if is_cancelled(future):
                         raise asyncio.CancelledError('Downloading was cancelled')
@@ -228,7 +257,8 @@ async def download_to_file(url, filename, *,
     # We're done downloading, now we have something cached we can use.
     log.debug('Saving header cache to %s', header_store)
     _downloaded_urls.add(url)
-    with open(header_store, 'w') as outfile:
+
+    with with_existing_dir(header_store, 'w') as outfile:
         json.dump({
             'ETag': str(response.headers.get('etag', '')),
             'Last-Modified': response.headers.get('Last-Modified'),
@@ -368,6 +398,66 @@ async def download_texture_thumbnail(texture_node, desired_size: str,
     loop.call_soon_threadsafe(thumbnail_loaded, texture_node, file_desc, thumb_path)
 
 
+async def download_file_by_uuid(file_uuid,
+                                target_directory: str,
+                                metadata_directory: str,
+                                *,
+                                file_loading: callable,
+                                file_loaded: callable,
+                                future: asyncio.Future):
+    if is_cancelled(future):
+        log.debug('download_file_by_uuid(%r) cancelled.', file_uuid)
+        return
+
+    loop = asyncio.get_event_loop()
+
+    # Find the File document.
+    api = pillar_api()
+    file_find = functools.partial(pillarsdk.File.find, params={
+        'projection': {'link': 1, 'filename': 1},
+    }, api=api)
+    file_desc = await loop.run_in_executor(None, file_find, file_uuid)
+
+    # Save the file document to disk
+    metadata_file = os.path.join(metadata_directory, 'files', '%s.json' % file_uuid)
+    save_as_json(file_desc, metadata_file)
+
+    file_path = os.path.join(target_directory, file_desc['filename'])
+    file_url = file_desc['link']
+    # log.debug('Texture %r:\n%s', file_uuid, pprint.pformat(file_desc.to_dict()))
+    loop.call_soon_threadsafe(file_loading, file_path, file_desc)
+
+    # Cached headers are stored in the project space
+    header_store = os.path.join(metadata_directory, 'files', '%s.headers' % file_uuid)
+
+    await download_to_file(file_url, file_path, header_store=header_store, future=future)
+
+    loop.call_soon_threadsafe(file_loaded, file_path, file_desc)
+
+
+async def download_texture(texture_node,
+                           target_directory: str,
+                           metadata_directory: str,
+                           *,
+                           texture_loading: callable,
+                           texture_loaded: callable,
+                           future: asyncio.Future):
+    if texture_node['node_type'] != 'texture':
+        raise TypeError("Node type should be 'texture', not %r" % texture_node['node_type'])
+
+    # Download every file. Eve doesn't support embedding from a list-of-dicts.
+    downloaders = (download_file_by_uuid(file_info['file'],
+                                         target_directory,
+                                         metadata_directory,
+                                         file_loading=texture_loading,
+                                         file_loaded=texture_loaded,
+                                         future=future)
+                   for file_info in texture_node['properties']['files'])
+
+    return await asyncio.gather(*downloaders)
+
+
 def is_cancelled(future: asyncio.Future) -> bool:
+    # assert future is not None  # for debugging purposes.
     cancelled = future is not None and future.cancelled()
     return cancelled