diff --git a/blender_cloud/gui.py b/blender_cloud/gui.py index be67861..a1504fb 100644 --- a/blender_cloud/gui.py +++ b/blender_cloud/gui.py @@ -107,7 +107,9 @@ class MenuItem: def update(self, node, file_desc, thumb_path: str, label_text): # We can get updated information about our Node, but a MenuItem should # always represent one node, and it shouldn't be shared between nodes. - assert self.node_uuid == node['_id'], "Don't change the node ID this MenuItem reflects, just create a new one." + if self.node_uuid != node['_id']: + raise ValueError("Don't change the node ID this MenuItem reflects, " + "just create a new one.") self.node = node self.file_desc = file_desc # pillarsdk.File object, or None if a 'folder' node. self.thumb_path = thumb_path @@ -427,14 +429,14 @@ class BlenderCloudBrowser(bpy.types.Operator): self.log.debug('Browsing assets at project %r node %r', self.project_uuid, self.node_uuid) self._new_async_task(self.async_download_previews(self.thumbnails_cache)) - def _new_async_task(self, async_task: asyncio.coroutine): + def _new_async_task(self, async_task: asyncio.coroutine, future: asyncio.Future=None): """Stops the currently running async task, and starts another one.""" self.log.debug('Setting up a new task %r, so any existing task must be stopped', async_task) self._stop_async_task() # Download the previews asynchronously. - self.signalling_future = asyncio.Future() + self.signalling_future = future or asyncio.Future() self.async_task = asyncio.ensure_future(async_task) self.log.debug('Created new task %r', self.async_task) @@ -548,19 +550,39 @@ class BlenderCloudBrowser(bpy.types.Operator): def handle_item_selection(self, context, item: MenuItem): """Called when the user clicks on a menu item that doesn't represent a folder.""" - # FIXME: Download all files from the texture node, instead of just one. - # FIXME: Properly set up header store. + self.clear_images() self._state = 'DOWNLOADING_TEXTURE' - url = item.file_desc.link - local_path = os.path.join(context.scene.blender_cloud_dir, item.file_desc.filename) - local_path = bpy.path.abspath(local_path) - self.log.info('Downloading %s to %s', url, local_path) + + node_path_components = [node['name'] for node in self.path_stack if node is not None] + local_path_components = [self.project_uuid] + node_path_components + [self.node['name']] + + top_texture_directory = bpy.path.abspath(context.scene.blender_cloud_dir) + local_path = os.path.join(top_texture_directory, *local_path_components) + meta_path = os.path.join(top_texture_directory, '.blender_cloud') + + self.log.info('Downloading texture %r to %s', item.node_uuid, local_path) + self.log.debug('Metadata will be stored at %s', meta_path) + + file_paths = [] + + def texture_downloading(file_path, file_desc, *args): + self.log.info('Texture downloading to %s', file_path) + + def texture_downloaded(file_path, file_desc, *args): + self.log.info('Texture downloaded to %r.', file_path) + bpy.data.images.load(filepath=file_path) + file_paths.append(file_path) def texture_download_completed(_): - self.log.info('Texture download complete, inspect %r.', local_path) + self.log.info('Texture download complete, inspect:\n%s', '\n'.join(file_paths)) self._state = 'QUIT' - self._new_async_task(pillar.download_to_file(url, local_path, header_store='/dev/null')) + signalling_future = asyncio.Future() + self._new_async_task(pillar.download_texture(item.node, local_path, + metadata_directory=meta_path, + texture_loading=texture_downloading, + texture_loaded=texture_downloaded, + future=signalling_future)) self.async_task.add_done_callback(texture_download_completed) diff --git a/blender_cloud/pillar.py b/blender_cloud/pillar.py index a1d6422..6f5884c 100644 --- a/blender_cloud/pillar.py +++ b/blender_cloud/pillar.py @@ -3,7 +3,8 @@ import json import os import functools import logging -from contextlib import closing +from contextlib import closing, contextmanager +import pprint import requests import requests.structures @@ -19,6 +20,7 @@ uncached_session = requests.session() _testing_blender_id_profile = None # Just for testing, overrides what is returned by blender_id_profile. _downloaded_urls = set() # URLs we've downloaded this Blender session. + class UserNotLoggedInError(RuntimeError): """Raised when the user should be logged in on Blender ID, but isn't. @@ -26,6 +28,33 @@ class UserNotLoggedInError(RuntimeError): """ +class PillarError(RuntimeError): + """Raised when there is some issue with the communication with Pillar. + + This is only raised for logical errors (for example nodes that should + exist but still can't be found). HTTP communication errors are signalled + with other exceptions. + """ + + +@contextmanager +def with_existing_dir(filename: str, open_mode: str, encoding=None): + """Opens a file, ensuring its directory exists.""" + + directory = os.path.dirname(filename) + if not os.path.exists(directory): + log.debug('Creating directory %s', directory) + os.makedirs(directory, mode=0o700, exist_ok=True) + with open(filename, open_mode, encoding=encoding) as file_object: + yield file_object + + +def save_as_json(pillar_resource, json_filename): + with with_existing_dir(json_filename, 'w') as outfile: + log.debug('Saving metadata to %r' % json_filename) + json.dump(pillar_resource, outfile, sort_keys=True, cls=pillarsdk.utils.PillarJSONEncoder) + + def blender_id_profile() -> dict: """Returns the Blender ID profile of the currently logged in user.""" @@ -188,17 +217,17 @@ async def download_to_file(url, filename, *, if is_cancelled(future): log.debug('Downloading was cancelled before doing the GET.') raise asyncio.CancelledError('Downloading was cancelled') + log.debug('Performing GET request, waiting for response.') return uncached_session.get(url, headers=headers, stream=True, verify=True) # Download the file in a different thread. def download_loop(): - os.makedirs(os.path.dirname(filename), exist_ok=True) - - with closing(response), open(filename, 'wb') as outfile: - for block in response.iter_content(chunk_size=chunk_size): - if is_cancelled(future): - raise asyncio.CancelledError('Downloading was cancelled') - outfile.write(block) + with with_existing_dir(filename, 'wb') as outfile: + with closing(response): + for block in response.iter_content(chunk_size=chunk_size): + if is_cancelled(future): + raise asyncio.CancelledError('Downloading was cancelled') + outfile.write(block) # Check for cancellation even before we start our GET request if is_cancelled(future): @@ -228,7 +257,8 @@ async def download_to_file(url, filename, *, # We're done downloading, now we have something cached we can use. log.debug('Saving header cache to %s', header_store) _downloaded_urls.add(url) - with open(header_store, 'w') as outfile: + + with with_existing_dir(header_store, 'w') as outfile: json.dump({ 'ETag': str(response.headers.get('etag', '')), 'Last-Modified': response.headers.get('Last-Modified'), @@ -368,6 +398,66 @@ async def download_texture_thumbnail(texture_node, desired_size: str, loop.call_soon_threadsafe(thumbnail_loaded, texture_node, file_desc, thumb_path) +async def download_file_by_uuid(file_uuid, + target_directory: str, + metadata_directory: str, + *, + file_loading: callable, + file_loaded: callable, + future: asyncio.Future): + if is_cancelled(future): + log.debug('download_file_by_uuid(%r) cancelled.', file_uuid) + return + + loop = asyncio.get_event_loop() + + # Find the File document. + api = pillar_api() + file_find = functools.partial(pillarsdk.File.find, params={ + 'projection': {'link': 1, 'filename': 1}, + }, api=api) + file_desc = await loop.run_in_executor(None, file_find, file_uuid) + + # Save the file document to disk + metadata_file = os.path.join(metadata_directory, 'files', '%s.json' % file_uuid) + save_as_json(file_desc, metadata_file) + + file_path = os.path.join(target_directory, file_desc['filename']) + file_url = file_desc['link'] + # log.debug('Texture %r:\n%s', file_uuid, pprint.pformat(file_desc.to_dict())) + loop.call_soon_threadsafe(file_loading, file_path, file_desc) + + # Cached headers are stored in the project space + header_store = os.path.join(metadata_directory, 'files', '%s.headers' % file_uuid) + + await download_to_file(file_url, file_path, header_store=header_store, future=future) + + loop.call_soon_threadsafe(file_loaded, file_path, file_desc) + + +async def download_texture(texture_node, + target_directory: str, + metadata_directory: str, + *, + texture_loading: callable, + texture_loaded: callable, + future: asyncio.Future): + if texture_node['node_type'] != 'texture': + raise TypeError("Node type should be 'texture', not %r" % texture_node['node_type']) + + # Download every file. Eve doesn't support embedding from a list-of-dicts. + downloaders = (download_file_by_uuid(file_info['file'], + target_directory, + metadata_directory, + file_loading=texture_loading, + file_loaded=texture_loaded, + future=future) + for file_info in texture_node['properties']['files']) + + return await asyncio.gather(*downloaders) + + def is_cancelled(future: asyncio.Future) -> bool: + # assert future is not None # for debugging purposes. cancelled = future is not None and future.cancelled() return cancelled