diff --git a/blender_cloud/gui.py b/blender_cloud/gui.py index 576050b..be67861 100644 --- a/blender_cloud/gui.py +++ b/blender_cloud/gui.py @@ -548,6 +548,8 @@ class BlenderCloudBrowser(bpy.types.Operator): def handle_item_selection(self, context, item: MenuItem): """Called when the user clicks on a menu item that doesn't represent a folder.""" + # FIXME: Download all files from the texture node, instead of just one. + # FIXME: Properly set up header store. self._state = 'DOWNLOADING_TEXTURE' url = item.file_desc.link local_path = os.path.join(context.scene.blender_cloud_dir, item.file_desc.filename) @@ -558,7 +560,7 @@ class BlenderCloudBrowser(bpy.types.Operator): self.log.info('Texture download complete, inspect %r.', local_path) self._state = 'QUIT' - self._new_async_task(pillar.download_to_file(url, local_path)) + self._new_async_task(pillar.download_to_file(url, local_path, header_store='/dev/null')) self.async_task.add_done_callback(texture_download_completed) diff --git a/blender_cloud/pillar.py b/blender_cloud/pillar.py index b431138..501c32f 100644 --- a/blender_cloud/pillar.py +++ b/blender_cloud/pillar.py @@ -1,11 +1,12 @@ import asyncio -import sys +import json import os import functools import logging from contextlib import closing import requests +import requests.structures import pillarsdk import pillarsdk.exceptions import pillarsdk.utils @@ -124,6 +125,7 @@ async def get_nodes(project_uuid: str = None, parent_node_uuid: str = None, node_all = functools.partial(pillarsdk.Node.all, { 'projection': {'name': 1, 'parent': 1, 'node_type': 1, 'properties.order': 1, 'properties.status': 1, + 'properties.files': 1, 'properties.content_type': 1, 'picture': 1}, 'where': where, 'sort': 'properties.order', @@ -135,10 +137,20 @@ async def get_nodes(project_uuid: str = None, parent_node_uuid: str = None, return children['_items'] -async def download_to_file(url, filename, chunk_size=100 * 1024, *, future: asyncio.Future = None): +async def download_to_file(url, filename, *, + header_store: str, + chunk_size=100 * 1024, + future: asyncio.Future = None): """Downloads a file via HTTP(S) directly to the filesystem.""" - # TODO: use the file's ETag header to check whether we need to redownload the file at all. + stored_headers = {} + if os.path.exists(header_store): + log.debug('Loading cached headers %r', header_store) + try: + with open(header_store, 'r') as infile: + stored_headers = requests.structures.CaseInsensitiveDict(json.load(infile)) + except Exception as ex: + log.warning('Unable to load headers from %r, ignoring cache: %s', header_store, str(ex)) loop = asyncio.get_event_loop() @@ -146,7 +158,19 @@ async def download_to_file(url, filename, chunk_size=100 * 1024, *, future: asyn # the download in between. def perform_get_request() -> requests.Request: - return uncached_session.get(url, stream=True, verify=True) + headers = {} + try: + if stored_headers['Last-Modified']: + headers['If-Modified-Since'] = stored_headers['Last-Modified'] + except KeyError: + pass + try: + if stored_headers['ETag']: + headers['If-None-Match'] = stored_headers['ETag'] + except KeyError: + pass + + return uncached_session.get(url, headers=headers, stream=True, verify=True) # Download the file in a different thread. def download_loop(): @@ -168,6 +192,10 @@ async def download_to_file(url, filename, chunk_size=100 * 1024, *, future: asyn log.debug('Status %i from GET %s', response.status_code, url) response.raise_for_status() + if response.status_code == 304: + # The file we have cached is still good, just use that instead. + return + # After we performed the GET request, we should check whether we should start # the download at all. if is_cancelled(future): @@ -178,6 +206,14 @@ async def download_to_file(url, filename, chunk_size=100 * 1024, *, future: asyn await loop.run_in_executor(None, download_loop) log.debug('Done downloading response of GET %s', url) + # We're done downloading, now we have something cached we can use. + log.debug('Saving header cache to %s', header_store) + with open(header_store, 'w') as outfile: + json.dump({ + 'ETag': str(response.headers.get('etag', '')), + 'Last-Modified': response.headers.get('Last-Modified'), + }, outfile, sort_keys=True) + async def fetch_thumbnail_info(file: pillarsdk.File, directory: str, desired_size: str, *, future: asyncio.Future = None): @@ -236,50 +272,6 @@ async def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str, is aborted. """ - api = pillar_api() - loop = asyncio.get_event_loop() - - file_find = functools.partial(pillarsdk.File.find, params={ - 'projection': {'filename': 1, 'variations': 1, 'width': 1, 'height': 1}, - }, api=api) - - async def handle_texture_node(texture_node): - # Skip non-texture nodes, as we can't thumbnail them anyway. - if texture_node['node_type'] != 'texture': - return - - if is_cancelled(future): - log.debug('fetch_texture_thumbs cancelled before finding File for texture %r', - texture_node['_id']) - return - - # Find the File that belongs to this texture node - pic_uuid = texture_node['picture'] - loop.call_soon_threadsafe(thumbnail_loading, texture_node, texture_node) - file_desc = await loop.run_in_executor(None, file_find, pic_uuid) - - if file_desc is None: - log.warning('Unable to find file for texture node %s', pic_uuid) - thumb_path = None - else: - if is_cancelled(future): - log.debug('fetch_texture_thumbs cancelled before downloading file %r', - file_desc['_id']) - return - - # Get the thumbnail information from Pillar - thumb_url, thumb_path = await fetch_thumbnail_info(file_desc, thumbnail_directory, desired_size, - future=future) - if thumb_path is None: - # The task got cancelled, we should abort too. - log.debug('fetch_texture_thumbs cancelled while downloading file %r', - file_desc['_id']) - return - - await download_to_file(thumb_url, thumb_path, future=future) - - loop.call_soon_threadsafe(thumbnail_loaded, texture_node, file_desc, thumb_path) - # Download all texture nodes in parallel. log.debug('Getting child nodes of node %r', parent_node_uuid) texture_nodes = await get_nodes(parent_node_uuid=parent_node_uuid, @@ -298,7 +290,10 @@ async def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str, log.debug('fetch_texture_thumbs: Gathering texture[%i:%i] for parent node %r', i, i + chunk_size, parent_node_uuid) - coros = (handle_texture_node(texture_node) + coros = (download_texture_thumbnail(texture_node, desired_size, + thumbnail_directory, + thumbnail_loading=thumbnail_loading, + thumbnail_loaded=thumbnail_loaded) for texture_node in chunk) # raises any exception from failed handle_texture_node() calls. @@ -307,7 +302,59 @@ async def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str, log.info('fetch_texture_thumbs: Done downloading texture thumbnails') +async def download_texture_thumbnail(texture_node, desired_size: str, + thumbnail_directory: str, + *, + thumbnail_loading: callable, + thumbnail_loaded: callable, + future: asyncio.Future = None): + # Skip non-texture nodes, as we can't thumbnail them anyway. + if texture_node['node_type'] != 'texture': + return + + if is_cancelled(future): + log.debug('fetch_texture_thumbs cancelled before finding File for texture %r', + texture_node['_id']) + return + + api = pillar_api() + loop = asyncio.get_event_loop() + + file_find = functools.partial(pillarsdk.File.find, params={ + 'projection': {'filename': 1, 'variations': 1, 'width': 1, 'height': 1}, + }, api=api) + + # Find the File that belongs to this texture node + pic_uuid = texture_node['picture'] + loop.call_soon_threadsafe(thumbnail_loading, texture_node, texture_node) + file_desc = await loop.run_in_executor(None, file_find, pic_uuid) + + if file_desc is None: + log.warning('Unable to find file for texture node %s', pic_uuid) + thumb_path = None + else: + if is_cancelled(future): + log.debug('fetch_texture_thumbs cancelled before downloading file %r', + file_desc['_id']) + return + + # Get the thumbnail information from Pillar + thumb_url, thumb_path = await fetch_thumbnail_info(file_desc, thumbnail_directory, + desired_size, future=future) + if thumb_path is None: + # The task got cancelled, we should abort too. + log.debug('fetch_texture_thumbs cancelled while downloading file %r', + file_desc['_id']) + return + + # Cached headers are stored next to thumbnails in sidecar files. + header_store = '%s.headers' % thumb_path + + await download_to_file(thumb_url, thumb_path, header_store=header_store, future=future) + + loop.call_soon_threadsafe(thumbnail_loaded, texture_node, file_desc, thumb_path) + + def is_cancelled(future: asyncio.Future) -> bool: cancelled = future is not None and future.cancelled() - log.debug('%s.cancelled() = %s', future, cancelled) return cancelled