From 8533ac83ba2949cb089e0a983907ca364618aa6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Mon, 14 Mar 2016 09:33:43 +0100 Subject: [PATCH] Added parallel downloading of thumbnails. By using asyncio.gather, all thumbnail-downloading tasks are queued simultaneously, and downloaded in parallel by the task executor. --- blender_cloud/pillar.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/blender_cloud/pillar.py b/blender_cloud/pillar.py index a4f36a5..cbdaaaa 100644 --- a/blender_cloud/pillar.py +++ b/blender_cloud/pillar.py @@ -179,12 +179,10 @@ async def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str, 'projection': {'filename': 1, 'variations': 1, 'width': 1, 'height': 1}, }, api=api) - # TODO: Still single-threaded, could branch out to a few threads here to download in parallel. - texture_nodes = await get_nodes(parent_node_uuid=parent_node_uuid) - for texture_node in texture_nodes: + async def handle_texture_node(texture_node): # Skip non-texture nodes, as we can't thumbnail them anyway. if texture_node['node_type'] != 'texture': - continue + return # Find the File that belongs to this texture node pic_uuid = texture_node['picture'] @@ -201,6 +199,13 @@ async def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str, loop.call_soon_threadsafe(functools.partial(thumbnail_loaded, file_desc, thumb_path)) + # Download all texture nodes in parallel. + texture_nodes = await get_nodes(parent_node_uuid=parent_node_uuid) + + # raises any exception from failed handle_texture_node() calls. + await asyncio.gather(*(handle_texture_node(texture_node) + for texture_node in texture_nodes)) + print('Done downloading texture thumbnails')