Transitioned from synchronous to asynchronous
Using the new 'async def' functionality in Python 3.5! Still crappy GUI, we're going to replace that.
This commit is contained in:
parent
01b73a0439
commit
b274faf12c
@ -34,14 +34,16 @@ bl_info = {
|
||||
|
||||
import os.path
|
||||
import typing
|
||||
import asyncio
|
||||
|
||||
# Support reloading
|
||||
if 'pillar' in locals():
|
||||
import importlib
|
||||
|
||||
pillar = importlib.reload(pillar)
|
||||
async_loop = importlib.reload(async_loop)
|
||||
else:
|
||||
from . import pillar
|
||||
from . import pillar, async_loop
|
||||
|
||||
import bpy
|
||||
import bpy.utils.previews
|
||||
@ -151,31 +153,73 @@ def enum_previews_from_directory_items(self, context) -> typing.List[typing.AnyS
|
||||
return pcoll.previews
|
||||
|
||||
print('Loading previews for project {!r} node {!r}'.format(project_uuid, node_uuid))
|
||||
enum_items = []
|
||||
|
||||
if pcoll.async_task is not None and not pcoll.async_task.done():
|
||||
# We're still asynchronously downloading, but the UUIDs changed.
|
||||
print('Cancelling running async download task {}'.format(pcoll.async_task))
|
||||
pcoll.async_task.cancel()
|
||||
|
||||
# Download the previews asynchronously.
|
||||
pcoll.previews = []
|
||||
pcoll.project_uuid = project_uuid
|
||||
pcoll.node_uuid = node_uuid
|
||||
pcoll.context = context
|
||||
pcoll.async_task = asyncio.ensure_future(async_download_previews(wm.thumbnails_cache, pcoll))
|
||||
|
||||
# Start the async manager so everything happens.
|
||||
async_loop.ensure_async_loop()
|
||||
|
||||
return pcoll.previews
|
||||
|
||||
|
||||
async def async_download_previews(thumbnails_directory, pcoll):
|
||||
# If we have a node UUID, we fetch the textures
|
||||
# FIXME: support mixture of sub-nodes and textures under one node.
|
||||
if node_uuid:
|
||||
# Make sure we can go up again.
|
||||
parent = pillar.parent_node_uuid(node_uuid)
|
||||
enum_items.append(('node-{}'.format(parent), 'up', 'up',
|
||||
'FILE_FOLDER',
|
||||
len(enum_items)))
|
||||
enum_items = pcoll.previews
|
||||
|
||||
directory = os.path.join(wm.thumbnails_cache, project_uuid, node_uuid)
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
node_uuid = pcoll.node_uuid
|
||||
project_uuid = pcoll.project_uuid
|
||||
|
||||
for file_desc, thumb_path in pillar.fetch_texture_thumbs(node_uuid, 's', directory):
|
||||
def thumbnail_loading(file_desc):
|
||||
# TODO: trigger re-draw
|
||||
pass
|
||||
|
||||
try:
|
||||
region = pcoll.context.window.screen.regions[0]
|
||||
print('We have a region: {}'.format(region))
|
||||
except IndexError:
|
||||
region = None
|
||||
|
||||
def thumbnail_loaded(file_desc, thumb_path):
|
||||
thumb = pcoll.get(thumb_path)
|
||||
if thumb is None:
|
||||
thumb = pcoll.load(thumb_path, thumb_path, 'IMAGE')
|
||||
enum_items.append(('thumb-{}'.format(thumb_path), file_desc['filename'],
|
||||
thumb_path,
|
||||
# TODO: get something here that allows downloading the texture
|
||||
thumb.icon_id,
|
||||
len(enum_items)))
|
||||
# TODO: trigger re-draw
|
||||
if region is not None:
|
||||
try:
|
||||
region.tag_redraw()
|
||||
except Exception as e:
|
||||
print('WE DIE! ', e)
|
||||
|
||||
if node_uuid:
|
||||
# Make sure we can go up again.
|
||||
parent = await pillar.parent_node_uuid(node_uuid)
|
||||
enum_items.append(('node-{}'.format(parent), 'up', 'up',
|
||||
'FILE_FOLDER',
|
||||
len(enum_items)))
|
||||
|
||||
directory = os.path.join(thumbnails_directory, project_uuid, node_uuid)
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
|
||||
await pillar.fetch_texture_thumbs(node_uuid, 's', directory,
|
||||
thumbnail_loading=thumbnail_loading,
|
||||
thumbnail_loaded=thumbnail_loaded)
|
||||
elif project_uuid:
|
||||
children = pillar.get_nodes(project_uuid, '')
|
||||
children = await pillar.get_nodes(project_uuid, '')
|
||||
|
||||
for child in children:
|
||||
print(' - %(_id)s = %(name)s' % child)
|
||||
@ -183,11 +227,15 @@ def enum_previews_from_directory_items(self, context) -> typing.List[typing.AnyS
|
||||
'description',
|
||||
'FILE_FOLDER',
|
||||
len(enum_items)))
|
||||
|
||||
pcoll.previews = enum_items
|
||||
pcoll.project_uuid = project_uuid
|
||||
pcoll.node_uuid = node_uuid
|
||||
return pcoll.previews
|
||||
# TODO: trigger re-draw
|
||||
if region is not None:
|
||||
try:
|
||||
region.tag_redraw()
|
||||
except Exception as e:
|
||||
print('WE DIE! ', e)
|
||||
else:
|
||||
# TODO: add "nothing here" icon and trigger re-draw
|
||||
pass
|
||||
|
||||
|
||||
def enum_previews_from_directory_update(self, context):
|
||||
@ -224,7 +272,38 @@ class PreviewsExamplePanel(bpy.types.Panel):
|
||||
row.prop(wm, "blender_cloud_project")
|
||||
row.prop(wm, "blender_cloud_node")
|
||||
row.template_icon_view(wm, "blender_cloud_thumbnails", show_labels=True)
|
||||
row.prop(wm, "blender_cloud_thumbnails")
|
||||
# row.prop(wm, "blender_cloud_thumbnails")
|
||||
|
||||
|
||||
class AsyncOperator(Operator):
|
||||
bl_idname = 'async.action'
|
||||
bl_label = 'Asynchronous action'
|
||||
bl_description = ''
|
||||
|
||||
def execute(self, context):
|
||||
print('{}: executing'.format(self))
|
||||
|
||||
asyncio.ensure_future(do_async_stuff(context))
|
||||
async_loop.ensure_async_loop()
|
||||
print('{}: done'.format(self))
|
||||
|
||||
return {'FINISHED'}
|
||||
|
||||
|
||||
async def do_async_stuff(context):
|
||||
print('do_async_stuff(): starting')
|
||||
|
||||
wm = context.window_manager
|
||||
project_uuid = wm.blender_cloud_project
|
||||
print('Loading nodes for project {!r}'.format(project_uuid))
|
||||
|
||||
children = await pillar.get_nodes(project_uuid, '')
|
||||
|
||||
for child in children:
|
||||
print(' - %(_id)s = %(name)s' % child)
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
print('do_async_stuff(): done')
|
||||
|
||||
|
||||
def register():
|
||||
@ -260,6 +339,8 @@ def register():
|
||||
pcoll.previews = ()
|
||||
pcoll.project_uuid = ''
|
||||
pcoll.node_uuid = ''
|
||||
pcoll.async_task = None
|
||||
pcoll.context = None
|
||||
|
||||
preview_collections["blender_cloud"] = pcoll
|
||||
|
||||
|
||||
49
blender_cloud/async_loop.py
Normal file
49
blender_cloud/async_loop.py
Normal file
@ -0,0 +1,49 @@
|
||||
"""Manages the asyncio loop."""
|
||||
|
||||
import asyncio
|
||||
import bpy
|
||||
|
||||
|
||||
def kick_async_loop(*args):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
if loop.is_closed():
|
||||
print('{}: loop closed, stopping'.format(__name__))
|
||||
stop_async_loop()
|
||||
return
|
||||
|
||||
if not asyncio.Task.all_tasks():
|
||||
print('{}: no more scheduled tasks, stopping'.format(__name__))
|
||||
stop_async_loop()
|
||||
return
|
||||
|
||||
# Perform a single async loop step
|
||||
async def do_nothing():
|
||||
pass
|
||||
|
||||
loop.run_until_complete(do_nothing())
|
||||
|
||||
|
||||
def async_loop_handler() -> callable:
|
||||
name = kick_async_loop.__name__
|
||||
for handler in bpy.app.handlers.scene_update_pre:
|
||||
if getattr(handler, '__name__', '') == name:
|
||||
return handler
|
||||
return None
|
||||
|
||||
|
||||
def ensure_async_loop():
|
||||
if async_loop_handler() is not None:
|
||||
return
|
||||
bpy.app.handlers.scene_update_pre.append(kick_async_loop)
|
||||
|
||||
|
||||
def stop_async_loop():
|
||||
handler = async_loop_handler()
|
||||
if handler is None:
|
||||
return
|
||||
bpy.app.handlers.scene_update_pre.remove(handler)
|
||||
|
||||
# Optional: cancel all pending tasks.
|
||||
# for task in asyncio.Task.all_tasks():
|
||||
# task.cancel()
|
||||
@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
import concurrent.futures
|
||||
@ -14,6 +15,7 @@ if not any('pillar_sdk' in path for path in sys.path):
|
||||
|
||||
import pillarsdk
|
||||
import pillarsdk.exceptions
|
||||
import pillarsdk.utils
|
||||
|
||||
_pillar_api = None # will become a pillarsdk.Api object.
|
||||
|
||||
@ -62,14 +64,17 @@ def pillar_api() -> pillarsdk.Api:
|
||||
return _pillar_api
|
||||
|
||||
|
||||
def get_project_uuid(project_url: str) -> str:
|
||||
async def get_project_uuid(project_url: str) -> str:
|
||||
"""Returns the UUID for the project, given its '/p/<project_url>' string."""
|
||||
|
||||
try:
|
||||
project = pillarsdk.Project.find_one({
|
||||
find_one = functools.partial(pillarsdk.Project.find_one, {
|
||||
'where': {'url': project_url},
|
||||
'projection': {'permissions': 1},
|
||||
}, api=pillar_api())
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
project = await loop.run_in_executor(None, find_one)
|
||||
except pillarsdk.exceptions.ResourceNotFound:
|
||||
print('Project with URL %r does not exist' % project_url)
|
||||
return None
|
||||
@ -78,7 +83,7 @@ def get_project_uuid(project_url: str) -> str:
|
||||
return project['_id']
|
||||
|
||||
|
||||
def get_nodes(project_uuid: str = None, parent_node_uuid: str = None) -> list:
|
||||
async def get_nodes(project_uuid: str = None, parent_node_uuid: str = None) -> list:
|
||||
"""Gets nodes for either a project or given a parent node.
|
||||
|
||||
@param project_uuid: the UUID of the project, or None if only querying by parent_node_uuid.
|
||||
@ -102,73 +107,112 @@ def get_nodes(project_uuid: str = None, parent_node_uuid: str = None) -> list:
|
||||
if project_uuid:
|
||||
where['project'] = project_uuid
|
||||
|
||||
children = pillarsdk.Node.all({
|
||||
node_all = functools.partial(pillarsdk.Node.all, {
|
||||
'projection': {'name': 1, 'parent': 1, 'node_type': 1,
|
||||
'properties.order': 1, 'properties.status': 1,
|
||||
'properties.content_type': 1, 'picture': 1,
|
||||
'permissions': 1, 'project': 1, # for permission checking
|
||||
},
|
||||
'properties.content_type': 1, 'picture': 1},
|
||||
'where': where,
|
||||
'sort': 'properties.order'},
|
||||
api=pillar_api())
|
||||
'sort': 'properties.order'}, api=pillar_api())
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
children = await loop.run_in_executor(None, node_all)
|
||||
|
||||
return children['_items']
|
||||
|
||||
|
||||
def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str, thumbnail_directory: str):
|
||||
async def download_to_file(url, filename, chunk_size=10 * 1024):
|
||||
"""Downloads a file via HTTP(S) directly to the filesystem."""
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
await loop.run_in_executor(None, pillarsdk.utils.download_to_file, url, filename, chunk_size)
|
||||
|
||||
|
||||
async def stream_thumb_to_file(file: pillarsdk.File, directory: str, desired_size: str):
|
||||
"""Streams a thumbnail to a file.
|
||||
|
||||
@param file: the pillar File object that represents the image whose thumbnail to download.
|
||||
@param directory: the directory to save the file to.
|
||||
@param desired_size: thumbnail size
|
||||
@return: the absolute path of the downloaded file.
|
||||
"""
|
||||
|
||||
api = pillar_api()
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
thumb_link = await loop.run_in_executor(None, functools.partial(
|
||||
file.thumbnail_file, desired_size, api=api))
|
||||
|
||||
if thumb_link is None:
|
||||
raise ValueError("File {} has no thumbnail of size {}"
|
||||
.format(file['_id'], desired_size))
|
||||
|
||||
root, ext = os.path.splitext(file['file_path'])
|
||||
thumb_fname = "{0}-{1}.jpg".format(root, desired_size)
|
||||
thumb_path = os.path.abspath(os.path.join(directory, thumb_fname))
|
||||
|
||||
await download_to_file(thumb_link, thumb_path)
|
||||
|
||||
return thumb_path
|
||||
|
||||
|
||||
async def fetch_texture_thumbs(parent_node_uuid: str, desired_size: str,
|
||||
thumbnail_directory: str,
|
||||
*,
|
||||
thumbnail_loading: callable,
|
||||
thumbnail_loaded: callable):
|
||||
"""Generator, fetches all texture thumbnails in a certain parent node.
|
||||
|
||||
@param parent_node_uuid: the UUID of the parent node. All sub-nodes will be downloaded.
|
||||
@param desired_size: size indicator, from 'sbtmlh'.
|
||||
@param thumbnail_directory: directory in which to store the downloaded thumbnails.
|
||||
@returns: generator that yields (pillarsdk.File object, thumbnail path) tuples
|
||||
@param thumbnail_loading: callback function that takes (pillarsdk.File object)
|
||||
parameter, which is called before a thumbnail will be downloaded. This allows you to
|
||||
show a "downloading" indicator.
|
||||
@param thumbnail_loaded: callback function that takes (pillarsdk.File object, thumbnail path)
|
||||
parameters, which is called for every thumbnail after it's been downloaded.
|
||||
"""
|
||||
|
||||
api = pillar_api()
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def fetch_thumbnail_from_node(texture_node: pillarsdk.Node):
|
||||
# Fetch the File description JSON
|
||||
pic_uuid = texture_node['picture']
|
||||
file_desc = pillarsdk.File.find(pic_uuid, {
|
||||
file_find = functools.partial(pillarsdk.File.find, params={
|
||||
'projection': {'filename': 1, 'variations': 1, 'width': 1, 'height': 1},
|
||||
}, api=api)
|
||||
|
||||
# TODO: Still single-threaded, could branch out to a few threads here to download in parallel.
|
||||
texture_nodes = await get_nodes(parent_node_uuid=parent_node_uuid)
|
||||
for texture_node in texture_nodes:
|
||||
# Skip non-texture nodes, as we can't thumbnail them anyway.
|
||||
if texture_node['node_type'] != 'texture':
|
||||
continue
|
||||
|
||||
# Find the File that belongs to this texture node
|
||||
pic_uuid = texture_node['picture']
|
||||
file_desc = await loop.run_in_executor(None, file_find, pic_uuid)
|
||||
loop.call_soon_threadsafe(functools.partial(thumbnail_loading, file_desc))
|
||||
|
||||
if file_desc is None:
|
||||
print('Unable to find picture {}'.format(pic_uuid))
|
||||
return None, None
|
||||
|
||||
print('Unable to find file for texture node {}'.format(pic_uuid))
|
||||
thumb_path = None
|
||||
else:
|
||||
# Save the thumbnail
|
||||
thumb_path = file_desc.stream_thumb_to_file(thumbnail_directory, desired_size, api=api)
|
||||
thumb_path = await stream_thumb_to_file(file_desc, thumbnail_directory, desired_size)
|
||||
# print('Texture node {} has file {}'.format(texture_node['_id'], thumb_path))
|
||||
|
||||
return file_desc, thumb_path
|
||||
|
||||
texture_nodes = (node for node in get_nodes(parent_node_uuid=parent_node_uuid)
|
||||
if node['node_type'] == 'texture')
|
||||
|
||||
# # Single-threaded, not maintained:
|
||||
# for node in texture_nodes:
|
||||
# node, file = fetch_thumbnail_from_node(node)
|
||||
# print('Node {} has picture {}'.format(node, file))
|
||||
|
||||
# Multi-threaded:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
||||
# Queue up fetching of thumbnails
|
||||
futures = [executor.submit(fetch_thumbnail_from_node, node)
|
||||
for node in texture_nodes]
|
||||
|
||||
for future in futures:
|
||||
file_desc, thumb_path = future.result()
|
||||
yield file_desc, thumb_path
|
||||
loop.call_soon_threadsafe(functools.partial(thumbnail_loaded, file_desc, thumb_path))
|
||||
|
||||
print('Done downloading texture thumbnails')
|
||||
|
||||
|
||||
@functools.lru_cache(128)
|
||||
def parent_node_uuid(node_uuid: str) -> str:
|
||||
async def parent_node_uuid(node_uuid: str) -> str:
|
||||
"""Returns the UUID of the node's parent node, or an empty string if this is the top level."""
|
||||
|
||||
api = pillar_api()
|
||||
node = pillarsdk.Node.find(node_uuid, {'projection': {'parent': 1}}, api=api)
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
find_node = functools.partial(pillarsdk.Node.find, node_uuid,
|
||||
{'projection': {'parent': 1}}, api=api)
|
||||
node = await loop.run_in_executor(None, find_node)
|
||||
if node is None:
|
||||
return ''
|
||||
|
||||
|
||||
Reference in New Issue
Block a user