From 570b1d4bfefa2afa69c99e522676c441dae71cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Fri, 13 Jan 2017 17:24:37 +0100 Subject: [PATCH] Initial Flamenco support. Lots to do: - Doesn't call BAM yet to copy files onto the job storage folder (even though you can configure that folder). - Uses the same project as Attract, so you have to select it in an unintuitive location. Also, you can only start Flamenco jobs on a project that is Attract-enabled (and not necessarily Flamenco-enabled). --- blender_cloud/__init__.py | 8 +- blender_cloud/blender.py | 102 ++++++------ blender_cloud/flamenco/__init__.py | 242 +++++++++++++++++++++++++++++ blender_cloud/flamenco/sdk.py | 13 ++ blender_cloud/pillar.py | 26 ++++ blender_cloud/utils.py | 38 +++++ 6 files changed, 378 insertions(+), 51 deletions(-) create mode 100644 blender_cloud/flamenco/__init__.py create mode 100644 blender_cloud/flamenco/sdk.py diff --git a/blender_cloud/__init__.py b/blender_cloud/__init__.py index 6f30e4e..b554d16 100644 --- a/blender_cloud/__init__.py +++ b/blender_cloud/__init__.py @@ -79,13 +79,15 @@ def register(): settings_sync = reload_mod('settings_sync') image_sharing = reload_mod('image_sharing') attract = reload_mod('attract') + flamenco = reload_mod('flamenco') else: from . import (blender, texture_browser, async_loop, settings_sync, blendfile, home_project, - image_sharing, attract) + image_sharing, attract, flamenco) async_loop.setup_asyncio_executor() async_loop.register() + flamenco.register() texture_browser.register() blender.register() settings_sync.register() @@ -111,7 +113,8 @@ def _monkey_patch_requests(): def unregister(): - from . import blender, texture_browser, async_loop, settings_sync, image_sharing, attract + from . import (blender, texture_browser, async_loop, settings_sync, image_sharing, attract, + flamenco) image_sharing.unregister() attract.unregister() @@ -119,3 +122,4 @@ def unregister(): blender.unregister() texture_browser.unregister() async_loop.unregister() + flamenco.unregister() diff --git a/blender_cloud/blender.py b/blender_cloud/blender.py index 0c60398..8458225 100644 --- a/blender_cloud/blender.py +++ b/blender_cloud/blender.py @@ -29,7 +29,8 @@ from bpy.types import AddonPreferences, Operator, WindowManager, Scene, Property from bpy.props import StringProperty, EnumProperty, PointerProperty, BoolProperty import rna_prop_ui -from . import pillar, async_loop +from . import pillar, async_loop, flamenco +from .utils import pyside_cache, redraw PILLAR_WEB_SERVER_URL = 'https://cloud.blender.org/' # PILLAR_WEB_SERVER_URL = 'http://pillar-web:5001/' @@ -41,39 +42,6 @@ log = logging.getLogger(__name__) icons = None -def redraw(self, context): - context.area.tag_redraw() - - -def pyside_cache(propname): - - if callable(propname): - raise TypeError('Usage: pyside_cache("property_name")') - - def decorator(wrapped): - """Stores the result of the callable in Python-managed memory. - - This is to work around the warning at - https://www.blender.org/api/blender_python_api_master/bpy.props.html#bpy.props.EnumProperty - """ - - import functools - - @functools.wraps(wrapped) - # We can't use (*args, **kwargs), because EnumProperty explicitly checks - # for the number of fixed positional arguments. - def wrapper(self, context): - result = None - try: - result = wrapped(self, context) - return result - finally: - rna_type, rna_info = getattr(self.bl_rna, propname) - rna_info['_cached_result'] = result - return wrapper - return decorator - - @pyside_cache('version') def blender_syncable_versions(self, context): """Returns the list of items used by SyncStatusProperties.version EnumProperty.""" @@ -207,6 +175,16 @@ class BlenderCloudPreferences(AddonPreferences): subtype='DIR_PATH', default='//../') + flamenco_manager = PointerProperty(type=flamenco.FlamencoManagerGroup) + # TODO: before making Flamenco public, change the defaults to something less Institute-specific. + # NOTE: The assumption is that the workers can also find the files in the same path. + # This assumption is true for the Blender Institute, but we should allow other setups too. + flamenco_job_file_path = StringProperty( + name='Job file path', + description='Path where to store job files', + subtype='DIR_PATH', + default='/render/_flamenco/storage') + def draw(self, context): import textwrap @@ -291,6 +269,10 @@ class BlenderCloudPreferences(AddonPreferences): attract_box = layout.box() self.draw_attract_buttons(attract_box, self.attract_project) + # Flamenco stuff + flamenco_box = layout.box() + self.draw_flamenco_buttons(flamenco_box, self.flamenco_manager) + def draw_subscribe_button(self, layout): layout.operator('pillar.subscribe', icon='WORLD') @@ -349,6 +331,35 @@ class BlenderCloudPreferences(AddonPreferences): attract_box.prop(self, 'attract_project_local_path') + def draw_flamenco_buttons(self, flamenco_box, bcp: flamenco.FlamencoManagerGroup): + flamenco_row = flamenco_box.row(align=True) + flamenco_row.label('Flamenco', icon_value=icon('CLOUD')) + + flamenco_row.enabled = bcp.status in {'NONE', 'IDLE'} + row_buttons = flamenco_row.row(align=True) + + if bcp.status in {'NONE', 'IDLE'}: + if not bcp.available_managers or not bcp.manager: + row_buttons.operator('flamenco.managers', + text='Find Flamenco Managers', + icon='FILE_REFRESH') + else: + row_buttons.prop(bcp, 'manager', text='Manager') + row_buttons.operator('flamenco.managers', + text='', + icon='FILE_REFRESH') + else: + row_buttons.label('Fetching available managers.') + + # TODO: make a reusable way to select projects, and use that for Attract and Flamenco. + note_box = flamenco_box.column(align=True) + note_box.label('NOTE: For now, Flamenco uses the same project as Attract.') + note_box.label('This will change in a future version of the add-on.') + + flamenco_box.prop(self, 'flamenco_job_file_path') + note_box = flamenco_box.column(align=True) + note_box.label('NOTE: Flamenco assumes the workers can use this path too.') + class PillarCredentialsUpdate(pillar.PillarOperatorMixin, Operator): @@ -414,7 +425,7 @@ class PILLAR_OT_subscribe(Operator): class PILLAR_OT_projects(async_loop.AsyncModalOperatorMixin, - pillar.PillarOperatorMixin, + pillar.AuthenticatedPillarOperatorMixin, Operator): """Fetches the projects available to the user""" bl_idname = 'pillar.projects' @@ -424,27 +435,20 @@ class PILLAR_OT_projects(async_loop.AsyncModalOperatorMixin, _log = logging.getLogger('bpy.ops.%s' % bl_idname) async def async_execute(self, context): + if not await self.authenticate(context): + return + import pillarsdk from .pillar import pillar_call - self._log.info('Checking credentials') - try: - db_user = await self.check_credentials(context, ()) - except pillar.UserNotLoggedInError as ex: - self._log.info('Not logged in error raised: %s', ex) - self.report({'ERROR'}, 'Please log in on Blender ID first.') - self.quit() - return - - user_id = db_user['_id'] - self.log.info('Going to fetch projects for user %s', user_id) + self.log.info('Going to fetch projects for user %s', self.user_id) preferences().attract_project.status = 'FETCHING' # Get all projects, except the home project. projects_user = await pillar_call( pillarsdk.Project.all, - {'where': {'user': user_id, + {'where': {'user': self.user_id, 'category': {'$ne': 'home'}}, 'sort': '-_created', 'projection': {'_id': True, @@ -453,8 +457,8 @@ class PILLAR_OT_projects(async_loop.AsyncModalOperatorMixin, projects_shared = await pillar_call( pillarsdk.Project.all, - {'where': {'user': {'$ne': user_id}, - 'permissions.groups.group': {'$in': db_user.groups}}, + {'where': {'user': {'$ne': self.user_id}, + 'permissions.groups.group': {'$in': self.db_user.groups}}, 'sort': '-_created', 'projection': {'_id': True, 'name': True}, diff --git a/blender_cloud/flamenco/__init__.py b/blender_cloud/flamenco/__init__.py new file mode 100644 index 0000000..a1aeb55 --- /dev/null +++ b/blender_cloud/flamenco/__init__.py @@ -0,0 +1,242 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +"""Flamenco interface. + +The preferences are managed blender.py, the rest of the Flamenco-specific stuff is here. +""" +import logging + +import bpy +from bpy.types import AddonPreferences, Operator, WindowManager, Scene, PropertyGroup +from bpy.props import StringProperty, EnumProperty, PointerProperty, BoolProperty, IntProperty + +from .. import async_loop, pillar +from ..utils import pyside_cache, redraw + +log = logging.getLogger(__name__) + + +@pyside_cache('manager') +def available_managers(self, context): + """Returns the list of items used by a manager-selector EnumProperty.""" + + from ..blender import preferences + + mngrs = preferences().flamenco_manager.available_managers + if not mngrs: + return [('', 'No managers available in your Blender Cloud', '')] + return [(p['_id'], p['name'], '') for p in mngrs] + + +class FlamencoManagerGroup(PropertyGroup): + manager = EnumProperty( + items=available_managers, + name='Flamenco Manager', + description='Which Flamenco Manager to use for jobs') + + status = EnumProperty( + items=[ + ('NONE', 'NONE', 'We have done nothing at all yet'), + ('IDLE', 'IDLE', 'User requested something, which is done, and we are now idle'), + ('FETCHING', 'FETCHING', 'Fetching available Flamenco managers from Blender Cloud'), + ], + name='status', + update=redraw) + + # List of managers is stored in 'available_managers' ID property, + # because I don't know how to store a variable list of strings in a proper RNA property. + @property + def available_managers(self) -> list: + return self.get('available_managers', []) + + @available_managers.setter + def available_managers(self, new_managers): + self['available_managers'] = new_managers + + +class FLAMENCO_OT_fmanagers(async_loop.AsyncModalOperatorMixin, + pillar.AuthenticatedPillarOperatorMixin, + Operator): + """Fetches the Flamenco Managers available to the user""" + bl_idname = 'flamenco.managers' + bl_label = 'Fetch available Flamenco Managers' + + stop_upon_exception = True + _log = logging.getLogger('bpy.ops.%s' % bl_idname) + + @property + def mypref(self) -> FlamencoManagerGroup: + from ..blender import preferences + + return preferences().flamenco_manager + + async def async_execute(self, context): + if not await self.authenticate(context): + return + + from .sdk import Manager + from ..pillar import pillar_call + + self.log.info('Going to fetch managers for user %s', self.user_id) + + self.mypref.status = 'FETCHING' + managers = await pillar_call(Manager.all) + + # We need to convert to regular dicts before storing in ID properties. + # Also don't store more properties than we need. + as_list = [{'_id': p['_id'], 'name': p['name']} for p in managers['_items']] + + self.mypref.available_managers = as_list + self.quit() + + def quit(self): + self.mypref.status = 'IDLE' + super().quit() + + +class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin, + pillar.AuthenticatedPillarOperatorMixin, + Operator): + """Performs a Blender render on Flamenco.""" + bl_idname = 'flamenco.render' + bl_label = 'Render on Flamenco' + + stop_upon_exception = True + _log = logging.getLogger('bpy.ops.%s' % bl_idname) + + async def async_execute(self, context): + if not await self.authenticate(context): + return + + import os.path + from ..blender import preferences + from pillarsdk import exceptions as sdk_exceptions + + prefs = preferences() + scene = context.scene + + try: + await create_job(self.user_id, + prefs.attract_project.project, + prefs.flamenco_manager.manager, + 'blender-render', + { + "blender_cmd": "{blender}", + "chunk_size": scene.flamenco_render_chunk_size, + "filepath": context.blend_data.filepath, + "frames": scene.flamenco_render_frame_range + }, + 'Render %s' % os.path.basename(context.blend_data.filepath)) + except sdk_exceptions.ResourceInvalid as ex: + self.report({'ERROR'}, 'Error creating Flamenco job: %s' % ex) + else: + self.report({'INFO'}, 'Flamenco job created.') + self.quit() + + +class FLAMENCO_OT_scene_to_frame_range(Operator): + """Sets the scene frame range as the Flamenco render frame range.""" + bl_idname = 'flamenco.scene_to_frame_range' + bl_label = 'Sets the scene frame range as the Flamenco render frame range' + + def execute(self, context): + s = context.scene + s.flamenco_render_frame_range = '%i-%i' % (s.frame_start, s.frame_end) + return {'FINISHED'} + + +async def create_job(user_id: str, + project_id: str, + manager_id: str, + job_type: str, + job_settings: dict, + job_name: str = None, + *, + job_description: str = None) -> str: + """Creates a render job at Flamenco Server, returning the job ID.""" + + import json + from .sdk import Job + from ..pillar import pillar_call + + job_attrs = { + 'status': 'queued', + 'priority': 50, + 'name': job_name, + 'settings': job_settings, + 'job_type': job_type, + 'user': user_id, + 'manager': manager_id, + 'project': project_id, + } + if job_description: + job_attrs['description'] = job_description + + log.info('Going to create Flamenco job:\n%s', + json.dumps(job_attrs, indent=4, sort_keys=True)) + + job = Job(job_attrs) + await pillar_call(job.create) + + log.info('Job created succesfully: %s', job._id) + return job._id + + +def draw_render_button(self, context): + layout = self.layout + + from ..blender import icon + + flamenco_box = layout.box() + flamenco_box.label('Flamenco', icon_value=icon('CLOUD')) + flamenco_box.prop(context.scene, 'flamenco_render_chunk_size') + + frange_row = flamenco_box.row(align=True) + frange_row.prop(context.scene, 'flamenco_render_frame_range') + frange_row.operator('flamenco.scene_to_frame_range', text='', icon='ARROW_LEFTRIGHT') + + flamenco_box.operator('flamenco.render', text='Render on Flamenco', icon='RENDER_ANIMATION') + + +def register(): + bpy.utils.register_class(FlamencoManagerGroup) + bpy.utils.register_class(FLAMENCO_OT_fmanagers) + bpy.utils.register_class(FLAMENCO_OT_render) + bpy.utils.register_class(FLAMENCO_OT_scene_to_frame_range) + + scene = bpy.types.Scene + scene.flamenco_render_chunk_size = IntProperty( + name='Chunk size', + description='Maximum number of frames to render per task', + default=10, + ) + scene.flamenco_render_frame_range = StringProperty( + name='Frame range', + description='Frames to render, in "printer range" notation' + ) + + bpy.types.RENDER_PT_render.append(draw_render_button) + + +def unregister(): + bpy.types.RENDER_PT_render.remove(draw_render_button) + bpy.utils.unregister_module(__name__) + + wm = bpy.types.WindowManager + del wm.flamenco_render_chunk_size diff --git a/blender_cloud/flamenco/sdk.py b/blender_cloud/flamenco/sdk.py new file mode 100644 index 0000000..33f2308 --- /dev/null +++ b/blender_cloud/flamenco/sdk.py @@ -0,0 +1,13 @@ +from pillarsdk.resource import List, Find, Create + + +class Manager(List, Find): + """Manager class wrapping the REST nodes endpoint""" + path = 'flamenco/managers' + + +class Job(List, Find, Create): + """Job class wrapping the REST nodes endpoint + """ + path = 'flamenco/jobs' + ensure_query_projections = {'project': 1} diff --git a/blender_cloud/pillar.py b/blender_cloud/pillar.py index 01ac390..9029d41 100644 --- a/blender_cloud/pillar.py +++ b/blender_cloud/pillar.py @@ -840,6 +840,32 @@ class PillarOperatorMixin: 'Please subscribe to the blender cloud at https://cloud.blender.org/join') +class AuthenticatedPillarOperatorMixin(PillarOperatorMixin): + """Checks credentials, to be used at the start of async_execute(). + + Sets self.user_id to the current user's ID, and self.db_user to the user info dict, + if authentication was succesful; sets both to None if not. + """ + + async def authenticate(self, context) -> bool: + from . import pillar + + self._log.info('Checking credentials') + self.user_id = None + self.db_user = None + try: + self.db_user = await self.check_credentials(context, ()) + except pillar.UserNotLoggedInError as ex: + self._log.info('Not logged in error raised: %s', ex) + self.report({'ERROR'}, 'Please log in on Blender ID first.') + self.quit() + return False + + self.user_id = self.db_user['_id'] + return True + + + async def find_or_create_node(where: dict, additional_create_props: dict = None, projection: dict = None, diff --git a/blender_cloud/utils.py b/blender_cloud/utils.py index 5512a3a..aca4bfb 100644 --- a/blender_cloud/utils.py +++ b/blender_cloud/utils.py @@ -62,3 +62,41 @@ def find_in_path(path: pathlib.Path, filename: str) -> pathlib.Path: return subpath return None + + +def pyside_cache(propname): + """Decorator, stores the result of the decorated callable in Python-managed memory. + + This is to work around the warning at + https://www.blender.org/api/blender_python_api_master/bpy.props.html#bpy.props.EnumProperty + """ + + if callable(propname): + raise TypeError('Usage: pyside_cache("property_name")') + + def decorator(wrapped): + """Stores the result of the callable in Python-managed memory. + + This is to work around the warning at + https://www.blender.org/api/blender_python_api_master/bpy.props.html#bpy.props.EnumProperty + """ + + import functools + + @functools.wraps(wrapped) + # We can't use (*args, **kwargs), because EnumProperty explicitly checks + # for the number of fixed positional arguments. + def wrapper(self, context): + result = None + try: + result = wrapped(self, context) + return result + finally: + rna_type, rna_info = getattr(self.bl_rna, propname) + rna_info['_cached_result'] = result + return wrapper + return decorator + + +def redraw(self, context): + context.area.tag_redraw()