archive/blender-cloud-addon
Archived
1
3
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Initial Flamenco support.

Browse Source 
Lots to do:

- Doesn't call BAM yet to copy files onto the job storage folder (even
  though you can configure that folder).
- Uses the same project as Attract, so you have to select it in an
  unintuitive location. Also, you can only start Flamenco jobs on a project
  that is Attract-enabled (and not necessarily Flamenco-enabled).
This commit is contained in:
Sybren A. Stüvel
2017-01-13 17:24:37 +01:00
parent 68b046c714
commit 570b1d4bfe
6 changed files with 378 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
blender_cloud/__init__.py
View File

@@ -79,13 +79,15 @@ def register():
settings_sync = reload_mod('settings_sync')
image_sharing = reload_mod('image_sharing')
attract = reload_mod('attract')
flamenco = reload_mod('flamenco')
else:
from . import (blender, texture_browser, async_loop, settings_sync, blendfile, home_project,
image_sharing, attract)
image_sharing, attract, flamenco)
async_loop.setup_asyncio_executor()
async_loop.register()
flamenco.register()
texture_browser.register()
blender.register()
settings_sync.register()
@@ -111,7 +113,8 @@ def _monkey_patch_requests():
def unregister():
from . import blender, texture_browser, async_loop, settings_sync, image_sharing, attract
from . import (blender, texture_browser, async_loop, settings_sync, image_sharing, attract,
flamenco)
image_sharing.unregister()
attract.unregister()
@@ -119,3 +122,4 @@ def unregister():
blender.unregister()
texture_browser.unregister()
async_loop.unregister()
flamenco.unregister()

102
blender_cloud/blender.py
View File

@@ -29,7 +29,8 @@ from bpy.types import AddonPreferences, Operator, WindowManager, Scene, Property
from bpy.props import StringProperty, EnumProperty, PointerProperty, BoolProperty
import rna_prop_ui
from . import pillar, async_loop
from . import pillar, async_loop, flamenco
from .utils import pyside_cache, redraw
PILLAR_WEB_SERVER_URL = 'https://cloud.blender.org/'
# PILLAR_WEB_SERVER_URL = 'http://pillar-web:5001/'
@@ -41,39 +42,6 @@ log = logging.getLogger(__name__)
icons = None
def redraw(self, context):
context.area.tag_redraw()
def pyside_cache(propname):
if callable(propname):
raise TypeError('Usage: pyside_cache("property_name")')
def decorator(wrapped):
"""Stores the result of the callable in Python-managed memory.
This is to work around the warning at
https://www.blender.org/api/blender_python_api_master/bpy.props.html#bpy.props.EnumProperty
"""
import functools
@functools.wraps(wrapped)
# We can't use (*args, **kwargs), because EnumProperty explicitly checks
# for the number of fixed positional arguments.
def wrapper(self, context):
result = None
try:
result = wrapped(self, context)
return result
finally:
rna_type, rna_info = getattr(self.bl_rna, propname)
rna_info['_cached_result'] = result
return wrapper
return decorator
@pyside_cache('version')
def blender_syncable_versions(self, context):
"""Returns the list of items used by SyncStatusProperties.version EnumProperty."""
@@ -207,6 +175,16 @@ class BlenderCloudPreferences(AddonPreferences):
subtype='DIR_PATH',
default='//../')
flamenco_manager = PointerProperty(type=flamenco.FlamencoManagerGroup)
# TODO: before making Flamenco public, change the defaults to something less Institute-specific.
# NOTE: The assumption is that the workers can also find the files in the same path.
# This assumption is true for the Blender Institute, but we should allow other setups too.
flamenco_job_file_path = StringProperty(
name='Job file path',
description='Path where to store job files',
subtype='DIR_PATH',
default='/render/_flamenco/storage')
def draw(self, context):
import textwrap
@@ -291,6 +269,10 @@ class BlenderCloudPreferences(AddonPreferences):
attract_box = layout.box()
self.draw_attract_buttons(attract_box, self.attract_project)
# Flamenco stuff
flamenco_box = layout.box()
self.draw_flamenco_buttons(flamenco_box, self.flamenco_manager)
def draw_subscribe_button(self, layout):
layout.operator('pillar.subscribe', icon='WORLD')
@@ -349,6 +331,35 @@ class BlenderCloudPreferences(AddonPreferences):
attract_box.prop(self, 'attract_project_local_path')
def draw_flamenco_buttons(self, flamenco_box, bcp: flamenco.FlamencoManagerGroup):
flamenco_row = flamenco_box.row(align=True)
flamenco_row.label('Flamenco', icon_value=icon('CLOUD'))
flamenco_row.enabled = bcp.status in {'NONE', 'IDLE'}
row_buttons = flamenco_row.row(align=True)
if bcp.status in {'NONE', 'IDLE'}:
if not bcp.available_managers or not bcp.manager:
row_buttons.operator('flamenco.managers',
text='Find Flamenco Managers',
icon='FILE_REFRESH')
else:
row_buttons.prop(bcp, 'manager', text='Manager')
row_buttons.operator('flamenco.managers',
text='',
icon='FILE_REFRESH')
else:
row_buttons.label('Fetching available managers.')
# TODO: make a reusable way to select projects, and use that for Attract and Flamenco.
note_box = flamenco_box.column(align=True)
note_box.label('NOTE: For now, Flamenco uses the same project as Attract.')
note_box.label('This will change in a future version of the add-on.')
flamenco_box.prop(self, 'flamenco_job_file_path')
note_box = flamenco_box.column(align=True)
note_box.label('NOTE: Flamenco assumes the workers can use this path too.')
class PillarCredentialsUpdate(pillar.PillarOperatorMixin,
Operator):
@@ -414,7 +425,7 @@ class PILLAR_OT_subscribe(Operator):
class PILLAR_OT_projects(async_loop.AsyncModalOperatorMixin,
pillar.PillarOperatorMixin,
pillar.AuthenticatedPillarOperatorMixin,
Operator):
"""Fetches the projects available to the user"""
bl_idname = 'pillar.projects'
@@ -424,27 +435,20 @@ class PILLAR_OT_projects(async_loop.AsyncModalOperatorMixin,
_log = logging.getLogger('bpy.ops.%s' % bl_idname)
async def async_execute(self, context):
if not await self.authenticate(context):
return
import pillarsdk
from .pillar import pillar_call
self._log.info('Checking credentials')
try:
db_user = await self.check_credentials(context, ())
except pillar.UserNotLoggedInError as ex:
self._log.info('Not logged in error raised: %s', ex)
self.report({'ERROR'}, 'Please log in on Blender ID first.')
self.quit()
return
user_id = db_user['_id']
self.log.info('Going to fetch projects for user %s', user_id)
self.log.info('Going to fetch projects for user %s', self.user_id)
preferences().attract_project.status = 'FETCHING'
# Get all projects, except the home project.
projects_user = await pillar_call(
pillarsdk.Project.all,
{'where': {'user': user_id,
{'where': {'user': self.user_id,
'category': {'$ne': 'home'}},
'sort': '-_created',
'projection': {'_id': True,
@@ -453,8 +457,8 @@ class PILLAR_OT_projects(async_loop.AsyncModalOperatorMixin,
projects_shared = await pillar_call(
pillarsdk.Project.all,
{'where': {'user': {'$ne': user_id},
'permissions.groups.group': {'$in': db_user.groups}},
{'where': {'user': {'$ne': self.user_id},
'permissions.groups.group': {'$in': self.db_user.groups}},
'sort': '-_created',
'projection': {'_id': True,
'name': True},

242
blender_cloud/flamenco/__init__.py Normal file
View File

@@ -0,0 +1,242 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
"""Flamenco interface.
The preferences are managed blender.py, the rest of the Flamenco-specific stuff is here.
"""
import logging
import bpy
from bpy.types import AddonPreferences, Operator, WindowManager, Scene, PropertyGroup
from bpy.props import StringProperty, EnumProperty, PointerProperty, BoolProperty, IntProperty
from .. import async_loop, pillar
from ..utils import pyside_cache, redraw
log = logging.getLogger(__name__)
@pyside_cache('manager')
def available_managers(self, context):
"""Returns the list of items used by a manager-selector EnumProperty."""
from ..blender import preferences
mngrs = preferences().flamenco_manager.available_managers
if not mngrs:
return [('', 'No managers available in your Blender Cloud', '')]
return [(p['_id'], p['name'], '') for p in mngrs]
class FlamencoManagerGroup(PropertyGroup):
manager = EnumProperty(
items=available_managers,
name='Flamenco Manager',
description='Which Flamenco Manager to use for jobs')
status = EnumProperty(
items=[
('NONE', 'NONE', 'We have done nothing at all yet'),
('IDLE', 'IDLE', 'User requested something, which is done, and we are now idle'),
('FETCHING', 'FETCHING', 'Fetching available Flamenco managers from Blender Cloud'),
],
name='status',
update=redraw)
# List of managers is stored in 'available_managers' ID property,
# because I don't know how to store a variable list of strings in a proper RNA property.
@property
def available_managers(self) -> list:
return self.get('available_managers', [])
@available_managers.setter
def available_managers(self, new_managers):
self['available_managers'] = new_managers
class FLAMENCO_OT_fmanagers(async_loop.AsyncModalOperatorMixin,
pillar.AuthenticatedPillarOperatorMixin,
Operator):
"""Fetches the Flamenco Managers available to the user"""
bl_idname = 'flamenco.managers'
bl_label = 'Fetch available Flamenco Managers'
stop_upon_exception = True
_log = logging.getLogger('bpy.ops.%s' % bl_idname)
@property
def mypref(self) -> FlamencoManagerGroup:
from ..blender import preferences
return preferences().flamenco_manager
async def async_execute(self, context):
if not await self.authenticate(context):
return
from .sdk import Manager
from ..pillar import pillar_call
self.log.info('Going to fetch managers for user %s', self.user_id)
self.mypref.status = 'FETCHING'
managers = await pillar_call(Manager.all)
# We need to convert to regular dicts before storing in ID properties.
# Also don't store more properties than we need.
as_list = [{'_id': p['_id'], 'name': p['name']} for p in managers['_items']]
self.mypref.available_managers = as_list
self.quit()
def quit(self):
self.mypref.status = 'IDLE'
super().quit()
class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
pillar.AuthenticatedPillarOperatorMixin,
Operator):
"""Performs a Blender render on Flamenco."""
bl_idname = 'flamenco.render'
bl_label = 'Render on Flamenco'
stop_upon_exception = True
_log = logging.getLogger('bpy.ops.%s' % bl_idname)
async def async_execute(self, context):
if not await self.authenticate(context):
return
import os.path
from ..blender import preferences
from pillarsdk import exceptions as sdk_exceptions
prefs = preferences()
scene = context.scene
try:
await create_job(self.user_id,
prefs.attract_project.project,
prefs.flamenco_manager.manager,
'blender-render',
{
"blender_cmd": "{blender}",
"chunk_size": scene.flamenco_render_chunk_size,
"filepath": context.blend_data.filepath,
"frames": scene.flamenco_render_frame_range
},
'Render %s' % os.path.basename(context.blend_data.filepath))
except sdk_exceptions.ResourceInvalid as ex:
self.report({'ERROR'}, 'Error creating Flamenco job: %s' % ex)
else:
self.report({'INFO'}, 'Flamenco job created.')
self.quit()
class FLAMENCO_OT_scene_to_frame_range(Operator):
"""Sets the scene frame range as the Flamenco render frame range."""
bl_idname = 'flamenco.scene_to_frame_range'
bl_label = 'Sets the scene frame range as the Flamenco render frame range'
def execute(self, context):
s = context.scene
s.flamenco_render_frame_range = '%i-%i' % (s.frame_start, s.frame_end)
return {'FINISHED'}
async def create_job(user_id: str,
project_id: str,
manager_id: str,
job_type: str,
job_settings: dict,
job_name: str = None,
*,
job_description: str = None) -> str:
"""Creates a render job at Flamenco Server, returning the job ID."""
import json
from .sdk import Job
from ..pillar import pillar_call
job_attrs = {
'status': 'queued',
'priority': 50,
'name': job_name,
'settings': job_settings,
'job_type': job_type,
'user': user_id,
'manager': manager_id,
'project': project_id,
}
if job_description:
job_attrs['description'] = job_description
log.info('Going to create Flamenco job:\n%s',
json.dumps(job_attrs, indent=4, sort_keys=True))
job = Job(job_attrs)
await pillar_call(job.create)
log.info('Job created succesfully: %s', job._id)
return job._id
def draw_render_button(self, context):
layout = self.layout
from ..blender import icon
flamenco_box = layout.box()
flamenco_box.label('Flamenco', icon_value=icon('CLOUD'))
flamenco_box.prop(context.scene, 'flamenco_render_chunk_size')
frange_row = flamenco_box.row(align=True)
frange_row.prop(context.scene, 'flamenco_render_frame_range')
frange_row.operator('flamenco.scene_to_frame_range', text='', icon='ARROW_LEFTRIGHT')
flamenco_box.operator('flamenco.render', text='Render on Flamenco', icon='RENDER_ANIMATION')
def register():
bpy.utils.register_class(FlamencoManagerGroup)
bpy.utils.register_class(FLAMENCO_OT_fmanagers)
bpy.utils.register_class(FLAMENCO_OT_render)
bpy.utils.register_class(FLAMENCO_OT_scene_to_frame_range)
scene = bpy.types.Scene
scene.flamenco_render_chunk_size = IntProperty(
name='Chunk size',
description='Maximum number of frames to render per task',
default=10,
)
scene.flamenco_render_frame_range = StringProperty(
name='Frame range',
description='Frames to render, in "printer range" notation'
)
bpy.types.RENDER_PT_render.append(draw_render_button)
def unregister():
bpy.types.RENDER_PT_render.remove(draw_render_button)
bpy.utils.unregister_module(__name__)
wm = bpy.types.WindowManager
del wm.flamenco_render_chunk_size

13
blender_cloud/flamenco/sdk.py Normal file
View File

@@ -0,0 +1,13 @@
from pillarsdk.resource import List, Find, Create
class Manager(List, Find):
"""Manager class wrapping the REST nodes endpoint"""
path = 'flamenco/managers'
class Job(List, Find, Create):
"""Job class wrapping the REST nodes endpoint
"""
path = 'flamenco/jobs'
ensure_query_projections = {'project': 1}

26
blender_cloud/pillar.py
View File

@@ -840,6 +840,32 @@ class PillarOperatorMixin:
'Please subscribe to the blender cloud at https://cloud.blender.org/join')
class AuthenticatedPillarOperatorMixin(PillarOperatorMixin):
"""Checks credentials, to be used at the start of async_execute().
Sets self.user_id to the current user's ID, and self.db_user to the user info dict,
if authentication was succesful; sets both to None if not.
"""
async def authenticate(self, context) -> bool:
from . import pillar
self._log.info('Checking credentials')
self.user_id = None
self.db_user = None
try:
self.db_user = await self.check_credentials(context, ())
except pillar.UserNotLoggedInError as ex:
self._log.info('Not logged in error raised: %s', ex)
self.report({'ERROR'}, 'Please log in on Blender ID first.')
self.quit()
return False
self.user_id = self.db_user['_id']
return True
async def find_or_create_node(where: dict,
additional_create_props: dict = None,
projection: dict = None,

38
blender_cloud/utils.py
View File

@@ -62,3 +62,41 @@ def find_in_path(path: pathlib.Path, filename: str) -> pathlib.Path:
return subpath
return None
def pyside_cache(propname):
"""Decorator, stores the result of the decorated callable in Python-managed memory.
This is to work around the warning at
https://www.blender.org/api/blender_python_api_master/bpy.props.html#bpy.props.EnumProperty
"""
if callable(propname):
raise TypeError('Usage: pyside_cache("property_name")')
def decorator(wrapped):
"""Stores the result of the callable in Python-managed memory.
This is to work around the warning at
https://www.blender.org/api/blender_python_api_master/bpy.props.html#bpy.props.EnumProperty
"""
import functools
@functools.wraps(wrapped)
# We can't use (*args, **kwargs), because EnumProperty explicitly checks
# for the number of fixed positional arguments.
def wrapper(self, context):
result = None
try:
result = wrapped(self, context)
return result
finally:
rna_type, rna_info = getattr(self.bl_rna, propname)
rna_info['_cached_result'] = result
return wrapper
return decorator
def redraw(self, context):
context.area.tag_redraw()