diff --git a/blender_cloud/blender.py b/blender_cloud/blender.py index 8458225..22639f6 100644 --- a/blender_cloud/blender.py +++ b/blender_cloud/blender.py @@ -181,7 +181,7 @@ class BlenderCloudPreferences(AddonPreferences): # This assumption is true for the Blender Institute, but we should allow other setups too. flamenco_job_file_path = StringProperty( name='Job file path', - description='Path where to store job files', + description='Path where to store job files, should be accesible for Workers too', subtype='DIR_PATH', default='/render/_flamenco/storage') @@ -351,14 +351,15 @@ class BlenderCloudPreferences(AddonPreferences): else: row_buttons.label('Fetching available managers.') + path_box = flamenco_box.row(align=True) + path_box.prop(self, 'flamenco_job_file_path') + path_box.operator('flamenco.open_job_file_path', text='', icon='DISK_DRIVE') + # TODO: make a reusable way to select projects, and use that for Attract and Flamenco. note_box = flamenco_box.column(align=True) note_box.label('NOTE: For now, Flamenco uses the same project as Attract.') note_box.label('This will change in a future version of the add-on.') - flamenco_box.prop(self, 'flamenco_job_file_path') - note_box = flamenco_box.column(align=True) - note_box.label('NOTE: Flamenco assumes the workers can use this path too.') class PillarCredentialsUpdate(pillar.PillarOperatorMixin, diff --git a/blender_cloud/flamenco/__init__.py b/blender_cloud/flamenco/__init__.py index ae32d43..1275f1f 100644 --- a/blender_cloud/flamenco/__init__.py +++ b/blender_cloud/flamenco/__init__.py @@ -20,7 +20,10 @@ The preferences are managed blender.py, the rest of the Flamenco-specific stuff is here. """ + import logging +from pathlib import Path +import typing import bpy from bpy.types import AddonPreferences, Operator, WindowManager, Scene, PropertyGroup @@ -116,6 +119,7 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin, """Performs a Blender render on Flamenco.""" bl_idname = 'flamenco.render' bl_label = 'Render on Flamenco' + bl_description = __doc__.rstrip('.') stop_upon_exception = True _log = logging.getLogger('bpy.ops.%s' % bl_idname) @@ -124,36 +128,100 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin, if not await self.authenticate(context): return - import os.path - from ..blender import preferences from pillarsdk import exceptions as sdk_exceptions + from ..blender import preferences + filepath = Path(context.blend_data.filepath) + + # BAM-pack the files to the destination directory. + outfile, missing_sources = await self.bam_pack(filepath) + if not outfile: + return + + # Create the job at Flamenco Server. prefs = preferences() scene = context.scene - + settings = {"blender_cmd": "{blender}", + "chunk_size": scene.flamenco_render_chunk_size, + "filepath": str(outfile), + "frames": scene.flamenco_render_frame_range} try: - await create_job(self.user_id, - prefs.attract_project.project, - prefs.flamenco_manager.manager, - 'blender-render', - { - "blender_cmd": "{blender}", - "chunk_size": scene.flamenco_render_chunk_size, - "filepath": context.blend_data.filepath, - "frames": scene.flamenco_render_frame_range - }, - 'Render %s' % os.path.basename(context.blend_data.filepath)) + job_info = await create_job(self.user_id, + prefs.attract_project.project, + prefs.flamenco_manager.manager, + 'blender-render', + settings, + 'Render %s' % filepath.name) except sdk_exceptions.ResourceInvalid as ex: self.report({'ERROR'}, 'Error creating Flamenco job: %s' % ex) + self.quit() + return + + # Store the job ID in a file in the output dir. + with open(str(outfile.parent / 'jobinfo.json'), 'w', encoding='utf8') as outfile: + import json + + job_info['missing_files'] = [str(mf) for mf in missing_sources] + json.dump(job_info, outfile, sort_keys=True, indent=4) + + # Do a final report. + if missing_sources: + names = (ms.name for ms in missing_sources) + self.report({'WARNING'}, 'Flamenco job created with missing files: %s' % + '; '.join(names)) else: self.report({'INFO'}, 'Flamenco job created.') self.quit() + async def bam_pack(self, filepath: Path) -> (typing.Optional[Path], typing.List[Path]): + """BAM-packs the blendfile to the destination directory. + + Returns the path of the destination blend file. + + :param filepath: the blend file to pack (i.e. the current blend file) + :returns: the destination blend file, or None if there were errors BAM-packing, + and a list of missing paths. + """ + + from datetime import datetime + from ..blender import preferences + from . import bam_interface + + prefs = preferences() + + # Create a unique directory that is still more or less identifyable. + # This should work better than a random ID. + # BAM doesn't like output directories that end in '.blend'. + unique_dir = '%s-%s-%s' % (datetime.now().isoformat('-').replace(':', ''), + self.db_user['username'], + filepath.name.replace('.blend', '')) + outdir = Path(prefs.flamenco_job_file_path) / unique_dir + outfile = outdir / filepath.name + + try: + outdir.mkdir(parents=True) + except Exception as ex: + self.log.exception('Unable to create output path %s', outdir) + self.report({'ERROR'}, 'Unable to create output path: %s' % ex) + self.quit() + return None, [] + + try: + missing_sources = await bam_interface.bam_copy(filepath, outfile) + except bam_interface.CommandExecutionError as ex: + self.log.exception('Unable to execute BAM pack') + self.report({'ERROR'}, 'Unable to execute BAM pack: %s' % ex) + self.quit() + return None, [] + + return outfile, missing_sources + class FLAMENCO_OT_scene_to_frame_range(Operator): """Sets the scene frame range as the Flamenco render frame range.""" bl_idname = 'flamenco.scene_to_frame_range' bl_label = 'Sets the scene frame range as the Flamenco render frame range' + bl_description = __doc__.rstrip('.') def execute(self, context): s = context.scene @@ -161,6 +229,56 @@ class FLAMENCO_OT_scene_to_frame_range(Operator): return {'FINISHED'} +class FLAMENCO_OT_copy_files(Operator, + async_loop.AsyncModalOperatorMixin): + """Uses BAM to copy the current blendfile + dependencies to the target directory.""" + bl_idname = 'flamenco.copy_files' + bl_label = 'Copy files to target' + bl_description = __doc__.rstrip('.') + + stop_upon_exception = True + + async def async_execute(self, context): + from pathlib import Path + from . import bam_interface + from ..blender import preferences + + missing_sources = await bam_interface.bam_copy( + Path(context.blend_data.filepath), + Path(preferences().flamenco_job_file_path), + ) + + if missing_sources: + names = (ms.name for ms in missing_sources) + self.report({'ERROR'}, 'Missing source files: %s' % '; '.join(names)) + + self.quit() + + +class FLAMENCO_OT_open_job_file_path(Operator): + """Opens the Flamenco job storage path in a file explorer.""" + bl_idname = 'flamenco.open_job_file_path' + bl_label = 'Open in file explorer' + bl_description = __doc__.rstrip('.') + + def execute(self, context): + import platform + import subprocess + import os + + from ..blender import preferences + + path = preferences().flamenco_job_file_path + if platform.system() == "Windows": + os.startfile(path) + elif platform.system() == "Darwin": + subprocess.Popen(["open", path]) + else: + subprocess.Popen(["xdg-open", path]) + + return {'FINISHED'} + + async def create_job(user_id: str, project_id: str, manager_id: str, @@ -168,8 +286,8 @@ async def create_job(user_id: str, job_settings: dict, job_name: str = None, *, - job_description: str = None) -> str: - """Creates a render job at Flamenco Server, returning the job ID.""" + job_description: str = None) -> dict: + """Creates a render job at Flamenco Server, returning the job object as dictionary.""" import json from .sdk import Job @@ -195,23 +313,38 @@ async def create_job(user_id: str, await pillar_call(job.create) log.info('Job created succesfully: %s', job._id) - return job._id + return job.to_dict() -def draw_render_button(self, context): - layout = self.layout +class FLAMENCO_PT_render(bpy.types.Panel): + bl_label = "Flamenco Render" + bl_space_type = 'PROPERTIES' + bl_region_type = 'WINDOW' + bl_context = "render" + bl_options = {'DEFAULT_CLOSED'} - from ..blender import icon + def draw(self, context): + layout = self.layout - flamenco_box = layout.box() - flamenco_box.label('Flamenco', icon_value=icon('CLOUD')) - flamenco_box.prop(context.scene, 'flamenco_render_chunk_size') + from ..blender import preferences - frange_row = flamenco_box.row(align=True) - frange_row.prop(context.scene, 'flamenco_render_frame_range') - frange_row.operator('flamenco.scene_to_frame_range', text='', icon='ARROW_LEFTRIGHT') + layout.prop(context.scene, 'flamenco_render_chunk_size') - flamenco_box.operator('flamenco.render', text='Render on Flamenco', icon='RENDER_ANIMATION') + labeled_row = layout.split(0.2, align=True) + labeled_row.label('Frame range:') + prop_btn_row = labeled_row.row(align=True) + prop_btn_row.prop(context.scene, 'flamenco_render_frame_range', text='') + prop_btn_row.operator('flamenco.scene_to_frame_range', text='', icon='ARROW_LEFTRIGHT') + + labeled_row = layout.split(0.2, align=True) + labeled_row.label('Storage:') + prop_btn_row = labeled_row.row(align=True) + prop_btn_row.label(preferences().flamenco_job_file_path) + prop_btn_row.operator(FLAMENCO_OT_open_job_file_path.bl_idname, text='', icon='DISK_DRIVE') + + layout.operator(FLAMENCO_OT_render.bl_idname, + text='Render on Flamenco', + icon='RENDER_ANIMATION') def register(): @@ -219,6 +352,9 @@ def register(): bpy.utils.register_class(FLAMENCO_OT_fmanagers) bpy.utils.register_class(FLAMENCO_OT_render) bpy.utils.register_class(FLAMENCO_OT_scene_to_frame_range) + bpy.utils.register_class(FLAMENCO_OT_copy_files) + bpy.utils.register_class(FLAMENCO_OT_open_job_file_path) + bpy.utils.register_class(FLAMENCO_PT_render) scene = bpy.types.Scene scene.flamenco_render_chunk_size = IntProperty( @@ -231,11 +367,7 @@ def register(): description='Frames to render, in "printer range" notation' ) - bpy.types.RENDER_PT_render.append(draw_render_button) - - def unregister(): - bpy.types.RENDER_PT_render.remove(draw_render_button) bpy.utils.unregister_module(__name__) try: diff --git a/blender_cloud/flamenco/bam_interface.py b/blender_cloud/flamenco/bam_interface.py new file mode 100644 index 0000000..894b631 --- /dev/null +++ b/blender_cloud/flamenco/bam_interface.py @@ -0,0 +1,144 @@ +"""BAM packing interface for Flamenco.""" + +import logging +from pathlib import Path +import typing + +# Timeout of the BAM subprocess, in seconds. +SUBPROC_READLINE_TIMEOUT = 600 +log = logging.getLogger(__name__) + + +class CommandExecutionError(Exception): + """Raised when there was an error executing a BAM command.""" + pass + + +async def bam_copy(base_blendfile: Path, target_blendfile: Path) -> typing.List[Path]: + """Uses BAM to copy the given file and dependencies to the target blendfile. + + Due to the way blendfile_pack.py is programmed/structured, we cannot import it + and call a function; it has to be run in a subprocess. + + :raises: asyncio.CanceledError if the task was cancelled. + :raises: asyncio.TimeoutError if reading a line from the BAM process timed out. + :raises: CommandExecutionError if the subprocess failed or output invalid UTF-8. + :returns: a list of missing sources; hopefully empty. + """ + + import asyncio + import shlex + import subprocess + + import bpy + import io_blend_utils + + args = [ + bpy.app.binary_path_python, + '-m', 'bam.pack', + '--input', str(base_blendfile), + '--output', str(target_blendfile), + '--mode', 'FILE', + ] + + cmd_to_log = ' '.join(shlex.quote(s) for s in args) + log.info('Executing %s', cmd_to_log) + + proc = await asyncio.create_subprocess_exec( + *args, + env={'PYTHONPATH': io_blend_utils.pythonpath()}, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + missing_sources = [] + + try: + while not proc.stdout.at_eof(): + line = await asyncio.wait_for(proc.stdout.readline(), + SUBPROC_READLINE_TIMEOUT) + if not line: + # EOF received, so let's bail. + break + + try: + line = line.decode('utf8') + except UnicodeDecodeError as ex: + raise CommandExecutionError('Command produced non-UTF8 output, ' + 'aborting: %s' % ex) + + line = line.rstrip() + if 'source missing:' in line: + path = parse_missing_source(line) + missing_sources.append(path) + log.warning('Source is missing: %s', path) + + log.info(' %s', line) + finally: + if proc.returncode is None: + # Always wait for the process, to avoid zombies. + try: + proc.kill() + except ProcessLookupError: + # The process is already stopped, so killing is impossible. That's ok. + log.debug("The process was already stopped, aborting is impossible. That's ok.") + await proc.wait() + log.info('The process stopped with status code %i', proc.returncode) + + if proc.returncode: + raise CommandExecutionError('Process stopped with status %i' % proc.returncode) + + return missing_sources + + +def parse_missing_source(line: str) -> Path: + r"""Parses a "missing source" line into a pathlib.Path. + + >>> parse_missing_source(r" source missing: b'D\xc3\xaffficult \xc3\x9cTF-8 filename'") + PosixPath('Dïfficult ÜTF-8 filename') + >>> parse_missing_source(r" source missing: b'D\xfffficult Win1252 f\xeflen\xe6me'") + PosixPath('D�fficult Win1252 f�len�me') + """ + + _, missing_source = line.split(': ', 1) + missing_source_as_bytes = parse_byte_literal(missing_source.strip()) + + # The file could originate from any platform, so UTF-8 and the current platform's + # filesystem encodings are just guesses. + try: + missing_source = missing_source_as_bytes.decode('utf8') + except UnicodeDecodeError: + import sys + try: + missing_source = missing_source_as_bytes.decode(sys.getfilesystemencoding()) + except UnicodeDecodeError: + missing_source = missing_source_as_bytes.decode('ascii', errors='replace') + + path = Path(missing_source) + + return path + + +def parse_byte_literal(bytes_literal: str) -> bytes: + r"""Parses a repr(bytes) output into a bytes object. + + >>> parse_byte_literal(r"b'D\xc3\xaffficult \xc3\x9cTF-8 filename'") + b'D\xc3\xaffficult \xc3\x9cTF-8 filename' + >>> parse_byte_literal(r"b'D\xeffficult Win1252 f\xeflen\xe6me'") + b'D\xeffficult Win1252 f\xeflen\xe6me' + """ + + # Some very basic assertions to make sure we have a proper bytes literal. + assert bytes_literal[0] == "b" + assert bytes_literal[1] in {'"', "'"} + assert bytes_literal[-1] == bytes_literal[1] + + import ast + return ast.literal_eval(bytes_literal) + + +if __name__ == '__main__': + import doctest + + doctest.testmod()