archive/blender-cloud-addon
Archived
1
3
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Create job first, then send files

Browse Source 
This requires Flamenco Server 2.2 or newer.
This commit is contained in:
Sybren A. Stüvel 2019-02-28 12:52:51 +01:00
parent 26105add9c
commit cc97288018
3 changed files with 108 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
blender_cloud/flamenco/__init__.py
View File

@ -323,15 +323,8 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
self.quit()
return
# BAT-pack the files to the destination directory.
outdir, outfile, missing_sources = await self.bat_pack(filepath)
if not outfile:
return
settings['filepath'] = manager.replace_path(outfile)
# Create the job at Flamenco Server.
context.window_manager.flamenco_status = 'COMMUNICATING'
project_id = prefs.project.project
job_name = self._make_job_name(filepath)
try:
@ -356,7 +349,62 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
self.quit()
return
# Store the job ID in a file in the output dir.
# BAT-pack the files to the destination directory.
job_id = job_info['_id']
outdir, outfile, missing_sources = await self.bat_pack(job_id, filepath)
if not outfile:
return
# Store the job ID in a file in the output dir, if we can.
# TODO: Make it possible to create this file first and then send it to BAT for packing.
if outdir is not None:
await self._create_jobinfo_json(
outdir, job_info, manager_id, project_id, missing_sources)
# Now that the files have been transfered, PATCH the job at the Manager
# to kick off the job compilation.
job_filepath = manager.replace_path(outfile)
self.log.info('Final file path: %s', job_filepath)
new_settings = {'filepath': job_filepath}
await self.compile_job(job_id, new_settings)
# We can now remove the local copy we made with bpy.ops.wm.save_as_mainfile().
# Strictly speaking we can already remove it after the BAT-pack, but it may come in
# handy in case of failures.
try:
self.log.info('Removing temporary file %s', filepath)
filepath.unlink()
except Exception as ex:
self.report({'ERROR'}, 'Unable to remove file: %s' % ex)
self.quit()
return
if prefs.flamenco_open_browser_after_submit:
import webbrowser
from urllib.parse import urljoin
from ..blender import PILLAR_WEB_SERVER_URL
url = urljoin(PILLAR_WEB_SERVER_URL, '/flamenco/jobs/%s/redir' % job_id)
webbrowser.open_new_tab(url)
# Do a final report.
if missing_sources:
names = (ms.name for ms in missing_sources)
self.report({'WARNING'}, 'Flamenco job created with missing files: %s' %
'; '.join(names))
else:
self.report({'INFO'}, 'Flamenco job created.')
if self.quit_after_submit:
silently_quit_blender()
self.quit()
async def _create_jobinfo_json(self, outdir: Path, job_info: dict,
manager_id: str, project_id: str,
missing_sources: typing.List[Path]):
from ..blender import preferences
prefs = preferences()
with open(str(outdir / 'jobinfo.json'), 'w', encoding='utf8') as outfile:
import json
@ -384,38 +432,6 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
}
json.dump(info, outfile, sort_keys=True, indent=4, cls=utils.JSONEncoder)
# We can now remove the local copy we made with bpy.ops.wm.save_as_mainfile().
# Strictly speaking we can already remove it after the BAT-pack, but it may come in
# handy in case of failures.
try:
self.log.info('Removing temporary file %s', filepath)
filepath.unlink()
except Exception as ex:
self.report({'ERROR'}, 'Unable to remove file: %s' % ex)
self.quit()
return
if prefs.flamenco_open_browser_after_submit:
import webbrowser
from urllib.parse import urljoin
from ..blender import PILLAR_WEB_SERVER_URL
url = urljoin(PILLAR_WEB_SERVER_URL, '/flamenco/jobs/%s/redir' % job_info['_id'])
webbrowser.open_new_tab(url)
# Do a final report.
if missing_sources:
names = (ms.name for ms in missing_sources)
self.report({'WARNING'}, 'Flamenco job created with missing files: %s' %
'; '.join(names))
else:
self.report({'INFO'}, 'Flamenco job created.')
if self.quit_after_submit:
silently_quit_blender()
self.quit()
def _make_job_name(self, filepath: Path) -> str:
"""Turn a file to render into the render job name."""
@ -505,15 +521,20 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
return filepath
async def bat_pack(self, filepath: Path) \
-> typing.Tuple[Path, typing.Optional[Path], typing.List[Path]]:
async def bat_pack(self, job_id: str, filepath: Path) \
-> typing.Tuple[typing.Optional[Path], typing.Optional[PurePath], typing.List[Path]]:
"""BAT-packs the blendfile to the destination directory.
Returns the path of the destination blend file.
:param job_id: the job ID given to us by Flamenco Server.
:param filepath: the blend file to pack (i.e. the current blend file)
:returns: the destination directory, the destination blend file or None
if there were errors BAT-packing, and a list of missing paths.
:returns: A tuple of:
- The destination directory, or None if it does not exist on a
locally-reachable filesystem (for example when sending files to
a Shaman server).
- The destination blend file, or None if there were errors BAT-packing,
- A list of missing paths.
"""
from datetime import datetime
@ -521,19 +542,20 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
prefs = preferences()
proj_abspath = bpy.path.abspath(prefs.cloud_project_local_path)
projdir = Path(proj_abspath).resolve()
exclusion_filter = (prefs.flamenco_exclude_filter or '').strip()
relative_only = prefs.flamenco_relative_only
self.log.debug('projdir: %s', projdir)
# Create a unique directory that is still more or less identifyable.
# This should work better than a random ID.
unique_dir = '%s-%s-%s' % (datetime.now().isoformat('-').replace(':', ''),
self.db_user['username'],
filepath.stem)
outdir = Path(prefs.flamenco_job_file_path) / unique_dir
proj_abspath = bpy.path.abspath(prefs.cloud_project_local_path)
projdir = Path(proj_abspath).resolve()
exclusion_filter = (prefs.flamenco_exclude_filter or '').strip()
relative_only = prefs.flamenco_relative_only
self.log.debug('outdir : %s', outdir)
self.log.debug('projdir: %s', projdir)
try:
outdir.mkdir(parents=True)
@ -562,6 +584,20 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
bpy.context.window_manager.flamenco_status = 'DONE'
return outdir, outfile, missing_sources
async def compile_job(self, job_id: str, new_settings: dict) -> None:
"""Request Flamenco Server to start compiling the job."""
payload = {
'op': 'construct',
'settings': new_settings,
}
from .sdk import Job
from ..pillar import pillar_call
job = Job({'_id': job_id})
await pillar_call(job.patch, payload, caching=False)
def scene_frame_range(context) -> str:
"""Returns the frame range string for the current scene."""
@ -744,7 +780,7 @@ async def create_job(user_id: str,
from ..pillar import pillar_call
job_attrs = {
'status': 'queued',
'status': 'waiting-for-files',
'priority': priority,
'name': job_name,
'settings': job_settings,

13
blender_cloud/flamenco/bat_interface.py
View File

@ -96,10 +96,13 @@ class BatProgress(progress.Callback):
async def copy(context,
base_blendfile: pathlib.Path,
project: pathlib.Path,
target: pathlib.Path,
target: str,
exclusion_filter: str,
*,
relative_only: bool) -> typing.Tuple[pathlib.Path, typing.Set[pathlib.Path]]:
relative_only: bool,
packer_class=pack.Packer,
**packer_args) \
-> typing.Tuple[pathlib.Path, typing.Set[pathlib.Path]]:
"""Use BAT🦇 to copy the given file and dependencies to the target location.
:raises: FileTransferError if a file couldn't be transferred.
@ -108,11 +111,11 @@ async def copy(context,
global _running_packer
loop = asyncio.get_event_loop()
wm = bpy.context.window_manager
with pack.Packer(base_blendfile, project, target, compress=True, relative_only=relative_only) \
as packer:
packer = packer_class(base_blendfile, project, target,
compress=True, relative_only=relative_only, **packer_args)
with packer:
with _packer_lock:
if exclusion_filter:
# There was a mistake in an older version of the property tooltip,

14
blender_cloud/flamenco/sdk.py
View File

@ -32,6 +32,8 @@ class Manager(List, Find):
Tries to find platform-specific path prefixes, and replaces them with
variables.
"""
assert isinstance(some_path, pathlib.PurePath), \
'some_path should be a PurePath, not %r' % some_path
for varname, path in self._sorted_path_replacements():
replacement = self.PurePlatformPath(path)
@ -52,3 +54,15 @@ class Job(List, Find, Create):
"""
path = 'flamenco/jobs'
ensure_query_projections = {'project': 1}
def patch(self, payload: dict, api=None):
import pillarsdk.utils
import json
api = api or self.api
url = pillarsdk.utils.join_url(self.path, str(self['_id']))
headers = pillarsdk.utils.merge_dict(self.http_headers(),
{'Content-Type': 'application/json'})
response = api.patch(url, payload, headers=headers)
return response