archive/blender-cloud-addon
Archived
1
3
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Replaced BAM with BAT

Browse Source 
Blender Asset Tracer, or BAT, is a newly written replacement for BAM,
with a nicer API.
This commit is contained in:
Sybren A. Stüvel
2018-03-15 12:36:05 +01:00
parent 531ddad8f5
commit 9e5dcd0b55
7 changed files with 64 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
blender_cloud/blender.py
View File

@@ -406,10 +406,10 @@ class BlenderCloudPreferences(AddonPreferences):
# This is only needed when the project is set up for either Attract or Flamenco.
project_box.prop(self, 'cloud_project_local_path',
text='Local Cloud Project Path')
text='Local Project Path')
def draw_flamenco_buttons(self, flamenco_box, bcp: flamenco.FlamencoManagerGroup, context):
from .flamenco import bam_interface
from .flamenco import bat_interface
header_row = flamenco_box.row(align=True)
header_row.label('Flamenco:', icon_value=icon('CLOUD'))

46
blender_cloud/flamenco/__init__.py
View File

@@ -31,12 +31,12 @@ if "bpy" in locals():
import importlib
try:
bam_interface = importlib.reload(bam_interface)
bat_interface = importlib.reload(bat_interface)
sdk = importlib.reload(sdk)
except NameError:
from . import bam_interface, sdk
from . import bat_interface, sdk
else:
from . import bam_interface, sdk
from . import bat_interface, sdk
import bpy
from bpy.types import AddonPreferences, Operator, WindowManager, Scene, PropertyGroup
@@ -181,8 +181,8 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
return
self.log.info('Will output render files to %s', render_output)
# BAM-pack the files to the destination directory.
outfile, missing_sources = await self.bam_pack(filepath)
# BAT-pack the files to the destination directory.
outfile, missing_sources = await self.bat_pack(filepath)
if not outfile:
return
@@ -245,7 +245,7 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
json.dump(job_info, outfile, sort_keys=True, indent=4)
# We can now remove the local copy we made with bpy.ops.wm.save_as_mainfile().
# Strictly speaking we can already remove it after the BAM-pack, but it may come in
# Strictly speaking we can already remove it after the BAT-pack, but it may come in
# handy in case of failures.
try:
self.log.info('Removing temporary file %s', filepath)
@@ -314,13 +314,13 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
super().quit()
bpy.context.window_manager.flamenco_status = 'IDLE'
async def bam_pack(self, filepath: Path) -> (typing.Optional[Path], typing.List[Path]):
"""BAM-packs the blendfile to the destination directory.
async def bat_pack(self, filepath: Path) -> (typing.Optional[Path], typing.List[Path]):
"""BAT-packs the blendfile to the destination directory.
Returns the path of the destination blend file.
:param filepath: the blend file to pack (i.e. the current blend file)
:returns: the destination blend file, or None if there were errors BAM-packing,
:returns: the destination blend file, or None if there were errors BAT-packing,
and a list of missing paths.
"""
@@ -331,14 +331,12 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
# Create a unique directory that is still more or less identifyable.
# This should work better than a random ID.
# BAM doesn't like output directories that end in '.blend'.
unique_dir = '%s-%s-%s' % (datetime.now().isoformat('-').replace(':', ''),
self.db_user['username'],
filepath.stem)
outdir = Path(prefs.flamenco_job_file_path) / unique_dir
outfile = outdir / filepath.name
exclusion_filter = prefs.flamenco_exclude_filter or None
projdir = Path(prefs.cloud_project_local_path)
exclusion_filter = (prefs.flamenco_exclude_filter or '').strip()
try:
outdir.mkdir(parents=True)
@@ -349,10 +347,12 @@ class FLAMENCO_OT_render(async_loop.AsyncModalOperatorMixin,
return None, []
try:
missing_sources = await bam_interface.bam_copy(filepath, outfile, exclusion_filter)
except bam_interface.CommandExecutionError as ex:
self.log.exception('Unable to execute BAM pack')
self.report({'ERROR'}, 'Unable to execute BAM pack: %s' % ex)
outfile, missing_sources = await bat_interface.bat_copy(
filepath, projdir, outdir, exclusion_filter)
except bat_interface.FileTransferError as ex:
self.log.error('Could not transfer %d files, starting with %s',
len(ex.files_remaining), ex.files_remaining[0])
self.report({'ERROR'}, 'Unable to transfer %d files' % len(ex.files_remaining))
self.quit()
return None, []
@@ -380,7 +380,7 @@ class FLAMENCO_OT_scene_to_frame_range(FlamencoPollMixin, Operator):
class FLAMENCO_OT_copy_files(Operator,
FlamencoPollMixin,
async_loop.AsyncModalOperatorMixin):
"""Uses BAM to copy the current blendfile + dependencies to the target directory."""
"""Uses BAT to copy the current blendfile + dependencies to the target directory."""
bl_idname = 'flamenco.copy_files'
bl_label = 'Copy files to target'
bl_description = __doc__.rstrip('.')
@@ -392,11 +392,13 @@ class FLAMENCO_OT_copy_files(Operator,
from ..blender import preferences
context.window_manager.flamenco_status = 'PACKING'
exclusion_filter = preferences().flamenco_exclude_filter or None
prefs = preferences()
exclusion_filter = (prefs.flamenco_exclude_filter or '').strip()
missing_sources = await bam_interface.bam_copy(
missing_sources = await bat_interface.bat_copy(
Path(context.blend_data.filepath),
Path(preferences().flamenco_job_file_path),
Path(prefs.cloud_project_local_path),
Path(prefs.flamenco_job_file_path),
exclusion_filter
)
@@ -788,7 +790,7 @@ def register():
bpy.types.WindowManager.flamenco_status = EnumProperty(
items=[
('IDLE', 'IDLE', 'Not doing anything.'),
('PACKING', 'PACKING', 'BAM-packing all dependencies.'),
('PACKING', 'PACKING', 'BAT-packing all dependencies.'),
('COMMUNICATING', 'COMMUNICATING', 'Communicating with Flamenco Server.'),
],
name='flamenco_status',

185
blender_cloud/flamenco/bam_interface.py
View File

@@ -1,185 +0,0 @@
"""BAM packing interface for Flamenco."""
import logging
from pathlib import Path
import typing
# Timeout of the BAM subprocess, in seconds.
SUBPROC_READLINE_TIMEOUT = 600
log = logging.getLogger(__name__)
class CommandExecutionError(Exception):
"""Raised when there was an error executing a BAM command."""
pass
def wheel_pythonpath_278() -> str:
"""Returns the value of a PYTHONPATH environment variable needed to run BAM from its wheel file.
Workaround for Blender 2.78c not having io_blend_utils.pythonpath()
"""
import os
from ..wheels import wheel_filename
# Find the wheel to run.
wheelpath = wheel_filename('blender_bam')
log.info('Using wheel %s to run BAM-Pack', wheelpath)
# Update the PYTHONPATH to include that wheel.
existing_pypath = os.environ.get('PYTHONPATH', '')
if existing_pypath:
return os.pathsep.join((existing_pypath, wheelpath))
return wheelpath
async def bam_copy(base_blendfile: Path, target_blendfile: Path,
exclusion_filter: str) -> typing.List[Path]:
"""Uses BAM to copy the given file and dependencies to the target blendfile.
Due to the way blendfile_pack.py is programmed/structured, we cannot import it
and call a function; it has to be run in a subprocess.
:raises: asyncio.CanceledError if the task was cancelled.
:raises: asyncio.TimeoutError if reading a line from the BAM process timed out.
:raises: CommandExecutionError if the subprocess failed or output invalid UTF-8.
:returns: a list of missing sources; hopefully empty.
"""
import asyncio
import os
import shlex
import subprocess
import bpy
import io_blend_utils
args = [
bpy.app.binary_path_python,
'-m', 'bam.pack',
'--input', str(base_blendfile),
'--output', str(target_blendfile),
'--mode', 'FILE',
]
if exclusion_filter:
args.extend(['--exclude', exclusion_filter])
cmd_to_log = ' '.join(shlex.quote(s) for s in args)
log.info('Executing %s', cmd_to_log)
# Workaround for Blender 2.78c not having io_blend_utils.pythonpath()
if hasattr(io_blend_utils, 'pythonpath'):
pythonpath = io_blend_utils.pythonpath()
else:
pythonpath = wheel_pythonpath_278()
env = {
'PYTHONPATH': pythonpath,
# Needed on Windows because http://bugs.python.org/issue8557
'PATH': os.environ['PATH'],
}
if 'SYSTEMROOT' in os.environ: # Windows http://bugs.python.org/issue20614
env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
proc = await asyncio.create_subprocess_exec(
*args,
env=env,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
missing_sources = []
try:
while not proc.stdout.at_eof():
line = await asyncio.wait_for(proc.stdout.readline(),
SUBPROC_READLINE_TIMEOUT)
if not line:
# EOF received, so let's bail.
break
try:
line = line.decode('utf8')
except UnicodeDecodeError as ex:
raise CommandExecutionError('Command produced non-UTF8 output, '
'aborting: %s' % ex)
line = line.rstrip()
if 'source missing:' in line:
path = parse_missing_source(line)
missing_sources.append(path)
log.warning('Source is missing: %s', path)
log.info(' %s', line)
finally:
if proc.returncode is None:
# Always wait for the process, to avoid zombies.
try:
proc.kill()
except ProcessLookupError:
# The process is already stopped, so killing is impossible. That's ok.
log.debug("The process was already stopped, aborting is impossible. That's ok.")
await proc.wait()
log.info('The process stopped with status code %i', proc.returncode)
if proc.returncode:
raise CommandExecutionError('Process stopped with status %i' % proc.returncode)
return missing_sources
def parse_missing_source(line: str) -> Path:
r"""Parses a "missing source" line into a pathlib.Path.
>>> parse_missing_source(r" source missing: b'D\xc3\xaffficult \xc3\x9cTF-8 filename'")
PosixPath('Dïfficult ÜTF-8 filename')
>>> parse_missing_source(r" source missing: b'D\xfffficult Win1252 f\xeflen\xe6me'")
PosixPath('D<EFBFBD>fficult Win1252 f<>len<65>me')
"""
_, missing_source = line.split(': ', 1)
missing_source_as_bytes = parse_byte_literal(missing_source.strip())
# The file could originate from any platform, so UTF-8 and the current platform's
# filesystem encodings are just guesses.
try:
missing_source = missing_source_as_bytes.decode('utf8')
except UnicodeDecodeError:
import sys
try:
missing_source = missing_source_as_bytes.decode(sys.getfilesystemencoding())
except UnicodeDecodeError:
missing_source = missing_source_as_bytes.decode('ascii', errors='replace')
path = Path(missing_source)
return path
def parse_byte_literal(bytes_literal: str) -> bytes:
r"""Parses a repr(bytes) output into a bytes object.
>>> parse_byte_literal(r"b'D\xc3\xaffficult \xc3\x9cTF-8 filename'")
b'D\xc3\xaffficult \xc3\x9cTF-8 filename'
>>> parse_byte_literal(r"b'D\xeffficult Win1252 f\xeflen\xe6me'")
b'D\xeffficult Win1252 f\xeflen\xe6me'
"""
# Some very basic assertions to make sure we have a proper bytes literal.
assert bytes_literal[0] == "b"
assert bytes_literal[1] in {'"', "'"}
assert bytes_literal[-1] == bytes_literal[1]
import ast
return ast.literal_eval(bytes_literal)
if __name__ == '__main__':
import doctest
doctest.testmod()

35
blender_cloud/flamenco/bat_interface.py Normal file
View File

@@ -0,0 +1,35 @@
"""BAT🦇 packing interface for Flamenco."""
import asyncio
import logging
import typing
from pathlib import Path
from blender_asset_tracer import pack
from blender_asset_tracer.pack.transfer import FileTransferError
log = logging.getLogger(__name__)
async def bat_copy(base_blendfile: Path,
project: Path,
target: Path,
exclusion_filter: str) -> typing.Tuple[Path, typing.Set[Path]]:
"""Use BAT🦇 to copy the given file and dependencies to the target location.
:raises: FileTransferError if a file couldn't be transferred.
:returns: the path of the packed blend file, and a set of missing sources.
"""
loop = asyncio.get_event_loop()
with pack.Packer(base_blendfile, project, target) as packer:
if exclusion_filter:
packer.exclude(*exclusion_filter.split())
log.debug('awaiting strategise')
await loop.run_in_executor(None, packer.strategise)
log.debug('awaiting execute')
await loop.run_in_executor(None, packer.execute)
log.debug('done')
return packer.output_path, packer.missing_files

1
blender_cloud/wheels/__init__.py
View File

@@ -61,6 +61,7 @@ def wheel_filename(fname_prefix: str) -> str:
def load_wheels():
load_wheel('blender_asset_tracer', 'blender_asset_tracer')
load_wheel('lockfile', 'lockfile')
load_wheel('cachecontrol', 'CacheControl')
load_wheel('pillarsdk', 'pillarsdk')