From 98f67c5d7f43f9db36119d63bc9778af3ebf69df Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Fri, 23 Jan 2015 20:30:19 +1100 Subject: [PATCH] test local replay of edits, needed for local cache --- bam/blend/blendfile.py | 34 +++++++++-- bam/blend/blendfile_pack.py | 27 ++++++++- bam/blend/blendfile_path_walker.py | 35 ++++++++++- bam/cli.py | 58 +++++++++++++++++++ .../application/modules/resources/__init__.py | 8 +++ 5 files changed, 152 insertions(+), 10 deletions(-) diff --git a/bam/blend/blendfile.py b/bam/blend/blendfile.py index 46c88ba..21f1b48 100644 --- a/bam/blend/blendfile.py +++ b/bam/blend/blendfile.py @@ -350,12 +350,21 @@ class BlendFileBlock: assert(type(dna_type_id) is bytes) self.refine_type_from_index(self.file.sdna_index_from_id[dna_type_id]) - def get(self, path, + def get_file_offset(self, path, default=..., sdna_index_refine=None, - use_nil=True, use_str=True, base_index=0, ): + """ + Return (offset, length) + """ + assert(type(path) is bytes) + + ofs = self.file_offset + if base_index != 0: + assert(base_index < self.count) + ofs += (self.size // self.count) * base_index + self.file.handle.seek(ofs, os.SEEK_SET) if sdna_index_refine is None: sdna_index_refine = self.sdna_index @@ -363,13 +372,30 @@ class BlendFileBlock: self.file.ensure_subtype_smaller(self.sdna_index, sdna_index_refine) dna_struct = self.file.structs[sdna_index_refine] - ofs = self.file_offset + field = dna_struct.field_from_path( + self.file.header, self.file.handle, path) + return (self.file.handle.tell(), field.dna_name.array_size) + + def get(self, path, + default=..., + sdna_index_refine=None, + use_nil=True, use_str=True, + base_index=0, + ): + + ofs = self.file_offset if base_index != 0: assert(base_index < self.count) ofs += (self.size // self.count) * base_index - self.file.handle.seek(ofs, os.SEEK_SET) + + if sdna_index_refine is None: + sdna_index_refine = self.sdna_index + else: + self.file.ensure_subtype_smaller(self.sdna_index, sdna_index_refine) + + dna_struct = self.file.structs[sdna_index_refine] return dna_struct.field_get( self.file.header, self.file.handle, path, default=default, diff --git a/bam/blend/blendfile_pack.py b/bam/blend/blendfile_pack.py index 4851d9e..02db266 100755 --- a/bam/blend/blendfile_pack.py +++ b/bam/blend/blendfile_pack.py @@ -122,6 +122,13 @@ def pack( # Read variations from json files. use_variations=True, + + # do _everything_ except to write the paths. + # useful if we want to calculate deps to remap but postpone applying them. + readonly=False, + # dict of binary_edits: + # {file: [(ofs, bytes), ...], ...} + binary_edits=None, ): """ :param deps_remap: Store path deps_remap info as follows. @@ -135,7 +142,7 @@ def pack( # so we can modify in-place. # - temp files are only created once, (if we never touched them before), # this way, for linked libraries - a single blend file may be used - # multiple times, each access will apply new edits ontop of the old ones. + # multiple times, each access will apply new edits on top of the old ones. # - we track which libs we have touched (using 'lib_visit' arg), # this means that the same libs wont be touched many times to modify the same data # also prevents cyclic loops from crashing. @@ -275,7 +282,7 @@ def pack( for fp, (rootdir, fp_blend_basename) in blendfile_path_walker.FilePath.visit_from_blend( blendfile_src, - readonly=False, + readonly=readonly, temp_remap_cb=temp_remap_cb, recursive=True, recursive_all=all_deps, @@ -293,6 +300,14 @@ def pack( yield report(" %s: %s\n" % (colorize("blend", color='blue'), fp_blend)) fp_blend_basename_last = fp_blend_basename + if binary_edits is not None: + # TODO, temp_remap_cb makes paths, this isn't ideal, + # in this case we only want to remap! + tmp = temp_remap_cb(fp_blend, base_dir_src) + tmp = os.path.relpath(tmp[:-1], base_dir_dst_temp) + binary_edits_curr = binary_edits.setdefault(tmp, []) + del tmp + # assume the path might be relative path_src_orig = fp.filepath path_rel = blendfile_path_walker.utils.compatpath(path_src_orig) @@ -323,7 +338,13 @@ def pack( path_dst = os.path.join(base_dir_dst, path_dst) path_dst_final = b'//' + path_dst_final - fp.filepath = path_dst_final + + # Assign direct or add to edit-list (to apply later) + if not readonly: + fp.filepath = path_dst_final + if binary_edits is not None: + fp.filepath_assign_edits(path_dst_final, binary_edits_curr) + # add to copy-list # never copy libs (handled separately) if not isinstance(fp, blendfile_path_walker.FPElem_block_path) or fp.userdata[0].code != b'LI': diff --git a/bam/blend/blendfile_path_walker.py b/bam/blend/blendfile_path_walker.py index 97a57e1..c7d08a3 100644 --- a/bam/blend/blendfile_path_walker.py +++ b/bam/blend/blendfile_path_walker.py @@ -106,6 +106,24 @@ class FPElem: else: return utils.compatpath(filepath) + def filepath_assign_edits(self, filepath, binary_edits): + self._set_cb_edits(filepath, binary_edits) + + @staticmethod + def _filepath_assign_edits(block, path, filepath, binary_edits): + """ + Record the write to a separate entry (binary file-like object), + this lets us replay the edits later. + (so we can replay them onto the clients local cache without a file transfer). + """ + import struct + assert(type(filepath) is bytes) + assert(type(path) is bytes) + ofs, size = block.get_file_offset(path) + # ensure we dont write past the field size & allow for \0 + filepath = filepath[:size - 1] + binary_edits.append((ofs, filepath + b'\0')) + @property def filepath(self): return self._get_cb() @@ -134,11 +152,15 @@ class FPElem_block_path(FPElem): block, path = self.userdata block[path] = filepath + def _set_cb_edits(self, filepath, binary_edits): + block, path = self.userdata + self._filepath_assign_edits(block, path, filepath, binary_edits) + class FPElem_sequence_single(FPElem): """ Movie sequence - userdata = (block, path) + userdata = (block, path, sub_block, sub_path) """ __slots__ = () @@ -148,16 +170,23 @@ class FPElem_sequence_single(FPElem): def _set_cb(self, filepath): block, path, sub_block, sub_path = self.userdata - head, sep, tail = utils.splitpath(filepath) block[path] = head + sep sub_block[sub_path] = tail + def _set_cb_edits(self, filepath, binary_edits): + block, path, sub_block, sub_path = self.userdata + head, sep, tail = utils.splitpath(filepath) + + self._filepath_assign_edits(block, path, head + sep, binary_edits) + self._filepath_assign_edits(sub_block, sub_path, tail, binary_edits) + + class FPElem_sequence_image_seq(FPElem_sequence_single): """ Image sequence - userdata = (block, path) + userdata = (block, path, sub_block, sub_path) """ __slots__ = () diff --git a/bam/cli.py b/bam/cli.py index 0d724a7..b2f3c9d 100755 --- a/bam/cli.py +++ b/bam/cli.py @@ -454,6 +454,62 @@ class bam_commands: sys.stdout.write("\nwritten: %r\n" % session_rootdir) + # ------------------- + # replay binary edits + # + # We've downloaded the files pristine from their repo. + # This means we can use local cache and avoid re-downloading. + # + # But for files to work locally we have to apply binary edits given to us by the server. + + sys.stdout.write("replaying edits...\n") + with open(os.path.join(session_rootdir, ".bam_paths_edit.data"), 'rb') as fh: + import pickle + binary_edits_all = pickle.load(fh) + paths_uuid_update = {} + for blendfile, binary_edits in binary_edits_all.items(): + if binary_edits: + sys.stdout.write(" operating on: %r\n" % blendfile) + sys.stdout.flush() + blendfile_abs = os.path.join(session_rootdir, blendfile.decode('utf-8')) + # we don't want to read, just edit whats there. + with open(blendfile_abs, 'rb+') as fh_blend: + for ofs, data in binary_edits: + # sys.stdout.write("\n%r\n" % data) + sys.stdout.flush() + # ensure we're writing to the correct location. + # fh_blend.seek(ofs) + # sys.stdout.write(repr(b'existing data: ' + fh_blend.read(len(data) + 1))) + fh_blend.seek(ofs) + fh_blend.write(data) + sys.stdout.write("\n") + sys.stdout.flush() + + # update hash! + # we could do later, but the file is fresh in cache, so do now + from bam.utils.system import uuid_from_file + paths_uuid_update[blendfile.decode('utf-8')] = uuid_from_file(blendfile_abs) + del uuid_from_file + del pickle + del binary_edits_all + + if paths_uuid_update: + # freshen the UUID's based on the replayed binary_edits + from bam.utils.system import write_json_to_file + paths_uuid = bam_session.load_paths_uuid(session_rootdir) + assert(set(paths_uuid_update.keys()).issubset(set(paths_uuid.keys()))) + paths_uuid.update(paths_uuid_update) + write_json_to_file(os.path.join(session_rootdir, ".bam_paths_uuid.json"), paths_uuid) + del write_json_to_file + del paths_uuid + del paths_uuid_update + + # we will need to keep these later + os.remove(os.path.join(session_rootdir, ".bam_paths_edit.data")) + + # done with binary edits + # ---------------------- + @staticmethod def update(paths): # Load project configuration @@ -874,6 +930,7 @@ class bam_commands: with open(filepath_remap, 'wb') as fh: import pickle pickle.dump(remap_data, fh, pickle.HIGHEST_PROTOCOL) + del pickle @staticmethod def remap_finish( @@ -896,6 +953,7 @@ class bam_commands: with open(filepath_remap, 'rb') as fh: import pickle remap_data = pickle.load(fh) + del pickle from bam.blend import blendfile_path_remap blendfile_path_remap.finish( diff --git a/webservice/bam/application/modules/resources/__init__.py b/webservice/bam/application/modules/resources/__init__.py index c019561..4248703 100644 --- a/webservice/bam/application/modules/resources/__init__.py +++ b/webservice/bam/application/modules/resources/__init__.py @@ -394,6 +394,8 @@ class FileAPI(Resource): paths_remap = {} paths_uuid = {} + binary_edits = {} + if filepath.endswith(".blend"): # find the path relative to the project's root @@ -407,6 +409,8 @@ class FileAPI(Resource): all_deps=all_deps, report=report, blendfile_src_dir_fakeroot=blendfile_src_dir_fakeroot.encode('utf-8'), + readonly=True, + binary_edits=binary_edits, ) except: log.exception("Error packing the blend file") @@ -453,7 +457,11 @@ class FileAPI(Resource): write_dict_as_json(".bam_paths_remap.json", paths_remap) write_dict_as_json(".bam_paths_uuid.json", paths_uuid) + import pickle + zip_handle.writestr(".bam_paths_edit.data", pickle.dumps(binary_edits, pickle.HIGHEST_PROTOCOL)) del write_dict_as_json + + del binary_edits # done writing json! @staticmethod