Path remapping for packing

This commit changes how relative paths are handled depending on weather they are nested or not.

Layout is as follows.

- ./shots/01/shot_01.blend
- ./shots/01/maps/special.png
- ./maps/generic.png

Maps to...
- ./shot_01.blend
- ./_maps/special.png
- ./maps/generic.png

modules/blendfile_pack.py

@@ -33,11 +33,80 @@ del os, sys, path
 # --------
 
 
-def pack(blendfile_src, blendfile_dst, mode='FILE',
+# ----------------------
-         paths_remap_relbase=None,
+# debug low level output
-         deps_remap=None, paths_remap=None, paths_uuid=None,
+#
-         # yield reports
+# ... when internals _really_ fail & we want to know why
-         report=None):
+def _dbg(text):
+    import sys
+    from bam_utils.system import colorize
+    if type(text) is bytes:
+        text = text.decode('utf-8')
+    sys.__stdout__.write(colorize(text, color='red') + "\n")
+    sys.__stdout__.flush()
+
+
+def _relpath_remap(
+        path_src,
+        base_dir_src,
+        fp_basedir,
+        blendfile_src_dir_fakeroot,
+        ):
+
+    import os
+
+    path_dst = os.path.relpath(path_src, base_dir_src)
+
+    if blendfile_src_dir_fakeroot is None:
+        # /foo/../bar.png --> /foo/__/bar.png
+        path_dst = path_dst.replace(b'..', b'__')
+        path_dst_final = os.path.relpath(path_dst, fp_basedir)
+    else:
+        if b'..' in path_dst:
+            # remap, relative to project root
+
+            # paths
+            path_dst = os.path.join(blendfile_src_dir_fakeroot, path_dst)
+            path_dst = os.path.normpath(path_dst)
+            # if there are paths outside the root still...
+            # This means they are outside the project directory, We dont support this,
+            # so name accordingly
+            if b'..' in path_dst:
+                # SHOULD NEVER HAPPEN
+                path_dst = path_dst.replace(b'..', b'__nonproject__')
+            path_dst = os.path.normpath(path_dst)
+        else:
+            path_dst = b'_' + path_dst
+
+        path_dst_final = os.path.join(os.path.relpath(fp_basedir, base_dir_src), path_dst)
+        path_dst_final = os.path.normpath(path_dst_final)
+
+    return path_dst, path_dst_final
+
+
+def pack(
+        # store the blendfile relative to this directory, can be:
+        #    os.path.dirname(blendfile_src)
+        # but in some cases we wan't to use a path higher up.
+        # base_dir_src,
+        blendfile_src, blendfile_dst, mode='FILE',
+        paths_remap_relbase=None,
+        deps_remap=None, paths_remap=None, paths_uuid=None,
+        # yield reports
+        report=None,
+
+        # The project path, eg:
+        # /home/me/myproject/mysession/path/to/blend/file.blend
+        # the path would be:         b'path/to/blend'
+        #
+        # This is needed so we can choose to store paths
+        # relative to project or relative to the current file.
+        #
+        # When None, map _all_ paths are relative to the current blend.
+        # converting:  '../../bar' --> '__/__/bar'
+        # so all paths are nested and not moved outside the session path.
+        blendfile_src_dir_fakeroot=None,
+        ):
     """
     :param deps_remap: Store path deps_remap info as follows.
        {"file.blend": {"path_new": "path_old", ...}, ...}
@@ -63,7 +132,6 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
     path_temp_files = set()
     path_copy_files = set()
 
-    SUBDIR = b'data'
     TEMP_SUFFIX = b'@'
 
     if report is None:
@@ -75,32 +143,39 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
         import time
         t = time.time()
 
-    def temp_remap_cb(filepath, level):
+    base_dir_src = os.path.dirname(blendfile_src)
+    base_dir_dst = os.path.dirname(blendfile_dst)
+    _dbg(blendfile_src)
+    _dbg(blendfile_dst)
+
+    if mode == 'ZIP':
+        base_dir_dst_temp = os.path.join(base_dir_dst, b'__blendfile_temp__')
+    else:
+        base_dir_dst_temp = os.path.join(base_dir_dst, b'__blendfile_pack__')
+
+    def temp_remap_cb(filepath, rootdir):
         """
         Create temp files in the destination path.
         """
         filepath = blendfile_path_walker.utils.compatpath(filepath)
+        print(repr((os.path.join(rootdir, b'dummy'), base_dir_src, base_dir_src, blendfile_src_dir_fakeroot)))
 
-        if level == 0:
+        # first remap this blend file to the location it will end up (so we can get images relative to _that_)
-            filepath_tmp = os.path.join(base_dir_dst, os.path.basename(filepath)) + TEMP_SUFFIX
+        # TODO(cam) cache the results
-        else:
+        fp_basedir_conv = _relpath_remap(os.path.join(rootdir, b'dummy'), base_dir_src, base_dir_src, blendfile_src_dir_fakeroot)[0]
-            filepath_tmp = os.path.join(base_dir_dst, SUBDIR, os.path.basename(filepath)) + TEMP_SUFFIX
+        fp_basedir_conv = os.path.join(base_dir_src, os.path.dirname(fp_basedir_conv))
 
-        filepath_tmp = os.path.normpath(filepath_tmp)
+        # then get the file relative to the new location
+        filepath_tmp = _relpath_remap(filepath, base_dir_src, fp_basedir_conv, blendfile_src_dir_fakeroot)[0]
+        filepath_tmp = os.path.normpath(os.path.join(base_dir_dst_temp, filepath_tmp)) + TEMP_SUFFIX
 
         # only overwrite once (so we can write into a path already containing files)
         if filepath_tmp not in path_temp_files:
+            os.makedirs(os.path.dirname(filepath_tmp), exist_ok=True)
             shutil.copy(filepath, filepath_tmp)
             path_temp_files.add(filepath_tmp)
         return filepath_tmp
 
-    # base_dir_src = os.path.dirname(blendfile_src)
-    base_dir_dst = os.path.dirname(blendfile_dst)
-
-    base_dir_dst_subdir = os.path.join(base_dir_dst, SUBDIR)
-    if not os.path.exists(base_dir_dst_subdir):
-        os.makedirs(base_dir_dst_subdir)
-
     lib_visit = {}
     fp_blend_basename_last = b''
 
@@ -122,17 +197,23 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
         # assume the path might be relative
         path_src_orig = fp.filepath
         path_rel = blendfile_path_walker.utils.compatpath(path_src_orig)
-        path_base = path_rel.split(os.sep.encode('ascii'))[-1]
         path_src = blendfile_path_walker.utils.abspath(path_rel, fp.basedir)
 
-        # rename in the blend
+        # destination path realtive to the root
-        path_dst = os.path.join(base_dir_dst_subdir, path_base)
+        # assert(b'..' not in path_src)
+        assert(b'..' not in base_dir_src)
 
-        if fp.level == 0:
+        # first remap this blend file to the location it will end up (so we can get images relative to _that_)
-            path_dst_final = b"//" + os.path.join(SUBDIR, path_base)
+        # TODO(cam) cache the results
-        else:
+        fp_basedir_conv = _relpath_remap(fp_blend, base_dir_src, base_dir_src, blendfile_src_dir_fakeroot)[0]
-            path_dst_final = b'//' + path_base
+        fp_basedir_conv = os.path.join(base_dir_src, os.path.dirname(fp_basedir_conv))
 
+        # then get the file relative to the new location
+        path_dst, path_dst_final = _relpath_remap(path_src, base_dir_src, fp_basedir_conv, blendfile_src_dir_fakeroot)
+
+        path_dst = os.path.join(base_dir_dst, path_dst)
+
+        path_dst_final = b'//' + path_dst_final
         fp.filepath = path_dst_final
 
         # add to copy-list
@@ -187,12 +268,12 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
         for src, dst in path_copy_files:
             paths_uuid[os.path.relpath(dst, base_dir_dst).decode('utf-8')] = sha1_from_file(src)
         # XXX, better way to store temp target
-        blendfile_dst_tmp = temp_remap_cb(blendfile_src, 0)
+        blendfile_dst_tmp = temp_remap_cb(blendfile_src, base_dir_src)
         paths_uuid[os.path.basename(blendfile_src).decode('utf-8')] = sha1_from_file(blendfile_dst_tmp)
 
         # blend libs
         for dst in path_temp_files:
-            k = os.path.relpath(dst[:-len(TEMP_SUFFIX)], base_dir_dst).decode('utf-8')
+            k = os.path.relpath(dst[:-len(TEMP_SUFFIX)], base_dir_dst_temp).decode('utf-8')
             if k not in paths_uuid:
                 paths_uuid[k] = sha1_from_file(dst)
             del k
@@ -204,7 +285,7 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
     # Handle File Copy/Zip
 
     if mode == 'FILE':
-        blendfile_dst_tmp = temp_remap_cb(blendfile_src, 0)
+        blendfile_dst_tmp = temp_remap_cb(blendfile_src, base_dir_src)
 
         shutil.move(blendfile_dst_tmp, blendfile_dst)
         path_temp_files.remove(blendfile_dst_tmp)
@@ -229,11 +310,12 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
         with zipfile.ZipFile(blendfile_dst.decode('utf-8'), 'w', zipfile.ZIP_DEFLATED) as zip_handle:
             for fn in path_temp_files:
                 yield report("  %s: %r -> <archive>\n" % (colorize("copying", color='blue'), fn))
-                zip_handle.write(fn.decode('utf-8'),
+                zip_handle.write(
-                          arcname=os.path.relpath(fn[:-1], base_dir_dst).decode('utf-8'))
+                        fn.decode('utf-8'),
+                        arcname=os.path.relpath(fn[:-1], base_dir_dst_temp).decode('utf-8'))
                 os.remove(fn)
 
-            shutil.rmtree(base_dir_dst_subdir)
+            shutil.rmtree(base_dir_dst_temp)
 
             for src, dst in path_copy_files:
                 assert(b'.blend' not in dst)
@@ -245,6 +327,12 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
                     zip_handle.write(src.decode('utf-8'),
                               arcname=os.path.relpath(dst, base_dir_dst).decode('utf-8'))
 
+                    """
+                    _dbg(b"")
+                    _dbg(b"REAL_FILE:     " + dst)
+                    _dbg(b"RELATIVE_FILE: " + os.path.relpath(dst, base_dir_dst))
+                    """
+
         yield report("  %s: %r\n" % (colorize("written", color='green'), blendfile_dst))
     else:
         raise Exception("%s not a known mode" % mode)

modules/blendfile_path_walker.py

@@ -269,7 +269,7 @@ class FilePath:
                     return blend.find_blocks_from_code(b'LI')
 
         if temp_remap_cb is not None:
-            filepath_tmp = temp_remap_cb(filepath, level)
+            filepath_tmp = temp_remap_cb(filepath, rootdir)
         else:
             filepath_tmp = filepath
 

tests/test_cli.py

@@ -49,6 +49,7 @@ if not VERBOSE:
     import werkzeug._internal
     werkzeug._internal._log = lambda *a, **b: None
     del werkzeug
+# --------
 
 
 # -----------------------------------------
@@ -330,6 +331,12 @@ def file_quick_image(path, filepart=None):
         f.write(write_png(b'0000' * 4, 2, 2))
 
 
+def _dbg_dump_path(path):
+    stdout, stderr, returncode = run(["find", path], path)
+    print("Contents of: %r" % path)
+    print("\n".join(sorted(stdout.decode('utf-8').split("\n"))))
+
+
 def blendfile_template_create(blendfile, blendfile_root, create_id, deps):
     returncode_test = 123
     blendfile_deps_json = os.path.join(TEMP_LOCAL, "blend_template_deps.json")
@@ -838,7 +845,7 @@ class BamRelativeAbsoluteTest(BamSessionTestCase):
                 print("Exists?", f_abs)
             self.assertTrue(os.path.exists(f_abs))
 
-    def _test_absolute_relative_mix(self):
+    def test_absolute_relative_mix(self):
         """
         Layout is as follows.
 
@@ -855,10 +862,16 @@ class BamRelativeAbsoluteTest(BamSessionTestCase):
         # absolute path: (project relative) -->
         # checkout path: (relative to blend)
         blendfile = ("shots/01/shot_01.blend", "shot_01.blend")
-        images = (
+        if 1:
-            ("shots/01/maps/special.png", "_maps/special.png"),
+            images = (
-            ("maps/generic.png", "maps/generic.png"),
+                ("shots/01/maps/special.png", "_maps/special.png"),
-            )
+                ("maps/generic.png", "maps/generic.png"),
+                )
+        else:
+            images = (
+                ("shots/01/maps/special.png", "maps/special.png"),
+                ("maps/generic.png", "__/__/maps/generic.png"),
+                )
 
         self._test_from_files(blendfile, images)
 

webservice/bam/application/__init__.py

@@ -356,13 +356,19 @@ class FileAPI(Resource):
         paths_uuid = {}
 
         if filepath.endswith(".blend"):
+
+            # find the path relative to the project's root
+            blendfile_src_dir_fakeroot = os.path.dirname(os.path.relpath(filepath, paths_remap_relbase))
+
             try:
                 yield from blendfile_pack.pack(
                         filepath.encode('utf-8'), filepath_zip.encode('utf-8'), mode='ZIP',
                         paths_remap_relbase=paths_remap_relbase.encode('utf-8'),
                         # TODO(cam) this just means the json is written in the zip
                         deps_remap=deps_remap, paths_remap=paths_remap, paths_uuid=paths_uuid,
-                        report=report)
+                        report=report,
+                        blendfile_src_dir_fakeroot=blendfile_src_dir_fakeroot.encode('utf-8'),
+                        )
             except:
                 log.exception("Error packing the blend file")
                 return