bam cli: path remap command

client/cli/bam.py

@@ -822,6 +822,73 @@ class bam_commands:
             for f_src, f_dst, f_dst_abs, f_status in status_walker():
                 print("  %r -> (%r = %r) %s" % (f_src, f_dst, f_dst_abs, f_status))
 
+    @staticmethod
+    def remap_start(
+            paths,
+            ):
+        filepath_remap = "bam_remap.data"
+
+        for p in paths:
+            if not os.path.exists(p):
+                fatal("Path %r not found!" % p)
+        paths = [p.encode('utf-8') for p in paths]
+
+
+        if os.path.exists(filepath_remap):
+            fatal("Remap in progress, run with 'finish' or remove %r" % filepath_remap)
+
+        import blendfile_path_remap
+        remap_data = blendfile_path_remap.start(
+                paths,
+                )
+
+        with open(filepath_remap, 'wb') as fh:
+            import pickle
+            pickle.dump(remap_data, fh, pickle.HIGHEST_PROTOCOL)
+
+    @staticmethod
+    def remap_finish(
+            paths,
+            force_relative=False,
+            dry_run=False,
+            ):
+        filepath_remap = "bam_remap.data"
+
+        for p in paths:
+            if not os.path.exists(p):
+                fatal("Path %r not found!" % p)
+        # bytes needed for blendfile_path_remap API
+        paths = [p.encode('utf-8') for p in paths]
+
+
+        if not os.path.exists(filepath_remap):
+            fatal("Remap not started, run with 'start', (%r not found)" % filepath_remap)
+
+        with open(filepath_remap, 'rb') as fh:
+            import pickle
+            remap_data = pickle.load(fh)
+
+        import blendfile_path_remap
+        blendfile_path_remap.finish(
+                paths, remap_data,
+                force_relative=force_relative,
+                dry_run=dry_run,
+                )
+
+        if not dry_run:
+            os.remove(filepath_remap)
+
+    @staticmethod
+    def remap_reset(
+            ):
+        filepath_remap = "bam_remap.data"
+        if os.path.exists(filepath_remap):
+            os.remove(filepath_remap)
+        else:
+            fatal("remapping not started, nothing to do!")
+
+
+
 # -----------------------------------------------------------------------------
 # Argument Parser
 
@@ -1000,6 +1067,68 @@ def create_argparse_deps(subparsers):
                     )
 
 
+def create_argparse_remap(subparsers):
+    subparse = subparsers.add_parser(
+            "remap",
+            help="Remap blend file paths",
+            )
+
+    subparse_remap_commands = subparse.add_subparsers(
+            title="Remap commands",
+            description='valid subcommands',
+            help='additional help',
+            )
+    sub_subparse = subparse_remap_commands.add_parser(
+            "start",
+            help="Start remapping the blend files",
+            )
+
+    sub_subparse.add_argument(
+            dest="paths", nargs="*",
+            help="Path(s) to operate on",
+            )
+    sub_subparse.set_defaults(
+            func=lambda args:
+            bam_commands.remap_start(
+                    args.paths or ["."],
+                    ),
+                    )
+
+    sub_subparse = subparse_remap_commands.add_parser(
+            "finish",
+            help="Finish remapping the blend files",
+            )
+    sub_subparse.add_argument(
+            dest="paths", nargs="*",
+            help="Path(s) to operate on",
+            )
+    sub_subparse.add_argument(
+            "-r", "--force-relative", dest="force_relative", action='store_true',
+            help="Make all remapped paths relative (even if they were originally absolute)",
+            )
+    sub_subparse.add_argument(
+            "-d", "--dry-run", dest="dry_run", action='store_true',
+            help="Just print output as if the paths are being run",
+            )
+    sub_subparse.set_defaults(
+            func=lambda args:
+            bam_commands.remap_finish(
+                    args.paths or ["."],
+                    force_relative=args.force_relative,
+                    dry_run=args.dry_run,
+                    ),
+                    )
+
+    sub_subparse = subparse_remap_commands.add_parser(
+            "reset",
+            help="Cancel path remapping",
+            )
+    sub_subparse.set_defaults(
+            func=lambda args:
+            bam_commands.remap_reset(),
+            )
+
+
 def create_argparse():
     import argparse
 
@@ -1025,6 +1154,7 @@ def create_argparse():
     create_argparse_status(subparsers)
     create_argparse_list(subparsers)
     create_argparse_deps(subparsers)
+    create_argparse_remap(subparsers)
 
     return parser
 

modules/blendfile_path_remap.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+
+"""
+Module for remapping paths from one directory to another.
+"""
+
+import os
+
+
+# ----------------------------------------------------------------------------
+# private utility functions
+
+def _is_blend(f):
+    return f.lower().endswith(b'.blend')
+
+
+def _warn(msg):
+    print("Warning: %s" % msg)
+
+
+def _uuid_from_file(fn, block_size=1 << 20):
+    with open(fn, 'rb') as f:
+        # first get the size
+        f.seek(0, os.SEEK_END)
+        size = f.tell()
+        f.seek(0, os.SEEK_SET)
+        # done!
+
+        import hashlib
+        sha1 = hashlib.new('sha512')
+        while True:
+            data = f.read(block_size)
+            if not data:
+                break
+            sha1.update(data)
+        return (size, sha1.hexdigest())
+
+
+def _iter_files(paths, check_ext=None):
+    for p in paths:
+        p = os.path.abspath(p)
+        for dirpath, dirnames, filenames in os.walk(p):
+            # skip '.svn'
+            if dirpath.startswith(b'.') and dirpath != b'.':
+                continue
+
+            for filename in filenames:
+                if check_ext is None or check_ext(filename):
+                    filepath = os.path.join(dirpath, filename)
+                    yield filepath
+
+
+# ----------------------------------------------------------------------------
+# Public Functions
+
+def start(
+    paths,
+    dry_run=False,
+    ):
+    # {(sha1, length): "filepath"}
+    remap_uuid = {}
+
+    # all files we need to map
+    # absolute paths
+    files_to_map = set()
+
+    # TODO, validate paths aren't nested! ["/foo", "/foo/bar"]
+    # it will cause problems touching files twice!
+
+    # ------------------------------------------------------------------------
+    # First walk over all blends
+    import blendfile_path_walker
+
+    for blendfile in _iter_files(paths, check_ext=_is_blend):
+        for fp, (rootdir, fp_blend_basename) in blendfile_path_walker.FilePath.visit_from_blend(
+                blendfile,
+                readonly=True,
+                recursive=False,
+                ):
+            # TODO. warn when referencing files outside 'paths'
+
+            # so we can update the reference
+            f_abs = fp.filepath_absolute
+            f_abs = os.path.normpath(f_abs)
+            if os.path.exists(f_abs):
+                files_to_map.add(f_abs)
+            else:
+                _warn("file %r from %r not found!" % (f_abs, blendfile))
+
+        # so we can know where its moved to
+        files_to_map.add(blendfile)
+    del blendfile_path_walker
+
+    # ------------------------------------------------------------------------
+    # Store UUID
+    #
+    # note, sorting is only to give predictable warnings/behavior
+    for f in sorted(files_to_map):
+        f_uuid = _uuid_from_file(f)
+
+        f_match = remap_uuid.get(f_uuid)
+        if f_match is not None:
+            _warn("duplicate file found! (%r, %r)" % (f_match, f))
+
+        remap_uuid[f_uuid] = f
+
+    # now find all deps
+    remap_data_args = (
+            remap_uuid,
+            )
+
+    return remap_data_args
+
+
+def finish(
+    paths, remap_data_args,
+    force_relative=False,
+    dry_run=False,
+    ):
+
+    (remap_uuid,
+     ) = remap_data_args
+
+    remap_src_to_dst = {}
+    remap_dst_to_src = {}
+
+    for f_dst in _iter_files(paths):
+        f_uuid = _uuid_from_file(f_dst)
+        f_src = remap_uuid.get(f_uuid)
+        if f_src is not None:
+            remap_src_to_dst[f_src] = f_dst
+            remap_dst_to_src[f_dst] = f_src
+
+    # now the fun begins, remap _all_ paths
+    import blendfile_path_walker
+
+
+    for blendfile_dst in _iter_files(paths, check_ext=_is_blend):
+        blendfile_src = remap_dst_to_src.get(blendfile_dst)
+        if blendfile_src is None:
+            _warn("new blendfile added since beginning 'remap': %r" % blendfile_dst)
+            continue
+
+        blendfile_src_basedir = os.path.dirname(blendfile_src)
+        blendfile_dst_basedir = os.path.dirname(blendfile_dst)
+        for fp, (rootdir, fp_blend_basename) in blendfile_path_walker.FilePath.visit_from_blend(
+                blendfile_dst,
+                readonly=False,
+                recursive=False,
+                ):
+            # TODO. warn when referencing files outside 'paths'
+
+            # so we can update the reference
+            f_src_rel = fp.filepath
+            is_relative = f_src_rel.startswith(b'//')
+            if is_relative:
+                f_src_abs = fp.filepath_absolute_resolve(basedir=blendfile_src_basedir)
+            else:
+                f_src_abs = f_src_rel
+
+            f_src_abs = os.path.normpath(f_src_abs)
+            f_dst_abs = remap_src_to_dst.get(f_src_abs)
+
+            if f_dst_abs is None:
+                _warn("file %r from %r not found in map!" % (f_src_abs, blendfile_dst))
+                continue
+
+            # now remap!
+            if is_relative or force_relative:
+                f_dst_rel = b'//' + os.path.relpath(f_dst_abs, blendfile_dst_basedir)
+            else:
+                f_dst_rel = f_dst_abs
+
+            if f_dst_rel != f_src_rel:
+                if not dry_run:
+                    fp.filepath = f_dst_abs
+                # print("remap %r -> %r" % (f_src_abs, fp.filepath))
+
+    del blendfile_path_walker
+