progress info when checking out

client/cli/bam

@@ -152,15 +152,33 @@ class bam_utils:
             local_filename += ".zip"
 
         with open(local_filename, 'wb') as f:
+            import struct
+            ID_MESSAGE = 1
+            ID_PAYLOAD = 2
+            head = r.raw.read(4)
+            if head != b'BAM\0':
+                print("Bad header...")
+                return
+
+            while True:
+                msg_type, msg_size = struct.unpack("<II", r.raw.read(8))
+                if msg_type == ID_MESSAGE:
+                    sys.stdout.write(r.raw.read(msg_size).decode('utf-8'))
+                    sys.stdout.flush()
+                elif msg_type == ID_PAYLOAD:
+                    # payload
+                    break
+
+            tot_size = 0
             for chunk in r.iter_content(chunk_size=1024):
                 if chunk: # filter out keep-alive new chunks
+                    tot_size += len(chunk)
                     f.write(chunk)
                     f.flush()
 
-                    sys.stdout.write(".")
+                    sys.stdout.write("\rdownload: [%03d%%]" % ((100 * tot_size) // msg_size))
                     sys.stdout.flush()
-
+        sys.stdout.write("\nwritten: %r\n" % local_filename)
-        print("Written:", local_filename)
 
     @staticmethod
     def commit(paths, message):

packer/packer.py

@@ -24,7 +24,9 @@ TIMEIT = True
 
 
 def pack(blendfile_src, blendfile_dst, mode='FILE',
-         deps_remap=None, paths_remap=None, paths_uuid=None):
+         deps_remap=None, paths_remap=None, paths_uuid=None,
+         # yield reports
+         report=None):
     """
     :param deps_remap: Store path deps_remap info as follows.
        {"file.blend": {"path_new": "path_old", ...}, ...}
@@ -54,6 +56,11 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
     # TODO, make configurable
     WRITE_JSON_REMAP = True
 
+    if report is None:
+        raise Exception("report not set!")
+
+    yield report("%s: %r...\n" % (colorize("\nscanning deps", color='bright_green'), blendfile_src))
+
     if TIMEIT:
         import time
         t = time.time()
@@ -77,6 +84,7 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
             path_temp_files.add(filepath_tmp)
         return filepath_tmp
 
+
     # base_dir_src = os.path.dirname(blendfile_src)
     base_dir_dst = os.path.dirname(blendfile_dst)
 
@@ -85,6 +93,7 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
         os.makedirs(base_dir_dst_subdir)
 
     lib_visit = {}
+    _last = b''
 
     for fp, (rootdir, fp_blend_basename) in blendfile_path_walker.FilePath.visit_from_blend(
             blendfile_src,
@@ -94,6 +103,10 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
             lib_visit=lib_visit,
             ):
 
+        if _last != fp_blend_basename:
+            yield report("  %s:       %s\n" % (colorize("blend", color='blue'), fp.basedir + fp_blend_basename))
+            _last = fp_blend_basename
+
         # assume the path might be relative
         path_src_orig = fp.filepath
         path_rel = blendfile_path_walker.utils.compatpath(path_src_orig)
@@ -126,6 +139,9 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
     if TIMEIT:
         print("  Time: %.4f\n" % (time.time() - t))
 
+    yield report(("%s: %d files\n") %
+                 (colorize("\narchiving", color='bright_green'), len(path_copy_files) + 1))
+
     # handle deps_remap and file renaming
     if deps_remap is not None:
         blendfile_src_basename = os.path.basename(blendfile_src).decode('utf-8')
@@ -183,19 +199,18 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
             assert(b'.blend' not in dst)
 
             if not os.path.exists(src):
-                print("  Source missing! %r" % src)
+                yield report("  %s: %r\n" % (colorize("source missing", color='red'), src))
             else:
-                print("  Copying %r -> %r" % (src, dst))
+                yield report("  %s: %r -> %r\n" % (colorize("copying", color='blue'), src, dst))
                 shutil.copy(src, dst)
 
-
+        yield report("  %s: %r\n" % (colorize("written", color='green'), blendfile_dst))
-        print("  Written:", blendfile_dst)
 
     elif mode == 'ZIP':
         import zipfile
         with zipfile.ZipFile(blendfile_dst.decode('utf-8'), 'w', zipfile.ZIP_DEFLATED) as zip:
             for fn in path_temp_files:
-                print("  Copying %r -> <zip>" % fn)
+                yield report("  %s: %r -> <archive>\n" % (colorize("copying", color='blue'), fn))
                 zip.write(fn.decode('utf-8'),
                           arcname=os.path.relpath(fn[:-1], base_dir_dst).decode('utf-8'))
                 os.remove(fn)
@@ -206,9 +221,9 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
                 assert(b'.blend' not in dst)
 
                 if not os.path.exists(src):
-                    print("  Source missing! %r" % src)
+                    yield report("  %s: %r\n" % (colorize("source missing", color='red'), src))
                 else:
-                    print("  Copying %r -> <zip>" % src)
+                    yield report("  %s: %r -> <archive>\n" % (colorize("copying", color='blue'), src))
                     zip.write(src.decode('utf-8'),
                               arcname=os.path.relpath(dst, base_dir_dst).decode('utf-8'))
 
@@ -233,7 +248,7 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
                 del write_dict_as_json
 
 
-        print("  Written:", blendfile_dst)
+        yield report("  %s: %r\n" % (colorize("written", color='green'), blendfile_dst))
     else:
         raise Exception("%s not a known mode" % mode)
 
@@ -309,3 +324,34 @@ def main():
 
 if __name__ == "__main__":
     main()
+
+# TODO(cam) de-duplicate
+USE_COLOR = True
+if USE_COLOR:
+    color_codes = {
+        'black':        '\033[0;30m',
+        'bright_gray':  '\033[0;37m',
+        'blue':         '\033[0;34m',
+        'white':        '\033[1;37m',
+        'green':        '\033[0;32m',
+        'bright_blue':  '\033[1;34m',
+        'cyan':         '\033[0;36m',
+        'bright_green': '\033[1;32m',
+        'red':          '\033[0;31m',
+        'bright_cyan':  '\033[1;36m',
+        'purple':       '\033[0;35m',
+        'bright_red':   '\033[1;31m',
+        'yellow':       '\033[0;33m',
+        'bright_purple':'\033[1;35m',
+        'dark_gray':    '\033[1;30m',
+        'bright_yellow':'\033[1;33m',
+        'normal':       '\033[0m',
+    }
+
+    def colorize(msg, color=None):
+        return (color_codes[color] + msg + color_codes['normal'])
+else:
+    def colorize(msg, color=None):
+        return msg
+
+

webservice/bam/application/__init__.py

@@ -150,16 +150,41 @@ class FileAPI(Resource):
             elif os.path.isdir(filepath):
                 return jsonify(message="Path is a directory %r" % filepath)
 
-            # pack the file!
+            def response_message_iter():
-            print("PACKING")
+                ID_MESSAGE = 1
-            filepath_zip = self.pack_fn(filepath)
+                ID_PAYLOAD = 2
+                import struct
 
-            # TODO, handle fail
+                def report(txt):
-            if filepath_zip is None:
+                    txt_bytes = txt.encode('utf-8')
-                return jsonify(message="Path not found %r" % filepath)
+                    return struct.pack('<II', ID_MESSAGE, len(txt_bytes)) + txt_bytes
 
-            f = open(filepath_zip, 'rb')
+                yield b'BAM\0'
-            return Response(f, direct_passthrough=True)
+
+                # pack the file!
+                import tempfile
+                filepath_zip = tempfile.mkstemp(suffix=".zip")
+                yield from self.pack_fn(filepath, filepath_zip, report)
+
+                # TODO, handle fail
+                if not os.path.exists(filepath_zip[-1]):
+                    yield report("%s: %r\n" % (colorize("failed to extract", color='red'), filepath))
+                    return
+
+                with open(filepath_zip[-1], 'rb') as f:
+                    f.seek(0, os.SEEK_END)
+                    f_size = f.tell()
+                    f.seek(0, os.SEEK_SET)
+
+                    yield struct.pack('<II', ID_PAYLOAD, f_size)
+                    while True:
+                        data = f.read(1024)
+                        if not data:
+                            break
+                        yield data
+
+            # return Response(f, direct_passthrough=True)
+            return Response(response_message_iter(), direct_passthrough=True)
 
         else:
             return jsonify(message="Command unknown")
@@ -214,7 +239,7 @@ class FileAPI(Resource):
             return jsonify(message='File not allowed')
 
     @staticmethod
-    def pack_fn(filepath):
+    def pack_fn(filepath, filepath_zip, report):
         import os
         assert(os.path.exists(filepath) and not os.path.isdir(filepath))
 
@@ -233,17 +258,17 @@ class FileAPI(Resource):
             sys.path.append(modpath)
         del modpath
 
-        import tempfile
         import packer
 
-        filepath_zip = tempfile.mkstemp(suffix=".zip")
         print("  Source path:", filepath)
         print("  Zip path:", filepath_zip)
 
         try:
-            packer.pack(filepath.encode('utf-8'), filepath_zip[-1].encode('utf-8'), mode='ZIP',
+            yield from packer.pack(
-                        # TODO(cam) this just means the json is written in the zip
+                    filepath.encode('utf-8'), filepath_zip[-1].encode('utf-8'), mode='ZIP',
-                        deps_remap={}, paths_remap={}, paths_uuid={})
+                    # TODO(cam) this just means the json is written in the zip
+                    deps_remap={}, paths_remap={}, paths_uuid={},
+                    report=report)
             return filepath_zip[-1]
         except:
             import traceback
@@ -259,3 +284,32 @@ class FileAPI(Resource):
 
 api.add_resource(FilesListAPI, '/file_list', endpoint='file_list')
 api.add_resource(FileAPI, '/file', endpoint='file')
+
+USE_COLOR = True
+if USE_COLOR:
+    color_codes = {
+        'black':        '\033[0;30m',
+        'bright_gray':  '\033[0;37m',
+        'blue':         '\033[0;34m',
+        'white':        '\033[1;37m',
+        'green':        '\033[0;32m',
+        'bright_blue':  '\033[1;34m',
+        'cyan':         '\033[0;36m',
+        'bright_green': '\033[1;32m',
+        'red':          '\033[0;31m',
+        'bright_cyan':  '\033[1;36m',
+        'purple':       '\033[0;35m',
+        'bright_red':   '\033[1;31m',
+        'yellow':       '\033[0;33m',
+        'bright_purple':'\033[1;35m',
+        'dark_gray':    '\033[1;30m',
+        'bright_yellow':'\033[1;33m',
+        'normal':       '\033[0m',
+    }
+
+    def colorize(msg, color=None):
+        return (color_codes[color] + msg + color_codes['normal'])
+else:
+    def colorize(msg, color=None):
+        return msg
+