archive/blender-asset-manager
Archived
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

progress info when checking out

Browse Source
This commit is contained in:
Campbell Barton
2014-11-04 21:46:18 +01:00
parent 8015f0d106
commit 3222d10276
3 changed files with 144 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
client/cli/bam
View File

@@ -152,15 +152,33 @@ class bam_utils:
local_filename += ".zip"
with open(local_filename, 'wb') as f:
import struct
ID_MESSAGE = 1
ID_PAYLOAD = 2
head = r.raw.read(4)
if head != b'BAM\0':
print("Bad header...")
return
while True:
msg_type, msg_size = struct.unpack("<II", r.raw.read(8))
if msg_type == ID_MESSAGE:
sys.stdout.write(r.raw.read(msg_size).decode('utf-8'))
sys.stdout.flush()
elif msg_type == ID_PAYLOAD:
# payload
break
tot_size = 0
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
tot_size += len(chunk)
f.write(chunk)
f.flush()
sys.stdout.write(".")
sys.stdout.write("\rdownload: [%03d%%]" % ((100 * tot_size) // msg_size))
sys.stdout.flush()
print("Written:", local_filename)
sys.stdout.write("\nwritten: %r\n" % local_filename)
@staticmethod
def commit(paths, message):

64
packer/packer.py
View File

@@ -24,7 +24,9 @@ TIMEIT = True
def pack(blendfile_src, blendfile_dst, mode='FILE',
deps_remap=None, paths_remap=None, paths_uuid=None):
deps_remap=None, paths_remap=None, paths_uuid=None,
# yield reports
report=None):
"""
:param deps_remap: Store path deps_remap info as follows.
{"file.blend": {"path_new": "path_old", ...}, ...}
@@ -54,6 +56,11 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
# TODO, make configurable
WRITE_JSON_REMAP = True
if report is None:
raise Exception("report not set!")
yield report("%s: %r...\n" % (colorize("\nscanning deps", color='bright_green'), blendfile_src))
if TIMEIT:
import time
t = time.time()
@@ -77,6 +84,7 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
path_temp_files.add(filepath_tmp)
return filepath_tmp
# base_dir_src = os.path.dirname(blendfile_src)
base_dir_dst = os.path.dirname(blendfile_dst)
@@ -85,6 +93,7 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
os.makedirs(base_dir_dst_subdir)
lib_visit = {}
_last = b''
for fp, (rootdir, fp_blend_basename) in blendfile_path_walker.FilePath.visit_from_blend(
blendfile_src,
@@ -94,6 +103,10 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
lib_visit=lib_visit,
):
if _last != fp_blend_basename:
yield report(" %s: %s\n" % (colorize("blend", color='blue'), fp.basedir + fp_blend_basename))
_last = fp_blend_basename
# assume the path might be relative
path_src_orig = fp.filepath
path_rel = blendfile_path_walker.utils.compatpath(path_src_orig)
@@ -126,6 +139,9 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
if TIMEIT:
print(" Time: %.4f\n" % (time.time() - t))
yield report(("%s: %d files\n") %
(colorize("\narchiving", color='bright_green'), len(path_copy_files) + 1))
# handle deps_remap and file renaming
if deps_remap is not None:
blendfile_src_basename = os.path.basename(blendfile_src).decode('utf-8')
@@ -183,19 +199,18 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
assert(b'.blend' not in dst)
if not os.path.exists(src):
print(" Source missing! %r" % src)
yield report(" %s: %r\n" % (colorize("source missing", color='red'), src))
else:
print(" Copying %r -> %r" % (src, dst))
yield report(" %s: %r -> %r\n" % (colorize("copying", color='blue'), src, dst))
shutil.copy(src, dst)
print(" Written:", blendfile_dst)
yield report(" %s: %r\n" % (colorize("written", color='green'), blendfile_dst))
elif mode == 'ZIP':
import zipfile
with zipfile.ZipFile(blendfile_dst.decode('utf-8'), 'w', zipfile.ZIP_DEFLATED) as zip:
for fn in path_temp_files:
print(" Copying %r -> <zip>" % fn)
yield report(" %s: %r -> <archive>\n" % (colorize("copying", color='blue'), fn))
zip.write(fn.decode('utf-8'),
arcname=os.path.relpath(fn[:-1], base_dir_dst).decode('utf-8'))
os.remove(fn)
@@ -206,9 +221,9 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
assert(b'.blend' not in dst)
if not os.path.exists(src):
print(" Source missing! %r" % src)
yield report(" %s: %r\n" % (colorize("source missing", color='red'), src))
else:
print(" Copying %r -> <zip>" % src)
yield report(" %s: %r -> <archive>\n" % (colorize("copying", color='blue'), src))
zip.write(src.decode('utf-8'),
arcname=os.path.relpath(dst, base_dir_dst).decode('utf-8'))
@@ -233,7 +248,7 @@ def pack(blendfile_src, blendfile_dst, mode='FILE',
del write_dict_as_json
print(" Written:", blendfile_dst)
yield report(" %s: %r\n" % (colorize("written", color='green'), blendfile_dst))
else:
raise Exception("%s not a known mode" % mode)
@@ -309,3 +324,34 @@ def main():
if __name__ == "__main__":
main()
# TODO(cam) de-duplicate
USE_COLOR = True
if USE_COLOR:
color_codes = {
'black': '\033[0;30m',
'bright_gray': '\033[0;37m',
'blue': '\033[0;34m',
'white': '\033[1;37m',
'green': '\033[0;32m',
'bright_blue': '\033[1;34m',
'cyan': '\033[0;36m',
'bright_green': '\033[1;32m',
'red': '\033[0;31m',
'bright_cyan': '\033[1;36m',
'purple': '\033[0;35m',
'bright_red': '\033[1;31m',
'yellow': '\033[0;33m',
'bright_purple':'\033[1;35m',
'dark_gray': '\033[1;30m',
'bright_yellow':'\033[1;33m',
'normal': '\033[0m',
}
def colorize(msg, color=None):
return (color_codes[color] + msg + color_codes['normal'])
else:
def colorize(msg, color=None):
return msg

82
webservice/bam/application/__init__.py
View File

@@ -150,16 +150,41 @@ class FileAPI(Resource):
elif os.path.isdir(filepath):
return jsonify(message="Path is a directory %r" % filepath)
# pack the file!
print("PACKING")
filepath_zip = self.pack_fn(filepath)
def response_message_iter():
ID_MESSAGE = 1
ID_PAYLOAD = 2
import struct
# TODO, handle fail
if filepath_zip is None:
return jsonify(message="Path not found %r" % filepath)
def report(txt):
txt_bytes = txt.encode('utf-8')
return struct.pack('<II', ID_MESSAGE, len(txt_bytes)) + txt_bytes
f = open(filepath_zip, 'rb')
return Response(f, direct_passthrough=True)
yield b'BAM\0'
# pack the file!
import tempfile
filepath_zip = tempfile.mkstemp(suffix=".zip")
yield from self.pack_fn(filepath, filepath_zip, report)
# TODO, handle fail
if not os.path.exists(filepath_zip[-1]):
yield report("%s: %r\n" % (colorize("failed to extract", color='red'), filepath))
return
with open(filepath_zip[-1], 'rb') as f:
f.seek(0, os.SEEK_END)
f_size = f.tell()
f.seek(0, os.SEEK_SET)
yield struct.pack('<II', ID_PAYLOAD, f_size)
while True:
data = f.read(1024)
if not data:
break
yield data
# return Response(f, direct_passthrough=True)
return Response(response_message_iter(), direct_passthrough=True)
else:
return jsonify(message="Command unknown")
@@ -214,7 +239,7 @@ class FileAPI(Resource):
return jsonify(message='File not allowed')
@staticmethod
def pack_fn(filepath):
def pack_fn(filepath, filepath_zip, report):
import os
assert(os.path.exists(filepath) and not os.path.isdir(filepath))
@@ -233,17 +258,17 @@ class FileAPI(Resource):
sys.path.append(modpath)
del modpath
import tempfile
import packer
filepath_zip = tempfile.mkstemp(suffix=".zip")
print(" Source path:", filepath)
print(" Zip path:", filepath_zip)
try:
packer.pack(filepath.encode('utf-8'), filepath_zip[-1].encode('utf-8'), mode='ZIP',
# TODO(cam) this just means the json is written in the zip
deps_remap={}, paths_remap={}, paths_uuid={})
yield from packer.pack(
filepath.encode('utf-8'), filepath_zip[-1].encode('utf-8'), mode='ZIP',
# TODO(cam) this just means the json is written in the zip
deps_remap={}, paths_remap={}, paths_uuid={},
report=report)
return filepath_zip[-1]
except:
import traceback
@@ -259,3 +284,32 @@ class FileAPI(Resource):
api.add_resource(FilesListAPI, '/file_list', endpoint='file_list')
api.add_resource(FileAPI, '/file', endpoint='file')
USE_COLOR = True
if USE_COLOR:
color_codes = {
'black': '\033[0;30m',
'bright_gray': '\033[0;37m',
'blue': '\033[0;34m',
'white': '\033[1;37m',
'green': '\033[0;32m',
'bright_blue': '\033[1;34m',
'cyan': '\033[0;36m',
'bright_green': '\033[1;32m',
'red': '\033[0;31m',
'bright_cyan': '\033[1;36m',
'purple': '\033[0;35m',
'bright_red': '\033[1;31m',
'yellow': '\033[0;33m',
'bright_purple':'\033[1;35m',
'dark_gray': '\033[1;30m',
'bright_yellow':'\033[1;33m',
'normal': '\033[0m',
}
def colorize(msg, color=None):
return (color_codes[color] + msg + color_codes['normal'])
else:
def colorize(msg, color=None):
return msg