From 262a0e14e1e56e50f16ec5853a47eabd919a663c Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:49 -0500 Subject: [PATCH 01/17] remap: refactor hash_map scripts into single script - Prompt user for directories to create env file with - Promt user to update existing map or start fresh - Save new bbatch script for user with hard coded json_file_path --- scripts/remap/hash_map_make.py | 44 -------- scripts/remap/hash_map_update.py | 52 --------- scripts/remap/remap.py | 186 +++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 96 deletions(-) delete mode 100644 scripts/remap/hash_map_make.py delete mode 100644 scripts/remap/hash_map_update.py create mode 100644 scripts/remap/remap.py diff --git a/scripts/remap/hash_map_make.py b/scripts/remap/hash_map_make.py deleted file mode 100644 index a3848756..00000000 --- a/scripts/remap/hash_map_make.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 - -import os -import hashlib -import json - -json_file_path = "" # File Path to write JSON File to -folder_path = "" # Create Map for items in this folder recursively - - -def generate_checksum(filepath: str) -> str: - """ - Generate a checksum for a zip file - Arguments: - archive_path: String of the archive's file path - Returns: - sha256 checksum for the provided archive as string - """ - with open(filepath, "rb") as f: - digest = hashlib.file_digest(f, "sha256") - return digest.hexdigest() - - -def generate_json_for_directory(directory_path): - data = {} - - for root, _, files in os.walk(directory_path): - for file_name in files: - file_path = os.path.join(root, file_name) - sha256 = generate_checksum(file_path) - - if sha256 in data: - data[sha256]['old'].append(file_path) - else: - data[sha256] = {'old': [file_path], 'new': ''} - print(f"Making hash for {file_name}") - - with open(json_file_path, 'w') as json_file: - json.dump(data, json_file, indent=4) - - -if __name__ == "__main__": - directory_path = folder_path - generate_json_for_directory(directory_path) diff --git a/scripts/remap/hash_map_update.py b/scripts/remap/hash_map_update.py deleted file mode 100644 index 7d1f9b61..00000000 --- a/scripts/remap/hash_map_update.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 - -import os -import hashlib -import json -from pathlib import Path - -json_file_path = "" # File Path to read/write JSON File to -folder_path = "" # Create Map for items in this folder recursively - - -gold_file_map_json = Path(json_file_path) -gold_file_map_data = open(gold_file_map_json) -gold_file_map_dict = json.load(gold_file_map_data) - - -def generate_checksum(filepath: str) -> str: - """ - Generate a checksum for a zip file - Arguments: - archive_path: String of the archive's file path - Returns: - sha256 checksum for the provided archive as string - """ - with open(filepath, "rb") as f: - digest = hashlib.file_digest(f, "sha256") - return digest.hexdigest() - - -def generate_json_for_directory(directory_path): - data = gold_file_map_dict - - for root, _, files in os.walk(directory_path): - for file_name in files: - file_path = os.path.join(root, file_name) - sha256 = generate_checksum(file_path) - - if not data.get(sha256): - print(f"Cannot find file in dict {file_path}") - continue - - if sha256 in data: - data[sha256]['new'] = file_path - print(f"Updating path for {file_path}") - - with open(json_file_path, 'w') as json_file: - json.dump(data, json_file, indent=4) - - -if __name__ == "__main__": - directory_path = folder_path - generate_json_for_directory(directory_path) diff --git a/scripts/remap/remap.py b/scripts/remap/remap.py new file mode 100644 index 00000000..10334112 --- /dev/null +++ b/scripts/remap/remap.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import hashlib +import json +import os + +JSON_FILE_KEY = 'json_file_path' +CRAWL_DIR_KEY = 'folder_path' + + +def get_current_dir(): + return Path(__file__).parent.resolve() + + +def get_variable_file_path(): + directory = get_current_dir() + return directory.joinpath(".env") + + +def get_variable_file(): + env_file = get_variable_file_path() + if env_file.exists(): + return env_file + + +def remove_variable_file(): + env_file = get_variable_file_path() + Path.unlink(env_file) + + +def get_variables(): + # TODO FIX Saving JSON because os.environ.get() doesn't work when running this as a script + var_file = Path(get_variable_file()) + var_file_data = open(var_file) + var_file_dict = json.load(var_file_data) + + return var_file_dict + + +def set_variable_file(): + file_path = get_variable_file_path() + variables = {} + + dir_to_map = get_dir_to_map() + json_file = get_josn_file_dir() + + variables[JSON_FILE_KEY] = f"{json_file}" + variables[CRAWL_DIR_KEY] = f"{dir_to_map}" + + with open(file_path, 'w') as json_file: + json.dump(variables, json_file, indent=4) + return variables + + +def get_dir_to_map() -> str: + dir_to_map = input("Please enter directory to map: ") + if not Path(dir_to_map).is_dir(): + raise Exception('Provided path is not a directory') + return dir_to_map + + +def get_josn_file_dir() -> str: + json_dir = input("Please enter directory to store JSON map: ") + if not Path(json_dir).is_dir(): + raise Exception('Provided path is not a directory') + return Path(json_dir).joinpath("dir_map.json").__str__() + + +def get_bbatch_script_dir() -> Path: + bbatch_dir = input("Please select directory to save bbatch script to: ") + if not Path(bbatch_dir).is_dir(): + raise Exception('Provided path is not a directory') + return Path(bbatch_dir) + + +def save_bbatch_script(): + directory = get_current_dir() + file = directory.joinpath('remap_blender_paths.py') + variables = get_variables() + new_file_dir = get_bbatch_script_dir() + + with open(file, 'r', encoding='utf-8') as file: + data = file.readlines() + for line in data: + if line.startswith('json_file_path ='): + data[data.index(line)] = f'json_file_path = "{variables[JSON_FILE_KEY]}"' + mapped_dir_name = Path(variables[CRAWL_DIR_KEY]).name + new_file = new_file_dir.joinpath(f'remap_blender_paths_for_{mapped_dir_name}.py') + + with open(new_file, 'w', encoding='utf-8') as file: + file.writelines(data) + print(f"bbatch script has been saved to {new_file.__str__()}") + print("This is your bbatch command") + print( + f'python -m bbatch {variables[CRAWL_DIR_KEY]} --script {new_file.__str__()} --nosave --recursive' + ) + + +def generate_checksum(filepath: str) -> str: + """ + Generate a checksum for a zip file + Arguments: + archive_path: String of the archive's file path + Returns: + sha256 checksum for the provided archive as string + """ + with open(filepath, "rb") as f: + digest = hashlib.file_digest(f, "sha256") + return digest.hexdigest() + + +def generate_json_for_directory(directory_path, json_file_path): + # TODO Centralize duplicate code from 'update_json_for_directory()' + data = {} + + for root, _, files in os.walk(directory_path): + for file_name in files: + file_path = os.path.join(root, file_name) + sha256 = generate_checksum(file_path) + + if sha256 in data: + data[sha256]['old'].append(file_path) + else: + data[sha256] = {'old': [file_path], 'new': ''} + print(f"Making hash for {file_name}") + + with open(json_file_path, 'w') as json_file: + json.dump(data, json_file, indent=4) + + +def update_json_for_directory(directory_path, json_file_path): + file_map_json = Path(json_file_path) + file_map_data = open(file_map_json) + file_map_dict = json.load(file_map_data) + + data = file_map_dict + + for root, _, files in os.walk(directory_path): + for file_name in files: + file_path = os.path.join(root, file_name) + sha256 = generate_checksum(file_path) + + if not data.get(sha256): + print(f"Cannot find file in dict {file_path}") + continue + + if sha256 in data: + data[sha256]['new'] = file_path + print(f"Updating path for {file_path}") + + with open(json_file_path, 'w') as json_file: + json.dump(data, json_file, indent=4) + + +def main(): + print("Welcome to 'remap' a tool to assist in a re-organization of folders") + if not get_variable_file(): + print("Starting new remap session") + variables = set_variable_file() + print(f"Generating map for directory '{variables[CRAWL_DIR_KEY]}'") + generate_json_for_directory(variables[CRAWL_DIR_KEY], variables[JSON_FILE_KEY]) + print( + f"Directory '{variables[CRAWL_DIR_KEY]}' can now be re-organized before updating the map" + ) + + else: + variables = get_variables() + answer = input(f"Update map for dir '{variables[CRAWL_DIR_KEY]}' yes or no: ") + answer_lw = answer.lower() + if answer_lw == "yes" or answer_lw == 'y': + print(f"Updating map for directory '{variables[CRAWL_DIR_KEY]}'") + update_json_for_directory( + variables[CRAWL_DIR_KEY], variables[JSON_FILE_KEY] + ) + print('Map update is complete') + save_bbatch_script() + elif answer_lw == "no" or answer_lw == 'n': + remove_variable_file() + main() + else: + print("Please enter yes or no.") + + +if __name__ == "__main__": + main() -- 2.30.2 From 11848efcae41daac79a2d129cba52ac92e40ffa7 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:49 -0500 Subject: [PATCH 02/17] remap: Save bbatch script in DIST folder --- scripts/remap/remap.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/scripts/remap/remap.py b/scripts/remap/remap.py index 10334112..255f3700 100644 --- a/scripts/remap/remap.py +++ b/scripts/remap/remap.py @@ -67,33 +67,26 @@ def get_josn_file_dir() -> str: return Path(json_dir).joinpath("dir_map.json").__str__() -def get_bbatch_script_dir() -> Path: - bbatch_dir = input("Please select directory to save bbatch script to: ") - if not Path(bbatch_dir).is_dir(): - raise Exception('Provided path is not a directory') - return Path(bbatch_dir) - - def save_bbatch_script(): directory = get_current_dir() - file = directory.joinpath('remap_blender_paths.py') + source_file = directory.joinpath('remap_blender_paths.py') variables = get_variables() - new_file_dir = get_bbatch_script_dir() + new_file_dir = directory.joinpath('dist') - with open(file, 'r', encoding='utf-8') as file: - data = file.readlines() + with open(source_file, 'r', encoding='utf-8') as source_file: + data = source_file.readlines() for line in data: if line.startswith('json_file_path ='): data[data.index(line)] = f'json_file_path = "{variables[JSON_FILE_KEY]}"' mapped_dir_name = Path(variables[CRAWL_DIR_KEY]).name new_file = new_file_dir.joinpath(f'remap_blender_paths_for_{mapped_dir_name}.py') - with open(new_file, 'w', encoding='utf-8') as file: - file.writelines(data) + with open(new_file, 'w', encoding='utf-8') as source_file: + source_file.writelines(data) print(f"bbatch script has been saved to {new_file.__str__()}") print("This is your bbatch command") print( - f'python -m bbatch {variables[CRAWL_DIR_KEY]} --script {new_file.__str__()} --nosave --recursive' + f'python -m bbatch {variables[CRAWL_DIR_KEY]} --script {new_file.absolute().__str__()} --nosave --recursive' ) -- 2.30.2 From 90784b6b588dbef9ebfd308ea2f2e4492b4f4882 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:49 -0500 Subject: [PATCH 03/17] remap: recursively get directories --- scripts/remap/remap.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/remap/remap.py b/scripts/remap/remap.py index 255f3700..0f99569e 100644 --- a/scripts/remap/remap.py +++ b/scripts/remap/remap.py @@ -53,17 +53,25 @@ def set_variable_file(): return variables -def get_dir_to_map() -> str: - dir_to_map = input("Please enter directory to map: ") - if not Path(dir_to_map).is_dir(): +def get_dir_recursively(prompt: str): + iterations = 0 + dir_to_map = input(prompt) + if Path(dir_to_map).is_dir(): + iterations += 1 + return dir_to_map + if iterations > 10: raise Exception('Provided path is not a directory') - return dir_to_map + else: + print('Provided path is not a directory') + return get_dir_recursively(prompt) + + +def get_dir_to_map() -> str: + return get_dir_recursively("Please enter directory to map: ") def get_josn_file_dir() -> str: - json_dir = input("Please enter directory to store JSON map: ") - if not Path(json_dir).is_dir(): - raise Exception('Provided path is not a directory') + json_dir = get_dir_recursively("Please enter directory to store JSON map: ") return Path(json_dir).joinpath("dir_map.json").__str__() -- 2.30.2 From c31bf00cb47b9f6b95489c2abb9e9d0302c0d458 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:49 -0500 Subject: [PATCH 04/17] remap: Update Print Statement --- scripts/remap/remap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/remap/remap.py b/scripts/remap/remap.py index 0f99569e..970183ed 100644 --- a/scripts/remap/remap.py +++ b/scripts/remap/remap.py @@ -167,7 +167,9 @@ def main(): else: variables = get_variables() - answer = input(f"Update map for dir '{variables[CRAWL_DIR_KEY]}' yes or no: ") + answer = input( + f"Continune with existing session to update map for dir '{variables[CRAWL_DIR_KEY]}'? yes or no: " + ) answer_lw = answer.lower() if answer_lw == "yes" or answer_lw == 'y': print(f"Updating map for directory '{variables[CRAWL_DIR_KEY]}'") -- 2.30.2 From 2c881ccf420c632c9d7b63961618e69a510d31bb Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:49 -0500 Subject: [PATCH 05/17] remap: Fix bug where welcome message appears twice --- scripts/remap/remap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/remap/remap.py b/scripts/remap/remap.py index 970183ed..a52ef303 100644 --- a/scripts/remap/remap.py +++ b/scripts/remap/remap.py @@ -155,14 +155,13 @@ def update_json_for_directory(directory_path, json_file_path): def main(): - print("Welcome to 'remap' a tool to assist in a re-organization of folders") if not get_variable_file(): print("Starting new remap session") variables = set_variable_file() print(f"Generating map for directory '{variables[CRAWL_DIR_KEY]}'") generate_json_for_directory(variables[CRAWL_DIR_KEY], variables[JSON_FILE_KEY]) print( - f"Directory '{variables[CRAWL_DIR_KEY]}' can now be re-organized before updating the map" + f"Directory '{variables[CRAWL_DIR_KEY]}' can now be re-organized before re-running this tool to update it's map" ) else: @@ -186,4 +185,5 @@ def main(): if __name__ == "__main__": + print("Welcome to 'remap' a tool to assist in a re-organization of folders") main() -- 2.30.2 From e9f5a6399566250cab361929dd389436737cb598 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:49 -0500 Subject: [PATCH 06/17] remap: Fix typo --- scripts/remap/remap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/remap/remap.py b/scripts/remap/remap.py index a52ef303..7b6b431e 100644 --- a/scripts/remap/remap.py +++ b/scripts/remap/remap.py @@ -185,5 +185,5 @@ def main(): if __name__ == "__main__": - print("Welcome to 'remap' a tool to assist in a re-organization of folders") + print("Welcome to 'remap', a tool to assist in a re-organization of folders") main() -- 2.30.2 From 77d4185c2ea8a968fefd97646fab8d7d4c0fae11 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:49 -0500 Subject: [PATCH 07/17] remap: Update README --- scripts/remap/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/remap/README.md b/scripts/remap/README.md index 5e083886..4031c76f 100644 --- a/scripts/remap/README.md +++ b/scripts/remap/README.md @@ -3,8 +3,9 @@ This directory contains scripts that are useful for re-organizing production directories. The tools are intended to be used when some directories need to be changed, and references to these directories need to be updated in .blend files. ## Usage -1. Set the variable `json_file_path` to match in all script files. Set `folder_path` in both has_map script files. -2. Run `hash_map_make.py` to create a JSON file listing every file in directory via hash, plus a list directories leading to that file (duplicate files included). -3. Re-organize/rename items in the directory you have made a map for. -4. Run `hash_map_update.py` to find the new locations of these files using the Hash to match them up. This will add a `new` directory for each hash. -5. Using [`bbatch`](https://projects.blender.org/studio/blender-studio-pipeline/src/branch/main/scripts/bbatch/README.md) run the script `remap_blender_paths.py` to update references in .blend files from the old path to the new path. \ No newline at end of file +1. Enter remap directory `cd blender-studio-pipeline/scripts/remap` +2. Run the remap tool via `python -m remap`. You will be prompted for a directory to map, and a location to store the map (outside of your remap directory). +3. Now you are ready to re-organize your mapped directory, move files into different folders, rename files and remove duplicates. +4. Re-run the remap tool via `python -m remap` to update your map with the new file locations. The tool will print a bbatch, copy this for use in step 6. +5. Enter bbatch directory `cd blender-studio-pipeline/scripts/bbatch` +6. Run provided bbatch command, similar to `python -m bbatch /my_files/ --script /{path_to_script}/remap_blender_paths_for_my_files.py --nosave --recursive` to update all references to the remapped directory contents in your .blend files. -- 2.30.2 From d7721967cc59c286f40b49f8ed0b08c1bddb7fb0 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:49 -0500 Subject: [PATCH 08/17] remap: Add future TODO file --- scripts/remap/todo.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 scripts/remap/todo.txt diff --git a/scripts/remap/todo.txt b/scripts/remap/todo.txt new file mode 100644 index 00000000..3b76bf78 --- /dev/null +++ b/scripts/remap/todo.txt @@ -0,0 +1,4 @@ +# Remap TODO + +## Features +- [ ] Support mapping multiple directories at once \ No newline at end of file -- 2.30.2 From 720cf71c96a25d2b47c1a6854ff07572cfd83465 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:49 -0500 Subject: [PATCH 09/17] remap: Store variables in JSON file --- scripts/remap/remap.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/remap/remap.py b/scripts/remap/remap.py index 7b6b431e..62665963 100644 --- a/scripts/remap/remap.py +++ b/scripts/remap/remap.py @@ -15,7 +15,10 @@ def get_current_dir(): def get_variable_file_path(): directory = get_current_dir() - return directory.joinpath(".env") + variables_dir = directory.joinpath("var") + if not variables_dir.exists(): + variables_dir.mkdir() + return variables_dir.joinpath("remap_variables.json") def get_variable_file(): @@ -30,7 +33,6 @@ def remove_variable_file(): def get_variables(): - # TODO FIX Saving JSON because os.environ.get() doesn't work when running this as a script var_file = Path(get_variable_file()) var_file_data = open(var_file) var_file_dict = json.load(var_file_data) -- 2.30.2 From 226aa801b75801a1aef5e3f24942bef33f1e5506 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:49 -0500 Subject: [PATCH 10/17] remap: Cleanup `remap_blender_paths.py` --- scripts/remap/remap_blender_paths.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/remap/remap_blender_paths.py b/scripts/remap/remap_blender_paths.py index 0505f300..5547aff2 100644 --- a/scripts/remap/remap_blender_paths.py +++ b/scripts/remap/remap_blender_paths.py @@ -9,9 +9,9 @@ file_updated = False json_file_path = "" # File Path to read/write JSON File to -gold_file_map_json = Path(json_file_path) -gold_file_map_data = open(gold_file_map_json) -gold_file_map_dict = json.load(gold_file_map_data) +file_map_json = Path(json_file_path) +file_map_data = open(file_map_json) +file_map_dict = json.load(file_map_data) @contextlib.contextmanager @@ -56,12 +56,12 @@ def generate_checksum(filepath: str) -> str: def find_new_from_old(old_path): - for _, value in gold_file_map_dict.items(): + for _, value in file_map_dict.items(): for old_json_path in value['old']: if old_json_path.endswith(old_path.split("/..")[-1]): if value['new'] != old_json_path: return value['new'] - for _, value in gold_file_map_dict.items(): + for _, value in file_map_dict.items(): for old_json_path in value['old']: if old_json_path.endswith(old_path.split("/")[-1]): if value['new'] != old_json_path: -- 2.30.2 From 15134c1ea303b34376367937ff1e6097218baad7 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:50 -0500 Subject: [PATCH 11/17] remap: Update TODO --- scripts/remap/todo.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/remap/todo.txt b/scripts/remap/todo.txt index 3b76bf78..86bd0dc6 100644 --- a/scripts/remap/todo.txt +++ b/scripts/remap/todo.txt @@ -1,4 +1,6 @@ # Remap TODO ## Features +- [ ] Pass JSON file path as argument to `remap_blender_paths.py` + - [ ] Update bbatch to support this to: `just have the --factory-startup not be last and have the -P, script line last` - [ ] Support mapping multiple directories at once \ No newline at end of file -- 2.30.2 From 38e3737dd8ca9e9799c55b810f48dffb67143011 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:50 -0500 Subject: [PATCH 12/17] remap: Pass JSON path to bbatch script via parse_args --- scripts/remap/remap_blender_paths.py | 41 ++++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/scripts/remap/remap_blender_paths.py b/scripts/remap/remap_blender_paths.py index 5547aff2..1356f3bb 100644 --- a/scripts/remap/remap_blender_paths.py +++ b/scripts/remap/remap_blender_paths.py @@ -1,3 +1,4 @@ +import argparse import bpy from pathlib import Path import json @@ -7,11 +8,13 @@ import contextlib file_updated = False -json_file_path = "" # File Path to read/write JSON File to - -file_map_json = Path(json_file_path) -file_map_data = open(file_map_json) -file_map_dict = json.load(file_map_data) +# Command line arguments. +parser = argparse.ArgumentParser() +parser.add_argument( + "path", + help="Path to JSON file containing map", + type=str, +) @contextlib.contextmanager @@ -55,7 +58,7 @@ def generate_checksum(filepath: str) -> str: return sha256.hexdigest() -def find_new_from_old(old_path): +def find_new_from_old(old_path: str, file_map_dict: dict) -> str: for _, value in file_map_dict.items(): for old_json_path in value['old']: if old_json_path.endswith(old_path.split("/..")[-1]): @@ -68,7 +71,7 @@ def find_new_from_old(old_path): return value['new'] -def update_vse_references(): +def update_vse_references(file_map_dict: dict) -> None: global file_updated for scn in bpy.data.scenes: if not scn.sequence_editor: @@ -77,7 +80,7 @@ def update_vse_references(): for path in paths_for_vse_strip(strip): if path == "": continue - new_path = find_new_from_old(path) + new_path = find_new_from_old(path, file_map_dict) if not new_path: print(f"No new path for '{strip.name}' at '{path}' ") continue @@ -106,21 +109,21 @@ def update_vse_references(): file_updated = True -def update_referenced_images(): +def update_referenced_images(file_map_dict: dict) -> None: global file_updated for img in bpy.data.images: if img.filepath is not None and img.filepath != "": - new_path = find_new_from_old(img.filepath) + new_path = find_new_from_old(img.filepath, file_map_dict) if new_path: print(f"Remapping Image Datablock {img.filepath }") img.filepath = new_path file_updated = True -def update_libs(): +def update_libs(file_map_dict: dict) -> None: global file_updated for lib in bpy.data.libraries: - new_path = find_new_from_old(lib.filepath) + new_path = find_new_from_old(lib.filepath, file_map_dict) if new_path: lib.filepath = new_path print(f"Remapping {lib.filepath}") @@ -129,9 +132,17 @@ def update_libs(): def remap_all_blender_paths(): start = time.time() - update_vse_references() - update_referenced_images() - update_libs() + + args = parser.parse_args() + json_file_path = args.path # File Path to read/write JSON File to + + file_map_json = Path(json_file_path) + file_map_data = open(file_map_json) + file_map_dict = json.load(file_map_data) + + update_vse_references(file_map_dict) + update_referenced_images(file_map_dict) + update_libs(file_map_dict) bpy.ops.file.make_paths_relative() end = time.time() -- 2.30.2 From 48a4a9fa2f3c10cf999fcdf9c1217a594082cb7b Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:50 -0500 Subject: [PATCH 13/17] remap: Update documentation in bbatch script --- scripts/remap/remap_blender_paths.py | 56 +++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/scripts/remap/remap_blender_paths.py b/scripts/remap/remap_blender_paths.py index 1356f3bb..38f336ba 100644 --- a/scripts/remap/remap_blender_paths.py +++ b/scripts/remap/remap_blender_paths.py @@ -5,6 +5,7 @@ import json import hashlib import time import contextlib +from typing import List file_updated = False @@ -30,7 +31,16 @@ def override_save_version(): bpy.context.preferences.filepaths.save_version = save_version -def paths_for_vse_strip(strip): +def paths_for_vse_strip(strip: bpy.types.Sequence) -> List[str]: + """Returns all paths related to Movie, Image and Sound strips + in Blender's Video Sequence Editor + + Args: + strip (bpy.types.Sequence): Movie, Image or Sounds Strip + + Returns: + List[str]: List of all paths related to strip + """ if hasattr(strip, "filepath"): return [strip.filepath] if hasattr(strip, "directory"): @@ -41,9 +51,9 @@ def paths_for_vse_strip(strip): def generate_checksum(filepath: str) -> str: - """ - Generate a checksum for a zip file - Arguments: + """Generate a checksum for a zip file + + Args: archive_path: String of the archive's file path Returns: sha256 checksum for the provided archive as string @@ -58,20 +68,37 @@ def generate_checksum(filepath: str) -> str: return sha256.hexdigest() -def find_new_from_old(old_path: str, file_map_dict: dict) -> str: +def dict_find_new_from_old(old_path: str, file_map_dict: dict) -> str: + """Returns the matching 'new' filepath stored in file_map_dict + using the 'old' filepath. + + Args: + old_path (str): 'old' filepath referencing a file from Blender + file_map_dict (dict): Dictionary of 'old' and 'new' paths + + Returns: + str: 'new' filepath to replace the 'old' filepath + """ for _, value in file_map_dict.items(): for old_json_path in value['old']: + # Match paths using the filepath stored in Blender if old_json_path.endswith(old_path.split("/..")[-1]): if value['new'] != old_json_path: return value['new'] for _, value in file_map_dict.items(): for old_json_path in value['old']: + # Match paths using filename only if old_json_path.endswith(old_path.split("/")[-1]): if value['new'] != old_json_path: return value['new'] def update_vse_references(file_map_dict: dict) -> None: + """Update file references for VSE strips + + Args: + file_map_dict (dict): Dictionary of 'old' and 'new' paths + """ global file_updated for scn in bpy.data.scenes: if not scn.sequence_editor: @@ -80,7 +107,7 @@ def update_vse_references(file_map_dict: dict) -> None: for path in paths_for_vse_strip(strip): if path == "": continue - new_path = find_new_from_old(path, file_map_dict) + new_path = dict_find_new_from_old(path, file_map_dict) if not new_path: print(f"No new path for '{strip.name}' at '{path}' ") continue @@ -110,10 +137,15 @@ def update_vse_references(file_map_dict: dict) -> None: def update_referenced_images(file_map_dict: dict) -> None: + """Update file references for Image data-blocks + + Args: + file_map_dict (dict): Dictionary of 'old' and 'new' paths + """ global file_updated for img in bpy.data.images: if img.filepath is not None and img.filepath != "": - new_path = find_new_from_old(img.filepath, file_map_dict) + new_path = dict_find_new_from_old(img.filepath, file_map_dict) if new_path: print(f"Remapping Image Datablock {img.filepath }") img.filepath = new_path @@ -121,9 +153,14 @@ def update_referenced_images(file_map_dict: dict) -> None: def update_libs(file_map_dict: dict) -> None: + """Update file references for libraries (linked/appended data) + + Args: + file_map_dict (dict): Dictionary of 'old' and 'new' paths + """ global file_updated for lib in bpy.data.libraries: - new_path = find_new_from_old(lib.filepath, file_map_dict) + new_path = dict_find_new_from_old(lib.filepath, file_map_dict) if new_path: lib.filepath = new_path print(f"Remapping {lib.filepath}") @@ -131,10 +168,11 @@ def update_libs(file_map_dict: dict) -> None: def remap_all_blender_paths(): + """Remap all references to files from blender via dictionary""" start = time.time() args = parser.parse_args() - json_file_path = args.path # File Path to read/write JSON File to + json_file_path = args.path file_map_json = Path(json_file_path) file_map_data = open(file_map_json) -- 2.30.2 From 5f958ce5359081c087c184aff3d001198f332a35 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:50 -0500 Subject: [PATCH 14/17] remap: Pass JSON as argument to `remap_blender_paths.py` --- scripts/remap/remap.py | 23 ++++++----------------- scripts/remap/remap_blender_paths.py | 16 +++++----------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/scripts/remap/remap.py b/scripts/remap/remap.py index 62665963..31893040 100644 --- a/scripts/remap/remap.py +++ b/scripts/remap/remap.py @@ -77,26 +77,15 @@ def get_josn_file_dir() -> str: return Path(json_dir).joinpath("dir_map.json").__str__() -def save_bbatch_script(): +def get_bbatch_command(): directory = get_current_dir() source_file = directory.joinpath('remap_blender_paths.py') variables = get_variables() - new_file_dir = directory.joinpath('dist') - - with open(source_file, 'r', encoding='utf-8') as source_file: - data = source_file.readlines() - for line in data: - if line.startswith('json_file_path ='): - data[data.index(line)] = f'json_file_path = "{variables[JSON_FILE_KEY]}"' - mapped_dir_name = Path(variables[CRAWL_DIR_KEY]).name - new_file = new_file_dir.joinpath(f'remap_blender_paths_for_{mapped_dir_name}.py') - - with open(new_file, 'w', encoding='utf-8') as source_file: - source_file.writelines(data) - print(f"bbatch script has been saved to {new_file.__str__()}") - print("This is your bbatch command") print( - f'python -m bbatch {variables[CRAWL_DIR_KEY]} --script {new_file.absolute().__str__()} --nosave --recursive' + "To update your .blend file references open the bbatch directory and run the following command" + ) + print( + f'python -m bbatch {variables[CRAWL_DIR_KEY]} --nosave --recursive --script {source_file} --args {variables[JSON_FILE_KEY]}' ) @@ -178,7 +167,7 @@ def main(): variables[CRAWL_DIR_KEY], variables[JSON_FILE_KEY] ) print('Map update is complete') - save_bbatch_script() + get_bbatch_command() elif answer_lw == "no" or answer_lw == 'n': remove_variable_file() main() diff --git a/scripts/remap/remap_blender_paths.py b/scripts/remap/remap_blender_paths.py index 38f336ba..f3684504 100644 --- a/scripts/remap/remap_blender_paths.py +++ b/scripts/remap/remap_blender_paths.py @@ -1,4 +1,4 @@ -import argparse +import sys import bpy from pathlib import Path import json @@ -9,14 +9,6 @@ from typing import List file_updated = False -# Command line arguments. -parser = argparse.ArgumentParser() -parser.add_argument( - "path", - help="Path to JSON file containing map", - type=str, -) - @contextlib.contextmanager def override_save_version(): @@ -170,9 +162,11 @@ def update_libs(file_map_dict: dict) -> None: def remap_all_blender_paths(): """Remap all references to files from blender via dictionary""" start = time.time() + import sys - args = parser.parse_args() - json_file_path = args.path + argv = sys.argv + argv = argv[argv.index("--") + 1 :] + json_file_path = argv[0] file_map_json = Path(json_file_path) file_map_data = open(file_map_json) -- 2.30.2 From 42d2c144eff70493977002f0c4ca3fa87158327d Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 15:07:50 -0500 Subject: [PATCH 15/17] remap: Fix Typo in bbatch command --- scripts/remap/remap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/remap/remap.py b/scripts/remap/remap.py index 31893040..e4c5bfd7 100644 --- a/scripts/remap/remap.py +++ b/scripts/remap/remap.py @@ -85,7 +85,7 @@ def get_bbatch_command(): "To update your .blend file references open the bbatch directory and run the following command" ) print( - f'python -m bbatch {variables[CRAWL_DIR_KEY]} --nosave --recursive --script {source_file} --args {variables[JSON_FILE_KEY]}' + f'python -m bbatch {variables[CRAWL_DIR_KEY]} --nosave --recursive --script {source_file} --args "{variables[JSON_FILE_KEY]}"' ) -- 2.30.2 From 43478f540643b4ef4640a61825a563a1845e5470 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Wed, 6 Dec 2023 16:34:32 -0500 Subject: [PATCH 16/17] remap: Clear old TODO --- scripts/remap/todo.txt | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 scripts/remap/todo.txt diff --git a/scripts/remap/todo.txt b/scripts/remap/todo.txt deleted file mode 100644 index 86bd0dc6..00000000 --- a/scripts/remap/todo.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Remap TODO - -## Features -- [ ] Pass JSON file path as argument to `remap_blender_paths.py` - - [ ] Update bbatch to support this to: `just have the --factory-startup not be last and have the -P, script line last` -- [ ] Support mapping multiple directories at once \ No newline at end of file -- 2.30.2 From 104fb27eab223b16d1a2c50e24f240126b1d7c81 Mon Sep 17 00:00:00 2001 From: Nick Alberelli Date: Thu, 7 Dec 2023 10:15:12 -0500 Subject: [PATCH 17/17] remap: Update README --- scripts/remap/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/remap/README.md b/scripts/remap/README.md index 4031c76f..9c79ba66 100644 --- a/scripts/remap/README.md +++ b/scripts/remap/README.md @@ -8,4 +8,4 @@ This directory contains scripts that are useful for re-organizing production dir 3. Now you are ready to re-organize your mapped directory, move files into different folders, rename files and remove duplicates. 4. Re-run the remap tool via `python -m remap` to update your map with the new file locations. The tool will print a bbatch, copy this for use in step 6. 5. Enter bbatch directory `cd blender-studio-pipeline/scripts/bbatch` -6. Run provided bbatch command, similar to `python -m bbatch /my_files/ --script /{path_to_script}/remap_blender_paths_for_my_files.py --nosave --recursive` to update all references to the remapped directory contents in your .blend files. +6. Run provided bbatch command, similar to `python -m bbatch {my_files}/ --nosave --recursive --script {path_to_script}/remap_blender_paths.py --args "{path_to_json}/dir_map.json"` to update all references to the remapped directory contents in your .blend files. -- 2.30.2