diff --git a/scripts/project-tools/consistency_check.py b/scripts/project-tools/consistency_check.py new file mode 100755 index 00000000..b4e6e36e --- /dev/null +++ b/scripts/project-tools/consistency_check.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +import os +import pathlib +import json + + +def create_path_dict(startpath, max_depth): + path_structure_dict = {} + start_folder_name = os.path.basename(start_search_path) + for root, dirs, files in os.walk(startpath, followlinks=True): + # We are only interested in the files and folders inside the start path. + cur_path = root.replace(startpath, start_folder_name) + level = cur_path.count(os.sep) + # Sanity check. We don't expect the directory tree to be too deep. + # Therefore, we will stop if we go too deep. + # This avoids infinite loops that can happen when we follow symlinks + if level > max_depth: + print("We have gone too deep in the file structure, stopping...") + exit(1) + + # Insert the data into the dictionary + nested_dict = path_structure_dict + key_path = cur_path.split(os.sep) + final_key = key_path[-1] + for key in key_path[:-1]: + nested_dict = nested_dict[key] + + files_dict = {} + for f in files: + files_dict[f] = "file" + + nested_dict[final_key] = files_dict + + # Print the files structure to we can see the traversed file tree + indent = ' ' * 4 * (level) + print('{}{}/'.format(indent, os.path.basename(root))) + subindent = ' ' * 4 * (level + 1) + + for f in files: + print('{}{}'.format(subindent, f)) + return path_structure_dict + + +def check_if_structure_is_consistent(cur_path, path_dict, error_list): + for path in path_dict: + # Get next path to check for consistency + next_path = (cur_path / path).resolve() + print("Checking path: %s" % next_path) + if next_path.exists(): + nested_item = path_dict[path] + if type(nested_item) is not dict: + if next_path.is_file(): + continue + else: + # This must be a file, warn if it is not + error_list += ["ERROR: %s is not a file, when it should be!" % next_path] + check_if_structure_is_consistent(next_path, nested_item, error_list) + else: + error_list += ["ERROR: %s doesn't exist!" % next_path] + + +current_file_folder = pathlib.Path(__file__).parent +start_search_path = current_file_folder.parent.parent.resolve() +# path_dict = create_path_dict(str(start_search_path), 5) + +# path_dict pre-generated. This is the stucture the consistency check will ensure is there +path_dict = {} +with open(current_file_folder / "folder_structure.json") as json_file: + path_dict = json.load(json_file) + +# TODO perhaps make a function to pretty print out the path_dict for easier inspection + +error_list = [] +check_if_structure_is_consistent(current_file_folder, path_dict, error_list) + +print() +if len(error_list) == 0: + print("Consistency check: PASSED") + exit(0) +else: + print("Consistency check: FAILED") + print() + for error in error_list: + print(error) +# Exit with error as we didn't pass the consistency check +exit(1) diff --git a/scripts/project-tools/folder_structure.json b/scripts/project-tools/folder_structure.json new file mode 100644 index 00000000..a5498571 --- /dev/null +++ b/scripts/project-tools/folder_structure.json @@ -0,0 +1,23 @@ +{ + "../../": { + "shared": { + "artifacts": {} + }, + "svn": { + "tools": { + "consistency_check.py": "file", + "folder_structure.json": "file", + "install_desktop_file.sh": "file", + "rollback_blender.py": "file", + "run_blender.py": "file", + "update_addons.py": "file", + "update_blender.py": "file" + } + }, + "local": { + "blender": {}, + "scripts": {}, + "config": {} + } + } +} diff --git a/scripts/project-tools/init_project_folder_structure.py b/scripts/project-tools/init_project_folder_structure.py new file mode 100755 index 00000000..ecfd9ca1 --- /dev/null +++ b/scripts/project-tools/init_project_folder_structure.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import argparse +import os +import pathlib +import json +import shutil +import sys + + +def valid_dir_arg(value): + """Determine if the value is a valid directory""" + filepath = pathlib.Path(value) + + if not filepath.exists() or not filepath.is_dir(): + msg = f"Error! This is not a directory: {value}" + raise argparse.ArgumentTypeError(msg) + else: + return filepath + + +def create_folder_structure(cur_path, path_dict, source_folder): + for path in path_dict: + # Get next path to check for consistency + next_path = (cur_path / path).resolve() + print("Checking path: %s" % next_path) + nested_item = path_dict[path] + if type(nested_item) is not dict: + # This is a file we should copy over + if next_path.exists(): + continue + print(f"Copying over: {next_path.name}") + shutil.copy(source_folder / next_path.name, next_path) + else: + print(f"Creating folder: {next_path}") + os.makedirs(next_path) + create_folder_structure(next_path, nested_item, source_folder) + + +def main(args): + parser = argparse.ArgumentParser(description="Generate project structure.") + parser.add_argument("-t", "--target", type=valid_dir_arg) + args = parser.parse_args(args) + target_folder = args.target or pathlib.Path.cwd().parent.parent + folder_structure = pathlib.Path(__file__).parent / "folder_structure.json" + + with open(folder_structure) as json_file: + path_dict = json.load(json_file) + create_folder_structure(target_folder, path_dict["../../"], folder_structure.parent) + print("Done!") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/scripts/project-tools/install_desktop_file.sh b/scripts/project-tools/install_desktop_file.sh new file mode 100755 index 00000000..c0cd67b5 --- /dev/null +++ b/scripts/project-tools/install_desktop_file.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Make sure we are in this files directory +cd "$(dirname "$0")" + +PROJECT_NAME="Pets" +DESKTOP_FILE_DST="$HOME/.local/share/applications/blender_$PROJECT_NAME.desktop" +BLENDER_BIN_PATH=$(realpath ./run_blender.py) + +cp ../../local/blender/linux/blender.desktop $DESKTOP_FILE_DST + +# Update the .desktop file data +sed -i -e "s:Exec=blender:Exec=$BLENDER_BIN_PATH:" -e "s:Blender:Blender $PROJECT_NAME:" "$DESKTOP_FILE_DST" diff --git a/scripts/project-tools/pyproject.toml b/scripts/project-tools/pyproject.toml new file mode 100644 index 00000000..6970e018 --- /dev/null +++ b/scripts/project-tools/pyproject.toml @@ -0,0 +1,4 @@ +[tool.black] +line-length = 100 +include = '\.pyi?$' +skip-string-normalization = true diff --git a/scripts/project-tools/rollback_blender.py b/scripts/project-tools/rollback_blender.py new file mode 100755 index 00000000..fc046154 --- /dev/null +++ b/scripts/project-tools/rollback_blender.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import filecmp +import os +import shutil + +# The project base path (where shared, local and svn are located) +PATH_BASE = Path(__file__).resolve().parent.parent.parent +PATH_ARTIFACTS = PATH_BASE / 'shared' / 'artifacts' / 'blender' +PATH_PREVIOUS = PATH_ARTIFACTS / 'previous' +BACKUP_DIR = PATH_PREVIOUS / '00' + +if not BACKUP_DIR.exists(): + BACKUP_DIR.mkdir() + # Backup the current files + for file in PATH_ARTIFACTS.iterdir(): + if file.is_file(): + shutil.copy(file, BACKUP_DIR) + +cur_date_file = PATH_ARTIFACTS / "download_date" + +paths = sorted(Path(PATH_PREVIOUS).iterdir()) + +print("Available builds:\n") + +for index, path in enumerate(paths): + date_file = path / "download_date" + if not date_file.exists(): + print("ERROR: The backup folder %s is missing a datefile, exiting!" % path) + + with open(date_file, 'r') as file: + date = file.read().rstrip() + + if filecmp.cmp(cur_date_file, date_file): + print("\033[1mID:\033[0m\033[100m%3i (%s) \033[0m" % (index, date)) + else: + print("\033[1mID:\033[0m%3i (%s)" % (index, date)) + +input_error_mess = "Please select an index between 0 and " + str(len(paths) - 1) +selected_index = 0 + +while True: + index_str = input("Select which Blender build number to switch to. (press ENTER to confirm): ") + if not index_str.isnumeric(): + print(input_error_mess) + continue + index = int(index_str) + if index >= 0 and index < len(paths): + selected_index = index + break + print(input_error_mess) + +# Remove current files and move the selected snapshot into current folder +for file in PATH_ARTIFACTS.iterdir(): + if file.is_file(): + os.remove(file) + +for file in paths[selected_index].iterdir(): + # Everything should be a file in here but have this check for sanity eitherway. + if file.is_file(): + shutil.copy(file, PATH_ARTIFACTS) diff --git a/scripts/project-tools/run_blender.py b/scripts/project-tools/run_blender.py new file mode 100755 index 00000000..045d0965 --- /dev/null +++ b/scripts/project-tools/run_blender.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 + +import filecmp +import glob +import logging +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import zipfile + +from pathlib import Path + + +# The project base path (where shared, local and svn are located) +PATH_BASE = Path(__file__).resolve().parent.parent.parent +PATH_ARTIFACTS = PATH_BASE / 'shared' / 'artifacts' +PATH_LOCAL = PATH_BASE / 'local' + + +def setup_logger(): + # Create a logger + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + # Create a StreamHandler that outputs log messages to stdout + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setLevel(logging.DEBUG) + + # Create a formatter for the log messages + formatter = logging.Formatter('%(levelname)s - %(message)s') + + # Set the formatter for the StreamHandler + stream_handler.setFormatter(formatter) + + # Add the StreamHandler to the logger + logger.addHandler(stream_handler) + + return logger + + +logger = setup_logger() + + +def extract_dmg(dmg_file: Path, internal_pah, dst_path: Path): + # Execute hdiutil to mount the dmg file + mount_process = subprocess.run( + ['hdiutil', 'attach', dmg_file, '-plist'], capture_output=True, text=True + ) + mount_output = mount_process.stdout + + # Parse the mount_output to retrieve the mounted volume name + import plistlib + + plist_data = plistlib.loads(mount_output.encode('utf-8')) + mount_point = plist_data['system-entities'][0]['mount-point'] + + # Ensure destination directory exists + dst_path = dst_path / internal_pah + dst_path.mkdir(parents=True, exist_ok=True) + + # Extract the contents of the mounted dmg to the destination directory + file_in_dmg = os.path.join(mount_point, internal_pah) + subprocess.run(['ditto', file_in_dmg, dst_path]) + + # Unmount the dmg file + subprocess.run(['hdiutil', 'detach', mount_point]) + + +def extract_tar_xz(file_path: Path, dst_path: Path): + dst_path.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + 'tar', + 'xf', + file_path, + '--directory', + dst_path, + '--strip-components=1', + '--checkpoint=.1000', + ] + ) + + +def extract_zip(file_path: Path, dst_path: Path): + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(file_path, 'r') as zip_ref: + zip_ref.extractall(temp_dir) + + try: + src_path = [subdir for subdir in Path(temp_dir).iterdir()][0] + except IndexError: + logger.fatal("The archive %s does not contain any directory" % file_path.name) + sys.exit(1) + + dst_path.mkdir(parents=True, exist_ok=True) + shutil.move(src_path, dst_path) + + shutil.rmtree(temp_dir) + + +def update_addon(addon_zip_name, path_in_zip_to_extract=''): + addon_zip_sha = addon_zip_name + '.sha256' + # This is the file that records all toplevel folders/files installed by this addon + # It is used to cleanup old files and folders when updating or removing addons + addon_zip_files = addon_zip_name + '.files' + + # Check if we have the latest add-ons from shared + addon_artifacts_folder = PATH_ARTIFACTS / 'addons' + artifact_archive = addon_artifacts_folder / addon_zip_name + artifact_checksum = addon_artifacts_folder / addon_zip_sha + + if not artifact_checksum.exists(): + logger.error("Missing file %s" % artifact_checksum) + logger.error("Could not update add-ons") + return + + local_checksum = PATH_LOCAL / addon_zip_sha + + if local_checksum.exists(): + if filecmp.cmp(local_checksum, artifact_checksum): + logger.info("Already up to date") + return + + if not artifact_archive.exists(): + logger.error("Shasum exists but the archive file %s does not!" % artifact_archive) + logger.error("Could not update add-ons") + return + + # Extract the archive in a temp location and move the addons content to local + tmp_dir = Path(tempfile.mkdtemp()) + + # Extract the zip file to the temporary directory + with zipfile.ZipFile(artifact_archive, 'r') as zip_ref: + zip_ref.extractall(tmp_dir) + + # Get the path of the folder to copy + src_path_base = tmp_dir / path_in_zip_to_extract + dst_path_base = PATH_LOCAL / 'scripts' / 'addons' + + # Remove all files previously installed by the archive + local_installed_files = PATH_LOCAL / addon_zip_files + if local_installed_files.exists(): + with open(local_installed_files) as file: + lines = [line.rstrip() for line in file] + for folder in lines: + shutil.rmtree(dst_path_base / folder) + + # Get a list of directories inside the given directory + addons = [subdir.name for subdir in src_path_base.iterdir() if subdir.is_dir()] + + with open(local_installed_files, 'w') as f: + for addon_name in addons: + f.write("%s\n" % addon_name) + + for addon_name in addons: + logger.debug("Moving %s" % addon_name) + src_dir_addon = src_path_base / addon_name + dst_dir_addon = dst_path_base / addon_name + shutil.move(src_dir_addon, dst_dir_addon) + + # Clean up the temporary directory + shutil.rmtree(tmp_dir) + + # Update the sha256 file + shutil.copy(artifact_checksum, local_checksum) + + +def update_blender(): + system_name = platform.system().lower() + architecture = platform.machine() + + # Check if we have the latest blender archive from shared + artifacts_path = PATH_ARTIFACTS / 'blender' + archive_name_pattern = "blender*" + system_name + "." + architecture + "*.sha256" + + # Look for the appropriate Blender archive for this system + matched_archives = glob.glob(str(artifacts_path / archive_name_pattern)) + + # Check if we found any files + if len(matched_archives) != 1: + if len(matched_archives) == 0: + logger.error("No Blender archives found for this system!") + logger.error("System is: %s %s" % (system_name, architecture)) + return + else: + logger.error( + "More than one candidate archive was found for this system. Only one is allowed!" + ) + logger.error("The following candidates were found: %s" % str(matched_archives)) + return + + blender_build_checksum = Path(matched_archives[0]) + blender_build_archive = blender_build_checksum.with_suffix('') + + if not blender_build_archive.exists(): + logger.error( + "Shasum exists but the target Blender archive %s does not!" % blender_build_archive + ) + logger.error("Could not update blender") + return + + local_checksum = PATH_LOCAL / 'blender' / f"{system_name}.sha256" + + if local_checksum.exists(): + if filecmp.cmp(local_checksum, blender_build_checksum): + logger.info("Already up to date") + return + + src = artifacts_path / blender_build_archive + dst = PATH_LOCAL / 'blender' / system_name + if dst.exists(): + shutil.rmtree(dst) + + if system_name == 'linux': + extract_tar_xz(src, dst) + elif system_name == 'darwin': + extract_dmg(src, 'Blender.app', dst) + elif system_name == 'windows': + extract_zip(src, dst) + shutil.copy(blender_build_checksum, local_checksum) + + +def launch_blender(): + system_name = platform.system().lower() + blender_path_base = PATH_LOCAL / 'blender' / system_name + if system_name == 'linux': + blender_path = blender_path_base / 'blender' + elif system_name == 'darwin': + blender_path = blender_path_base / 'Blender.app' / 'Contents' / 'MacOS' / 'Blender' + elif system_name == 'windows': + blender_path = blender_path_base / 'blender.exe' + else: + sys.exit(1) + + os.environ['BLENDER_USER_CONFIG'] = str(PATH_LOCAL / 'config') + os.environ['BLENDER_USER_SCRIPTS'] = str(PATH_LOCAL / 'scripts') + subprocess.run([blender_path]) + + +def update_addons(): + path_in_zip_to_extract = Path('blender-studio-pipeline/scripts-blender/addons') + update_addon('blender-studio-pipeline-main.zip', path_in_zip_to_extract) + + +if __name__ == '__main__': + logger.info('Updating Add-ons') + update_addons() + logger.info('Updating Blender') + update_blender() + logger.info('Launching Blender') + launch_blender() diff --git a/scripts/project-tools/setup.cfg b/scripts/project-tools/setup.cfg new file mode 100644 index 00000000..62e73811 --- /dev/null +++ b/scripts/project-tools/setup.cfg @@ -0,0 +1,2 @@ +[pycodestyle] +max-line-length = 100 diff --git a/scripts/project-tools/update_addons.py b/scripts/project-tools/update_addons.py new file mode 100755 index 00000000..ec6a2336 --- /dev/null +++ b/scripts/project-tools/update_addons.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import glob +import hashlib +import os +import pathlib +import requests + + +def download_file(url, out_folder, filename): + print("Downloading: " + url) + local_filename = out_folder / filename + + # TODO Can't check any shasums before downloading so always remove and redownload everything for now + prev_downloaded_files = glob.glob(f"{local_filename}*") + for file in prev_downloaded_files: + os.remove(file) + + # NOTE the stream=True parameter below + with requests.get(url, stream=True) as r: + r.raise_for_status() + with open(local_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=None): + if chunk: + f.write(chunk) + + local_hash_filename = local_filename.with_suffix(".zip.sha256") + with open(local_filename, "rb") as f: + digest = hashlib.file_digest(f, "sha256") + with open(local_hash_filename, "w") as hash_file: + hash_file.write(digest.hexdigest()) + + return local_filename + + +current_file_folder_path = pathlib.Path(__file__).parent +download_folder_path = (current_file_folder_path / "../../shared/artifacts/addons/").resolve() + +# Ensure that the download directory exists +os.makedirs(download_folder_path, exist_ok=True) + +download_file( + "https://projects.blender.org/studio/blender-studio-pipeline/archive/main.zip", + download_folder_path, + "blender-studio-pipeline-main.zip", +) diff --git a/scripts/project-tools/update_blender.py b/scripts/project-tools/update_blender.py new file mode 100755 index 00000000..b8f38e52 --- /dev/null +++ b/scripts/project-tools/update_blender.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +import email.utils +import glob +import hashlib +import os +import pathlib +import re +import requests +import shutil + + +HOMEPAGE = "https://builder.blender.org/download/" + +BLENDER_BRANCH = "main" + + +def download_file(url, out_folder): + print("Downloading: " + url) + local_filename = out_folder / url.split('/')[-1] + # NOTE the stream=True parameter below + with requests.get(url, stream=True) as r: + r.raise_for_status() + with open(local_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=None): + if chunk: + f.write(chunk) + return local_filename + + +def shasum_matches(file, sha_sum): + with open(file, "rb") as f: + digest = hashlib.file_digest(f, "sha256") + return digest.hexdigest() == sha_sum + + +current_file_folder_path = pathlib.Path(__file__).parent +download_folder_path = (current_file_folder_path / "../../shared/artifacts/blender").resolve() +backup_folder_path = download_folder_path / "previous/current_snapshot" + +os.makedirs(download_folder_path, exist_ok=True) + +# Backup the old files +os.makedirs(backup_folder_path, exist_ok=True) + +for f in os.listdir(download_folder_path): + if os.path.isfile(f): + path_to_file = f / download_folder_path + shutil.copy(path_to_file, backup_folder_path) + +# Get all urls for the blender builds +platforms_dict = { + "windows": "zip", + "darwin.x86_64": "dmg", + "darwin.arm64": "dmg", + "linux": "tar.xz", +} + +download_info = [] +branch_string = "+" + BLENDER_BRANCH +reqs = requests.get(HOMEPAGE) +for match in re.findall(' 1: + print( + f"Platform {platform} has multiple downloaded files in the artifacts directory, exiting!" + ) + exit(1) + # Check if we need to download the file by looking at the shasum of the currently downloaded file (if any) + if len(current_platform_file) == 1: + current_file = current_platform_file[0] + if shasum_matches(current_file, sha): + # We already have the current file + continue + else: + updated_current_files = True + os.remove(current_file) + os.remove(current_file + ".sha256") + + download_file(url_sha, download_folder_path) + downloaded_file = download_file(url, download_folder_path) + # Check that the file we downloaded is not corrupt + if not shasum_matches(downloaded_file, sha): + print(f"Downloaded file {downloaded_file} does not match its shasum, exiting!") + exit(1) + new_files_downloaded = True + +if new_files_downloaded: + # Save download date for use in the rollback script + with open(download_folder_path / "download_date", "w") as date_file: + date_file.write(email.utils.formatdate(localtime=True)) + print("Updated to the latest files") + +if updated_current_files: + backup_path = download_folder_path / "previous" + # Put the current backup first in the directory listing + os.rename(backup_folder_path, backup_path / "00") + backup_dirs = os.listdir(backup_path) + backup_dirs.sort(reverse=True) + + # Remove older backup folders if there are more than 10 + folders_to_remove = len(backup_dirs) - 10 + if folders_to_remove > 0: + for dir in backup_dirs[:folders_to_remove]: + shutil.rmtree(dir) + backup_dirs = backup_dirs[folders_to_remove:] + + # Bump all folder names + # Assign a number to each file, reverse the processing order to not overwrite any files. + folder_number = len(backup_dirs) + for dir in backup_dirs: + os.rename(dir, backup_path / str(folder_number).zfill(2)) + folder_number -= 1 +else: + shutil.rmtree(backup_folder_path) + print("Nothing downloaded, everything was up to date")