From 6518667f71cebd83283ee00291f78c795cd600d1 Mon Sep 17 00:00:00 2001 From: Thomas Barlow Date: Tue, 5 Dec 2023 02:32:02 +0000 Subject: [PATCH 1/2] FBX IO: Search for images on separate threads This offloads some IO work to find image files into separate threads. Texture loading is moved to the start of FBX import so that the rest of the fbx import can continue while searching for images runs in the background. This can significantly reduce import times when the Image Search option of the importer is enabled (it is enabled by default) and an imported file has images with paths that cause a search to begin in a directory with many nested subdirectories, which can happen when a relative path goes up many parent directories, the relative path starting with "../../../" or similar. While not as common, this can also significantly reduce import times when an imported file references many images with file paths that start with the same drive letter as a slower drive, such as a CD drive. Aside from these cases, import times are expected to be the same or slightly slower, likely due to the time spent starting threads or due to each thread contending for the GIL for its parts that don't release the GIL. --- io_scene_fbx/__init__.py | 23 ++--- io_scene_fbx/fbx_utils_threading.py | 89 +++++++++++++++++++- io_scene_fbx/import_fbx.py | 125 ++++++++++++++++++---------- 3 files changed, 181 insertions(+), 56 deletions(-) diff --git a/io_scene_fbx/__init__.py b/io_scene_fbx/__init__.py index b07e1ede9..4ce823ed2 100644 --- a/io_scene_fbx/__init__.py +++ b/io_scene_fbx/__init__.py @@ -198,19 +198,20 @@ class ImportFBX(bpy.types.Operator, ImportHelper): def execute(self, context): keywords = self.as_keywords(ignore=("filter_glob", "directory", "ui_tab", "filepath", "files")) - from . import import_fbx + from . import import_fbx, fbx_utils_threading import os - if self.files: - ret = {'CANCELLED'} - dirname = os.path.dirname(self.filepath) - for file in self.files: - path = os.path.join(dirname, file.name) - if import_fbx.load(self, context, filepath=path, **keywords) == {'FINISHED'}: - ret = {'FINISHED'} - return ret - else: - return import_fbx.load(self, context, filepath=self.filepath, **keywords) + with fbx_utils_threading.new_thread_pool_executor(thread_name_prefix=self.bl_idname) as executor: + if self.files: + ret = {'CANCELLED'} + dirname = os.path.dirname(self.filepath) + for file in self.files: + path = os.path.join(dirname, file.name) + if import_fbx.load(self, context, filepath=path, executor=executor, **keywords) == {'FINISHED'}: + ret = {'FINISHED'} + return ret + else: + return import_fbx.load(self, context, filepath=self.filepath, executor=executor, **keywords) class FBX_PT_import_include(bpy.types.Panel): diff --git a/io_scene_fbx/fbx_utils_threading.py b/io_scene_fbx/fbx_utils_threading.py index bf7631b51..4d113cc37 100644 --- a/io_scene_fbx/fbx_utils_threading.py +++ b/io_scene_fbx/fbx_utils_threading.py @@ -13,10 +13,51 @@ _MULTITHREADING_ENABLED = True # The concurrent.futures module may not work or may not be available on WebAssembly platforms wasm32-emscripten and # wasm32-wasi. try: - from concurrent.futures import ThreadPoolExecutor + from concurrent.futures import ThreadPoolExecutor, Executor, Future except ModuleNotFoundError: _MULTITHREADING_ENABLED = False ThreadPoolExecutor = None + Executor = object + + # So that the same code can be used regardless of whether concurrent.futures is available, create a basic + # implementation of concurrent.futures.Future with the expected methods. + class Future: + """Minimal Future implementation that is always considered done. + + The defined functions match those in concurrent.futures.Future""" + __slots__ = "_result", "_exception" + + def __init__(self): + self._result = None + self._exception = None + + def cancel(self): return False # The Future cannot be cancelled. + def cancelled(self): return False # The Future can never be cancelled. + def running(self): return False # The Future is never considered to be running. + def done(self): return True # The Future is always considered to be done. + + def result(self, timeout=None): + ex = self._exception + if ex is not None: + raise ex + else: + return self._result + + def exception(self, timeout=None): return self._exception + + def add_done_callback(self, fn): + try: + fn() + except Exception as e: + # The Future.add_done_callback documentation specifies that it logs and ignores raised Exception + # subclasses. The behaviour of other BaseException subclasses is undefined, so, for simplicity, we won't + # catch those. + print("Exception in Future done callback: %s" % e) + + # No need to implement this because a result or exception should always be set. + def set_running_or_notify_cancel(self): raise NotImplementedError() + def set_result(self, result): self._result = result + def set_exception(self, exception): self._exception = exception else: try: # The module may be available, but not be fully functional. An error may be raised when attempting to start a @@ -29,6 +70,52 @@ else: _MULTITHREADING_ENABLED = False +# To be able to use the same code on systems that cannot use multithreading, an executor class is needed that simply +# immediately runs submitted tasks on the calling thread. This class can also be used whenever the number of maximum +# worker threads is zero. +class ImmediateExecutor(Executor): + """Executor that immediately executes submitted callables on the calling thread. + + The Future objects returned by this executor will always be done.""" + __slots__ = "_shut_down", + + def __init__(self): + super().__init__() + self._shut_down = False + + def __enter__(self): return self + def __exit__(self, exc_type, exc_val, exc_tb): pass + def shutdown(self, wait=True, *, cancel_futures=False): self._shut_down = True + + def map(self, fn, *iterables, timeout=None, chunksize=1): + # To match ThreadPoolExecutor behaviour, all callables are submitted first and then the results are yielded in + # the order the callables were submitted. + futures = [self.submit(fn, *args) for args in zip(*iterables)] + for future in futures: + yield future.result() + + def submit(self, __fn, /, *args, **kwargs): + if self._shut_down: + raise RuntimeError("Cannot schedule new callables after shutdown") + future = Future() + try: + result = __fn(*args, **kwargs) + except Exception as ex: + future.set_exception(ex) + else: + future.set_result(result) + return future + + +def new_thread_pool_executor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()): + """Create a ThreadPoolExecutor if possible, otherwise fall back to creating an executor that immediately executes + submitted callables on the calling thread.""" + if (max_workers is not None and max_workers < 1) or not _MULTITHREADING_ENABLED or ThreadPoolExecutor is None: + return ImmediateExecutor() + else: + return ThreadPoolExecutor(max_workers, thread_name_prefix, initializer, initargs) + + def get_cpu_count(): """Get the number of cpus assigned to the current process if that information is available on this system. If not available, get the total number of cpus. diff --git a/io_scene_fbx/import_fbx.py b/io_scene_fbx/import_fbx.py index ff37e4d71..e6c698d4f 100644 --- a/io_scene_fbx/import_fbx.py +++ b/io_scene_fbx/import_fbx.py @@ -2091,7 +2091,7 @@ def blen_read_material(fbx_tmpl, fbx_obj, settings): # ------- # Image & Texture -def blen_read_texture_image(fbx_tmpl, fbx_obj, basedir, settings): +def blen_read_texture_image(executor, fbx_tmpl, fbx_obj, basedir, settings): import os from bpy_extras import image_utils @@ -2127,31 +2127,42 @@ def blen_read_texture_image(fbx_tmpl, fbx_obj, basedir, settings): else : filepath = filepath.replace('\\', '/') if (os.sep == '/') else filepath.replace('/', '\\') - image = image_cache.get(filepath) - if image is not None: + cached_image = image_cache.get(filepath) + if cached_image is not None: # Data is only embedded once, we may have already created the image but still be missing its data! - if not image.has_data: - pack_data_from_content(image, fbx_obj) - return image + def extra_pack_from_content(image): + if not image.has_data: + pack_data_from_content(image, fbx_obj) + image_future, finish_loading_callbacks = cached_image + finish_loading_callbacks.append(extra_pack_from_content) + return image_future - image = image_utils.load_image( - filepath, - dirname=basedir, - place_holder=True, - recursive=settings.use_image_search, - ) + image_future = executor.submit(image_utils.load_image, + filepath, + dirname=basedir, + place_holder=True, + recursive=settings.use_image_search, + ) - # Try to use embedded data, if available! - pack_data_from_content(image, fbx_obj) - - image_cache[filepath] = image # name can be ../a/b/c - image.name = os.path.basename(elem_name_utf8) + desired_image_name = os.path.basename(elem_name_utf8) - if settings.use_custom_props: - blen_read_custom_properties(fbx_obj, image, settings) + # We don't use image_future.add_done_callback because it logs and then ignores exceptions, and we need to ensure + # that images are named in a deterministic order in-case there are duplicate names. So we'll run callback + # functions on the main thread to finish loading the images in the order they were read. + def finish_loading(image): + # Try to use embedded data, if available! + pack_data_from_content(image, fbx_obj) - return image + # name can be ../a/b/c + image.name = desired_image_name + + if settings.use_custom_props: + blen_read_custom_properties(fbx_obj, image, settings) + + image_cache[filepath] = image_future, [finish_loading] + + return image_future def blen_read_camera(fbx_tmpl, fbx_obj, settings): @@ -3009,7 +3020,9 @@ def load(operator, context, filepath="", primary_bone_axis='Y', secondary_bone_axis='X', use_prepost_rot=True, - colors_type='SRGB'): + colors_type='SRGB', + executor=None, + ): global fbx_elem_nil fbx_elem_nil = FBXElem('', (), (), ()) @@ -3020,6 +3033,10 @@ def load(operator, context, filepath="", from . import parse_fbx from .fbx_utils import RIGHT_HAND_AXES, FBX_FRAMERATES + from .fbx_utils_threading import ImmediateExecutor + + if executor is None: + executor = ImmediateExecutor() start_time_proc = time.process_time() start_time_sys = time.time() @@ -3226,6 +3243,31 @@ def load(operator, context, filepath="", fbx_connection_map_reverse.setdefault(c_dst, []).append((c_src, fbx_link)) _(); del _ + perfmon.step("FBX import: Textures...") + + # ---- + # Load image & textures data + # This is done early on because searching for an image can take a long time, especially if starting from close to a + # top-level directory, but it can run in the background on a different thread. + def _(): + fbx_tmpl_tex = fbx_template_get((b'Texture', b'KFbxFileTexture')) + fbx_tmpl_img = fbx_template_get((b'Video', b'KFbxVideo')) + + # Important to run all 'Video' ones first, embedded images are stored in those nodes. + # XXX Note we simplify things here, assuming both matching Video and Texture will use same file path, + # this may be a bit weak, if issue arise we'll fallback to plain connection stuff... + for fbx_uuid, fbx_item in fbx_table_nodes.items(): + fbx_obj, blen_data = fbx_item + if fbx_obj.id != b'Video': + continue + fbx_item[1] = blen_read_texture_image(executor, fbx_tmpl_img, fbx_obj, basedir, settings) + for fbx_uuid, fbx_item in fbx_table_nodes.items(): + fbx_obj, blen_data = fbx_item + if fbx_obj.id != b'Texture': + continue + fbx_item[1] = blen_read_texture_image(executor, fbx_tmpl_tex, fbx_obj, basedir, settings) + _(); del _ + perfmon.step("FBX import: Meshes...") # ---- @@ -3242,7 +3284,7 @@ def load(operator, context, filepath="", fbx_item[1] = blen_read_geom(fbx_tmpl, fbx_obj, settings) _(); del _ - perfmon.step("FBX import: Materials & Textures...") + perfmon.step("FBX import: Materials...") # ---- # Load material data @@ -3258,27 +3300,6 @@ def load(operator, context, filepath="", fbx_item[1] = blen_read_material(fbx_tmpl, fbx_obj, settings) _(); del _ - # ---- - # Load image & textures data - def _(): - fbx_tmpl_tex = fbx_template_get((b'Texture', b'KFbxFileTexture')) - fbx_tmpl_img = fbx_template_get((b'Video', b'KFbxVideo')) - - # Important to run all 'Video' ones first, embedded images are stored in those nodes. - # XXX Note we simplify things here, assuming both matching Video and Texture will use same file path, - # this may be a bit weak, if issue arise we'll fallback to plain connection stuff... - for fbx_uuid, fbx_item in fbx_table_nodes.items(): - fbx_obj, blen_data = fbx_item - if fbx_obj.id != b'Video': - continue - fbx_item[1] = blen_read_texture_image(fbx_tmpl_img, fbx_obj, basedir, settings) - for fbx_uuid, fbx_item in fbx_table_nodes.items(): - fbx_obj, blen_data = fbx_item - if fbx_obj.id != b'Texture': - continue - fbx_item[1] = blen_read_texture_image(fbx_tmpl_tex, fbx_obj, basedir, settings) - _(); del _ - perfmon.step("FBX import: Cameras & Lamps...") # ---- @@ -3870,9 +3891,12 @@ def load(operator, context, filepath="", material = fbx_table_nodes.get(fbx_uuid, (None, None))[1] for (fbx_lnk, - image, + image_future, fbx_lnk_type) in connection_filter_reverse(fbx_uuid, b'Texture'): + # Waits for the image to load if it's still loading, or raises the exception it threw while loading. + image = image_future.result() + if fbx_lnk_type.props[0] == b'OP': lnk_type = fbx_lnk_type.props[3] @@ -3974,6 +3998,19 @@ def load(operator, context, filepath="", obj.visible_shadow = False _(); del _ + perfmon.step("FBX import: Finish remaining image loading...") + + def _(): + # Finish image loading on the main thread to simplify exception propagation and to ensure that images are named + # deterministically in the case of duplicate names. + for image_future, finish_loading_callbacks in image_cache.values(): + # Waits until the Future has a result or raises the Future's exception if it raised an exception. + image = image_future.result() + for func in finish_loading_callbacks: + func(image) + + _(); del _ + perfmon.level_down() perfmon.level_down("Import finished.") -- 2.30.2 From f9f3507c6c8f6756c278b387ed49b531c24f6d7a Mon Sep 17 00:00:00 2001 From: Thomas Barlow Date: Thu, 1 Feb 2024 02:21:55 +0000 Subject: [PATCH 2/2] Remove default `executor` argument for import_fbx.load() Add-ons shouldn't be calling `import_fbx` functions directly and this helps separate the `executor` argument from the operator properties which are passed to `import_fbx.load()` as keyword-arguments. --- io_scene_fbx/__init__.py | 4 ++-- io_scene_fbx/import_fbx.py | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/io_scene_fbx/__init__.py b/io_scene_fbx/__init__.py index 4ce823ed2..08d900c3e 100644 --- a/io_scene_fbx/__init__.py +++ b/io_scene_fbx/__init__.py @@ -207,11 +207,11 @@ class ImportFBX(bpy.types.Operator, ImportHelper): dirname = os.path.dirname(self.filepath) for file in self.files: path = os.path.join(dirname, file.name) - if import_fbx.load(self, context, filepath=path, executor=executor, **keywords) == {'FINISHED'}: + if import_fbx.load(self, context, executor, filepath=path, **keywords) == {'FINISHED'}: ret = {'FINISHED'} return ret else: - return import_fbx.load(self, context, filepath=self.filepath, executor=executor, **keywords) + return import_fbx.load(self, context, executor, filepath=self.filepath, **keywords) class FBX_PT_import_include(bpy.types.Panel): diff --git a/io_scene_fbx/import_fbx.py b/io_scene_fbx/import_fbx.py index e6c698d4f..7d12da8ed 100644 --- a/io_scene_fbx/import_fbx.py +++ b/io_scene_fbx/import_fbx.py @@ -2999,7 +2999,7 @@ class FbxImportHelperNode: return None -def load(operator, context, filepath="", +def load(operator, context, executor, filepath="", use_manual_orientation=False, axis_forward='-Z', axis_up='Y', @@ -3021,7 +3021,6 @@ def load(operator, context, filepath="", secondary_bone_axis='X', use_prepost_rot=True, colors_type='SRGB', - executor=None, ): global fbx_elem_nil @@ -3033,10 +3032,6 @@ def load(operator, context, filepath="", from . import parse_fbx from .fbx_utils import RIGHT_HAND_AXES, FBX_FRAMERATES - from .fbx_utils_threading import ImmediateExecutor - - if executor is None: - executor = ImmediateExecutor() start_time_proc = time.process_time() start_time_sys = time.time() -- 2.30.2