diff --git a/io_scene_fbx/__init__.py b/io_scene_fbx/__init__.py index b07e1ede9..08d900c3e 100644 --- a/io_scene_fbx/__init__.py +++ b/io_scene_fbx/__init__.py @@ -198,19 +198,20 @@ class ImportFBX(bpy.types.Operator, ImportHelper): def execute(self, context): keywords = self.as_keywords(ignore=("filter_glob", "directory", "ui_tab", "filepath", "files")) - from . import import_fbx + from . import import_fbx, fbx_utils_threading import os - if self.files: - ret = {'CANCELLED'} - dirname = os.path.dirname(self.filepath) - for file in self.files: - path = os.path.join(dirname, file.name) - if import_fbx.load(self, context, filepath=path, **keywords) == {'FINISHED'}: - ret = {'FINISHED'} - return ret - else: - return import_fbx.load(self, context, filepath=self.filepath, **keywords) + with fbx_utils_threading.new_thread_pool_executor(thread_name_prefix=self.bl_idname) as executor: + if self.files: + ret = {'CANCELLED'} + dirname = os.path.dirname(self.filepath) + for file in self.files: + path = os.path.join(dirname, file.name) + if import_fbx.load(self, context, executor, filepath=path, **keywords) == {'FINISHED'}: + ret = {'FINISHED'} + return ret + else: + return import_fbx.load(self, context, executor, filepath=self.filepath, **keywords) class FBX_PT_import_include(bpy.types.Panel): diff --git a/io_scene_fbx/fbx_utils_threading.py b/io_scene_fbx/fbx_utils_threading.py index bf7631b51..4d113cc37 100644 --- a/io_scene_fbx/fbx_utils_threading.py +++ b/io_scene_fbx/fbx_utils_threading.py @@ -13,10 +13,51 @@ _MULTITHREADING_ENABLED = True # The concurrent.futures module may not work or may not be available on WebAssembly platforms wasm32-emscripten and # wasm32-wasi. try: - from concurrent.futures import ThreadPoolExecutor + from concurrent.futures import ThreadPoolExecutor, Executor, Future except ModuleNotFoundError: _MULTITHREADING_ENABLED = False ThreadPoolExecutor = None + Executor = object + + # So that the same code can be used regardless of whether concurrent.futures is available, create a basic + # implementation of concurrent.futures.Future with the expected methods. + class Future: + """Minimal Future implementation that is always considered done. + + The defined functions match those in concurrent.futures.Future""" + __slots__ = "_result", "_exception" + + def __init__(self): + self._result = None + self._exception = None + + def cancel(self): return False # The Future cannot be cancelled. + def cancelled(self): return False # The Future can never be cancelled. + def running(self): return False # The Future is never considered to be running. + def done(self): return True # The Future is always considered to be done. + + def result(self, timeout=None): + ex = self._exception + if ex is not None: + raise ex + else: + return self._result + + def exception(self, timeout=None): return self._exception + + def add_done_callback(self, fn): + try: + fn() + except Exception as e: + # The Future.add_done_callback documentation specifies that it logs and ignores raised Exception + # subclasses. The behaviour of other BaseException subclasses is undefined, so, for simplicity, we won't + # catch those. + print("Exception in Future done callback: %s" % e) + + # No need to implement this because a result or exception should always be set. + def set_running_or_notify_cancel(self): raise NotImplementedError() + def set_result(self, result): self._result = result + def set_exception(self, exception): self._exception = exception else: try: # The module may be available, but not be fully functional. An error may be raised when attempting to start a @@ -29,6 +70,52 @@ else: _MULTITHREADING_ENABLED = False +# To be able to use the same code on systems that cannot use multithreading, an executor class is needed that simply +# immediately runs submitted tasks on the calling thread. This class can also be used whenever the number of maximum +# worker threads is zero. +class ImmediateExecutor(Executor): + """Executor that immediately executes submitted callables on the calling thread. + + The Future objects returned by this executor will always be done.""" + __slots__ = "_shut_down", + + def __init__(self): + super().__init__() + self._shut_down = False + + def __enter__(self): return self + def __exit__(self, exc_type, exc_val, exc_tb): pass + def shutdown(self, wait=True, *, cancel_futures=False): self._shut_down = True + + def map(self, fn, *iterables, timeout=None, chunksize=1): + # To match ThreadPoolExecutor behaviour, all callables are submitted first and then the results are yielded in + # the order the callables were submitted. + futures = [self.submit(fn, *args) for args in zip(*iterables)] + for future in futures: + yield future.result() + + def submit(self, __fn, /, *args, **kwargs): + if self._shut_down: + raise RuntimeError("Cannot schedule new callables after shutdown") + future = Future() + try: + result = __fn(*args, **kwargs) + except Exception as ex: + future.set_exception(ex) + else: + future.set_result(result) + return future + + +def new_thread_pool_executor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()): + """Create a ThreadPoolExecutor if possible, otherwise fall back to creating an executor that immediately executes + submitted callables on the calling thread.""" + if (max_workers is not None and max_workers < 1) or not _MULTITHREADING_ENABLED or ThreadPoolExecutor is None: + return ImmediateExecutor() + else: + return ThreadPoolExecutor(max_workers, thread_name_prefix, initializer, initargs) + + def get_cpu_count(): """Get the number of cpus assigned to the current process if that information is available on this system. If not available, get the total number of cpus. diff --git a/io_scene_fbx/import_fbx.py b/io_scene_fbx/import_fbx.py index ff37e4d71..7d12da8ed 100644 --- a/io_scene_fbx/import_fbx.py +++ b/io_scene_fbx/import_fbx.py @@ -2091,7 +2091,7 @@ def blen_read_material(fbx_tmpl, fbx_obj, settings): # ------- # Image & Texture -def blen_read_texture_image(fbx_tmpl, fbx_obj, basedir, settings): +def blen_read_texture_image(executor, fbx_tmpl, fbx_obj, basedir, settings): import os from bpy_extras import image_utils @@ -2127,31 +2127,42 @@ def blen_read_texture_image(fbx_tmpl, fbx_obj, basedir, settings): else : filepath = filepath.replace('\\', '/') if (os.sep == '/') else filepath.replace('/', '\\') - image = image_cache.get(filepath) - if image is not None: + cached_image = image_cache.get(filepath) + if cached_image is not None: # Data is only embedded once, we may have already created the image but still be missing its data! - if not image.has_data: - pack_data_from_content(image, fbx_obj) - return image + def extra_pack_from_content(image): + if not image.has_data: + pack_data_from_content(image, fbx_obj) + image_future, finish_loading_callbacks = cached_image + finish_loading_callbacks.append(extra_pack_from_content) + return image_future - image = image_utils.load_image( - filepath, - dirname=basedir, - place_holder=True, - recursive=settings.use_image_search, - ) + image_future = executor.submit(image_utils.load_image, + filepath, + dirname=basedir, + place_holder=True, + recursive=settings.use_image_search, + ) - # Try to use embedded data, if available! - pack_data_from_content(image, fbx_obj) - - image_cache[filepath] = image # name can be ../a/b/c - image.name = os.path.basename(elem_name_utf8) + desired_image_name = os.path.basename(elem_name_utf8) - if settings.use_custom_props: - blen_read_custom_properties(fbx_obj, image, settings) + # We don't use image_future.add_done_callback because it logs and then ignores exceptions, and we need to ensure + # that images are named in a deterministic order in-case there are duplicate names. So we'll run callback + # functions on the main thread to finish loading the images in the order they were read. + def finish_loading(image): + # Try to use embedded data, if available! + pack_data_from_content(image, fbx_obj) - return image + # name can be ../a/b/c + image.name = desired_image_name + + if settings.use_custom_props: + blen_read_custom_properties(fbx_obj, image, settings) + + image_cache[filepath] = image_future, [finish_loading] + + return image_future def blen_read_camera(fbx_tmpl, fbx_obj, settings): @@ -2988,7 +2999,7 @@ class FbxImportHelperNode: return None -def load(operator, context, filepath="", +def load(operator, context, executor, filepath="", use_manual_orientation=False, axis_forward='-Z', axis_up='Y', @@ -3009,7 +3020,8 @@ def load(operator, context, filepath="", primary_bone_axis='Y', secondary_bone_axis='X', use_prepost_rot=True, - colors_type='SRGB'): + colors_type='SRGB', + ): global fbx_elem_nil fbx_elem_nil = FBXElem('', (), (), ()) @@ -3226,6 +3238,31 @@ def load(operator, context, filepath="", fbx_connection_map_reverse.setdefault(c_dst, []).append((c_src, fbx_link)) _(); del _ + perfmon.step("FBX import: Textures...") + + # ---- + # Load image & textures data + # This is done early on because searching for an image can take a long time, especially if starting from close to a + # top-level directory, but it can run in the background on a different thread. + def _(): + fbx_tmpl_tex = fbx_template_get((b'Texture', b'KFbxFileTexture')) + fbx_tmpl_img = fbx_template_get((b'Video', b'KFbxVideo')) + + # Important to run all 'Video' ones first, embedded images are stored in those nodes. + # XXX Note we simplify things here, assuming both matching Video and Texture will use same file path, + # this may be a bit weak, if issue arise we'll fallback to plain connection stuff... + for fbx_uuid, fbx_item in fbx_table_nodes.items(): + fbx_obj, blen_data = fbx_item + if fbx_obj.id != b'Video': + continue + fbx_item[1] = blen_read_texture_image(executor, fbx_tmpl_img, fbx_obj, basedir, settings) + for fbx_uuid, fbx_item in fbx_table_nodes.items(): + fbx_obj, blen_data = fbx_item + if fbx_obj.id != b'Texture': + continue + fbx_item[1] = blen_read_texture_image(executor, fbx_tmpl_tex, fbx_obj, basedir, settings) + _(); del _ + perfmon.step("FBX import: Meshes...") # ---- @@ -3242,7 +3279,7 @@ def load(operator, context, filepath="", fbx_item[1] = blen_read_geom(fbx_tmpl, fbx_obj, settings) _(); del _ - perfmon.step("FBX import: Materials & Textures...") + perfmon.step("FBX import: Materials...") # ---- # Load material data @@ -3258,27 +3295,6 @@ def load(operator, context, filepath="", fbx_item[1] = blen_read_material(fbx_tmpl, fbx_obj, settings) _(); del _ - # ---- - # Load image & textures data - def _(): - fbx_tmpl_tex = fbx_template_get((b'Texture', b'KFbxFileTexture')) - fbx_tmpl_img = fbx_template_get((b'Video', b'KFbxVideo')) - - # Important to run all 'Video' ones first, embedded images are stored in those nodes. - # XXX Note we simplify things here, assuming both matching Video and Texture will use same file path, - # this may be a bit weak, if issue arise we'll fallback to plain connection stuff... - for fbx_uuid, fbx_item in fbx_table_nodes.items(): - fbx_obj, blen_data = fbx_item - if fbx_obj.id != b'Video': - continue - fbx_item[1] = blen_read_texture_image(fbx_tmpl_img, fbx_obj, basedir, settings) - for fbx_uuid, fbx_item in fbx_table_nodes.items(): - fbx_obj, blen_data = fbx_item - if fbx_obj.id != b'Texture': - continue - fbx_item[1] = blen_read_texture_image(fbx_tmpl_tex, fbx_obj, basedir, settings) - _(); del _ - perfmon.step("FBX import: Cameras & Lamps...") # ---- @@ -3870,9 +3886,12 @@ def load(operator, context, filepath="", material = fbx_table_nodes.get(fbx_uuid, (None, None))[1] for (fbx_lnk, - image, + image_future, fbx_lnk_type) in connection_filter_reverse(fbx_uuid, b'Texture'): + # Waits for the image to load if it's still loading, or raises the exception it threw while loading. + image = image_future.result() + if fbx_lnk_type.props[0] == b'OP': lnk_type = fbx_lnk_type.props[3] @@ -3974,6 +3993,19 @@ def load(operator, context, filepath="", obj.visible_shadow = False _(); del _ + perfmon.step("FBX import: Finish remaining image loading...") + + def _(): + # Finish image loading on the main thread to simplify exception propagation and to ensure that images are named + # deterministically in the case of duplicate names. + for image_future, finish_loading_callbacks in image_cache.values(): + # Waits until the Future has a result or raises the Future's exception if it raised an exception. + image = image_future.result() + for func in finish_loading_callbacks: + func(image) + + _(); del _ + perfmon.level_down() perfmon.level_down("Import finished.")