From 7864ccac7d1529abd2d36f1a966c74dc1aaf7976 Mon Sep 17 00:00:00 2001 From: Thomas Barlow Date: Mon, 19 Jun 2023 05:55:50 +0100 Subject: [PATCH 1/5] FBX IO: Multithread exported array compression zlib.compress releases the GIL so can be multithreaded. This patch adds a context manager to FBXElem that temporarily enables multithreading of the compression of added arrays, by using the recently added utility to schedule CPU-bound tasks to run on separate threads. On my Ryzen 7 7800x, exporting non-animated rigged humanoid models typically results in about a 1.2 to 1.4 times faster export. Exporting only simple geometry, such as many subdivided default cubes, can be about 2 to 3 times faster. No changes are expected to the contents of exported files. --- io_scene_fbx/encode_bin.py | 78 +++++++++++++++++++++++++++++----- io_scene_fbx/export_fbx_bin.py | 38 +++++++++-------- 2 files changed, 89 insertions(+), 27 deletions(-) diff --git a/io_scene_fbx/encode_bin.py b/io_scene_fbx/encode_bin.py index a7e8071e2..69e2786fe 100644 --- a/io_scene_fbx/encode_bin.py +++ b/io_scene_fbx/encode_bin.py @@ -3,11 +3,13 @@ # SPDX-License-Identifier: GPL-2.0-or-later try: - from . import data_types + from . import data_types, fbx_utils except: import data_types + import fbx_utils from struct import pack +from contextlib import contextmanager import array import numpy as np import zlib @@ -51,6 +53,57 @@ class FBXElem: self._end_offset = -1 self._props_length = -1 + @classmethod + @contextmanager + def enable_multithreading_cm(cls): + """Temporarily enable multithreaded array compression. + + The context manager handles starting up and shutting down the threads. + + Only exits once all the threads are done (either all tasks were completed or an error occurred and the threads + were stopped prematurely). + + Writing to a file is temporarily disabled as a safeguard.""" + # __enter__() + orig_func = cls._add_compressed_array_helper + orig_write = cls._write + + def insert_compressed_array(props, insert_at, data, length): + # zlib.compress releases the GIL, so can be multithreaded. + data = zlib.compress(data, 1) + comp_len = len(data) + + encoding = 1 + data = pack('<3I', length, encoding, comp_len) + data + props[insert_at] = data + + with fbx_utils.MultiThreadedTaskConsumer.new_cpu_bound_cm(insert_compressed_array) as wrapped_func: + try: + def _add_compressed_array_helper_multi(self, data, length): + # Append a dummy value that will be replaced with the compressed array data later. + self.props.append(...) + # The index to insert the compressed array into. + insert_at = len(self.props) - 1 + # Schedule the array to be compressed on a separate thread and then inserted into the hierarchy at + # `insert_at`. + wrapped_func(self.props, insert_at, data, length) + + # As an extra safeguard, temporarily replace the `_write` function to raise an error if called. + def temp_write(*_args, **_kwargs): + raise RuntimeError("Writing is not allowed until multithreaded array compression has been disabled") + + cls._add_compressed_array_helper = _add_compressed_array_helper_multi + cls._write = temp_write + + # Return control back to the caller of __enter__(). + yield + finally: + # __exit__() + # Restore the original functions. + cls._add_compressed_array_helper = orig_func + cls._write = orig_write + # Exiting the MultiThreadedTaskConsumer context manager will wait for all scheduled tasks to complete. + def add_bool(self, data): assert(isinstance(data, bool)) data = pack('?', data) @@ -130,21 +183,26 @@ class FBXElem: self.props_type.append(data_types.STRING) self.props.append(data) + def _add_compressed_array_helper(self, data, length): + """Note: This function may be swapped out by enable_multithreading_cm with an equivalent that supports + multithreading.""" + data = zlib.compress(data, 1) + comp_len = len(data) + + encoding = 1 + data = pack('<3I', length, encoding, comp_len) + data + self.props.append(data) + def _add_array_helper(self, data, prop_type, length): + self.props_type.append(prop_type) # mimic behavior of fbxconverter (also common sense) # we could make this configurable. encoding = 0 if len(data) <= 128 else 1 if encoding == 0: - pass + data = pack('<3I', length, encoding, len(data)) + data + self.props.append(data) elif encoding == 1: - data = zlib.compress(data, 1) - - comp_len = len(data) - - data = pack('<3I', length, encoding, comp_len) + data - - self.props_type.append(prop_type) - self.props.append(data) + self._add_compressed_array_helper(data, length) def _add_parray_helper(self, data, array_type, prop_type): assert (isinstance(data, array.array)) diff --git a/io_scene_fbx/export_fbx_bin.py b/io_scene_fbx/export_fbx_bin.py index 2b384cae8..b32423921 100644 --- a/io_scene_fbx/export_fbx_bin.py +++ b/io_scene_fbx/export_fbx_bin.py @@ -3509,31 +3509,35 @@ def save_single(operator, scene, depsgraph, filepath="", # Generate some data about exported scene... scene_data = fbx_data_from_scene(scene, depsgraph, settings) - root = elem_empty(None, b"") # Root element has no id, as it is not saved per se! + # Enable multithreaded array compression in FBXElem and wait until all threads are done before exiting the context + # manager. + with encode_bin.FBXElem.enable_multithreading_cm(): + # Writing elements into an FBX hierarchy can now begin. + root = elem_empty(None, b"") # Root element has no id, as it is not saved per se! - # Mostly FBXHeaderExtension and GlobalSettings. - fbx_header_elements(root, scene_data) + # Mostly FBXHeaderExtension and GlobalSettings. + fbx_header_elements(root, scene_data) - # Documents and References are pretty much void currently. - fbx_documents_elements(root, scene_data) - fbx_references_elements(root, scene_data) + # Documents and References are pretty much void currently. + fbx_documents_elements(root, scene_data) + fbx_references_elements(root, scene_data) - # Templates definitions. - fbx_definitions_elements(root, scene_data) + # Templates definitions. + fbx_definitions_elements(root, scene_data) - # Actual data. - fbx_objects_elements(root, scene_data) + # Actual data. + fbx_objects_elements(root, scene_data) - # How data are inter-connected. - fbx_connections_elements(root, scene_data) + # How data are inter-connected. + fbx_connections_elements(root, scene_data) - # Animation. - fbx_takes_elements(root, scene_data) + # Animation. + fbx_takes_elements(root, scene_data) - # Cleanup! - fbx_scene_data_cleanup(scene_data) + # Cleanup! + fbx_scene_data_cleanup(scene_data) - # And we are down, we can write the whole thing! + # And we are done, all multithreaded tasks are complete, and we can write the whole thing to file! encode_bin.write(filepath, root, FBX_VERSION) # Clear cached ObjectWrappers! -- 2.30.2 From 5aa031177f17a5f94de22a876829b8256434bef1 Mon Sep 17 00:00:00 2001 From: Thomas Barlow Date: Tue, 21 Nov 2023 05:52:36 +0000 Subject: [PATCH 2/5] fix json2fbx.py --- io_scene_fbx/encode_bin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/io_scene_fbx/encode_bin.py b/io_scene_fbx/encode_bin.py index 69e2786fe..d06231985 100644 --- a/io_scene_fbx/encode_bin.py +++ b/io_scene_fbx/encode_bin.py @@ -3,10 +3,9 @@ # SPDX-License-Identifier: GPL-2.0-or-later try: - from . import data_types, fbx_utils + from . import data_types except: import data_types - import fbx_utils from struct import pack from contextlib import contextmanager @@ -65,6 +64,9 @@ class FBXElem: Writing to a file is temporarily disabled as a safeguard.""" # __enter__() + # Can't import fbx_utils at the top of this file because json2fbx.py imports this file, but json2fbx.py can't + # import fbx_utils due to fbx_utils importing bpy. + from . import fbx_utils orig_func = cls._add_compressed_array_helper orig_write = cls._write -- 2.30.2 From d32879355df7c75c5b00778b2dab56ceb8105eb0 Mon Sep 17 00:00:00 2001 From: Thomas Barlow Date: Sat, 2 Dec 2023 23:12:55 +0000 Subject: [PATCH 3/5] Update for multithreading utils being moved to fbx_utils_threading.py --- io_scene_fbx/encode_bin.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/io_scene_fbx/encode_bin.py b/io_scene_fbx/encode_bin.py index d06231985..00fc498e9 100644 --- a/io_scene_fbx/encode_bin.py +++ b/io_scene_fbx/encode_bin.py @@ -4,8 +4,10 @@ try: from . import data_types + from .fbx_utils_threading import MultiThreadedTaskConsumer except: import data_types + from fbx_utils_threading import MultiThreadedTaskConsumer from struct import pack from contextlib import contextmanager @@ -64,9 +66,6 @@ class FBXElem: Writing to a file is temporarily disabled as a safeguard.""" # __enter__() - # Can't import fbx_utils at the top of this file because json2fbx.py imports this file, but json2fbx.py can't - # import fbx_utils due to fbx_utils importing bpy. - from . import fbx_utils orig_func = cls._add_compressed_array_helper orig_write = cls._write @@ -79,7 +78,7 @@ class FBXElem: data = pack('<3I', length, encoding, comp_len) + data props[insert_at] = data - with fbx_utils.MultiThreadedTaskConsumer.new_cpu_bound_cm(insert_compressed_array) as wrapped_func: + with MultiThreadedTaskConsumer.new_cpu_bound_cm(insert_compressed_array) as wrapped_func: try: def _add_compressed_array_helper_multi(self, data, length): # Append a dummy value that will be replaced with the compressed array data later. -- 2.30.2 From 5112fd91f584603d7fbd4fdac60e69b92510b879 Mon Sep 17 00:00:00 2001 From: Thomas Barlow Date: Sat, 2 Dec 2023 23:18:11 +0000 Subject: [PATCH 4/5] Enable multithreaded array compression in json2fbx.py --- io_scene_fbx/json2fbx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/io_scene_fbx/json2fbx.py b/io_scene_fbx/json2fbx.py index 19bb09963..8cbea51ae 100755 --- a/io_scene_fbx/json2fbx.py +++ b/io_scene_fbx/json2fbx.py @@ -133,10 +133,10 @@ def json2fbx(fn): fn_fbx = "%s.fbx" % os.path.splitext(fn)[0] print("Writing: %r " % fn_fbx, end="") - json_root = [] with open(fn) as f_json: json_root = json.load(f_json) - fbx_root, fbx_version = parse_json(json_root) + with encode_bin.FBXElem.enable_multithreading_cm(): + fbx_root, fbx_version = parse_json(json_root) print("(Version %d) ..." % fbx_version) encode_bin.write(fn_fbx, fbx_root, fbx_version) -- 2.30.2 From 1bf83e19bab52ca4ab19608daf717bed8c68873a Mon Sep 17 00:00:00 2001 From: Thomas Barlow Date: Fri, 12 Jan 2024 20:35:49 +0000 Subject: [PATCH 5/5] Increase FBX IO version --- io_scene_fbx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io_scene_fbx/__init__.py b/io_scene_fbx/__init__.py index 7087f1f9a..8f749dc12 100644 --- a/io_scene_fbx/__init__.py +++ b/io_scene_fbx/__init__.py @@ -5,7 +5,7 @@ bl_info = { "name": "FBX format", "author": "Campbell Barton, Bastien Montagne, Jens Restemeier, @Mysteryem", - "version": (5, 11, 3), + "version": (5, 11, 4), "blender": (4, 1, 0), "location": "File > Import-Export", "description": "FBX IO meshes, UVs, vertex colors, materials, textures, cameras, lamps and actions", -- 2.30.2