Fix #104576: FBX export of Edit Mode Objects while Blender is not in Edit Mode can fail. #104633

Merged
2 changed files with 71 additions and 1 deletions

View File

@ -77,7 +77,7 @@ from .fbx_utils import (
# Animation.
AnimationCurveNodeWrapper,
# Objects.
ObjectWrapper, fbx_name_class,
ObjectWrapper, fbx_name_class, ensure_object_not_in_edit_mode,
# Top level.
FBXExportSettingsMedia, FBXExportSettings, FBXExportData,
)
@ -3497,6 +3497,14 @@ def save(operator, context,
ctx_objects = context.view_layer.objects
if use_visible:
ctx_objects = tuple(obj for obj in ctx_objects if obj.visible_get())
# Ensure no Objects are in Edit mode.
# Copy to a tuple for safety, to avoid the risk of modifying ctx_objects while iterating.
for obj in tuple(ctx_objects):

Should rather return {'CANCELLED'} (same below), since nothing happened.

Should rather return `{'CANCELLED'}` (same below), since nothing happened.

I chose returning {'FINISHED'} because if the Operator is run from Edit mode, the current mode is changed to Object mode (it would normally be changed back to the original mode at the end of the operator though this does not guarantee the same state as when starting). Similarly, ensure_object_not_in_edit_mode may have changed the mode of other Objects before finding an Object which could not be set out of Edit mode.

Even if I were to keep track of all Objects whose mode was changed, there's no guarantee that their original modes can be restored. Notably, Objects whose hide_viewport is True can exit Edit mode, but cannot enter it.

Another case is where multiple Objects are in multi-editing Edit mode at the start of the Operator. Changing to Object mode will affect all Objects in multi-editing regardless of their selection state, but changing back to Edit mode afterwards will only open selected Objects into Edit mode.

Returning {'FINISHED'} pushes an undo step so that the state from before the operator was called can be returned to easily.

If the potential changes to the modes of Objects are not considered important then I would agree with returning {'CANCELLED'}.

I chose returning `{'FINISHED'}` because if the Operator is run from Edit mode, the current mode is changed to Object mode (it would normally be changed back to the original mode at the end of the operator though this does not guarantee the same state as when starting). Similarly, `ensure_object_not_in_edit_mode` may have changed the mode of other Objects before finding an Object which could not be set out of Edit mode. Even if I were to keep track of all Objects whose mode was changed, there's no guarantee that their original modes can be restored. Notably, Objects whose `hide_viewport` is True can exit Edit mode, but cannot enter it. Another case is where multiple Objects are in multi-editing Edit mode at the start of the Operator. Changing to Object mode will affect all Objects in multi-editing regardless of their selection state, but changing back to Edit mode afterwards will only open selected Objects into Edit mode. Returning `{'FINISHED'}` pushes an undo step so that the state from before the operator was called can be returned to easily. If the potential changes to the modes of Objects are not considered important then I would agree with returning `{'CANCELLED'}`.

Yes, I think mode changes are not enough to justify a FINISHED return here.

The current situation is not fully ideal, but not very common either imho. If we wanted to be strict and bullet-proofed, I would just forbid export when any exported object is in Edit mode, but think it's more user-friendly to try to switch back to Object mode as much as we can. And accept that in some very rare, weird cases we will fail to do so and user will have to fix their scene manually before exporting.

Yes, I think mode changes are not enough to justify a `FINISHED` return here. The current situation is not fully ideal, but not very common either imho. If we wanted to be strict and bullet-proofed, I would just forbid export when any exported object is in Edit mode, but think it's more user-friendly to try to switch back to Object mode as much as we can. And accept that in some very rare, weird cases we will fail to do so and user will have to fix their scene manually before exporting.
if not ensure_object_not_in_edit_mode(context, obj):
operator.report({'ERROR'}, "%s could not be set out of Edit Mode, so cannot be exported" % obj.name)
return {'CANCELLED'}
kwargs_mod["context_objects"] = ctx_objects
depsgraph = context.evaluated_depsgraph_get()
@ -3528,6 +3536,16 @@ def save(operator, context,
else:
data_seq = tuple((scene, scene.name, 'objects') for scene in bpy.data.scenes if scene.objects)
# Ensure no Objects are in Edit mode.
for data, data_name, data_obj_propname in data_seq:
# Copy to a tuple for safety, to avoid the risk of modifying the data prop while iterating it.
for obj in tuple(getattr(data, data_obj_propname)):
if not ensure_object_not_in_edit_mode(context, obj):
operator.report({'ERROR'},
"%s in %s could not be set out of Edit Mode, so cannot be exported"
% (obj.name, data_name))
return {'CANCELLED'}
# call this function within a loop with BATCH_ENABLE == False
new_fbxpath = fbxpath # own dir option modifies, we need to keep an original

View File

@ -540,6 +540,58 @@ def fast_first_axis_unique(ar, return_unique=True, return_index=False, return_in
return result
def ensure_object_not_in_edit_mode(context, obj):
"""Objects in Edit mode usually cannot be exported because much of the API used when exporting is not available for
Objects in Edit mode.
Exiting the currently active Object (and any other Objects opened in multi-editing) from Edit mode is simple and
should be done with `bpy.ops.mesh.mode_set(mode='OBJECT')` instead of using this function.
This function is for the rare case where an Object is in Edit mode, but the current context mode is not Edit mode.
This can occur from a state where the current context mode is Edit mode, but then the active Object of the current
View Layer is changed to a different Object that is not in Edit mode. This changes the current context mode, but
leaves the other Object(s) in Edit mode.
"""
if obj.mode != 'EDIT':
return True
# Get the active View Layer.
view_layer = context.view_layer
# A View Layer belongs to a scene.
scene = view_layer.id_data
# Get the current active Object of this View Layer, so we can restore it once done.
orig_active = view_layer.objects.active
# Check if obj is in the View Layer. If obj is not in the View Layer, it cannot be set as the active Object.
# We don't use `obj.name in view_layer.objects` because an Object from a Library could have the same name.
is_in_view_layer = any(o == obj for o in view_layer.objects)
do_unlink_from_scene_collection = False
try:
if not is_in_view_layer:
# There might not be any enabled collections in the View Layer, so link obj into the Scene Collection
# instead, which is always available to all View Layers of that Scene.
scene.collection.objects.link(obj)
do_unlink_from_scene_collection = True
view_layer.objects.active = obj
# Now we're finally ready to attempt to change obj's mode.
if bpy.ops.object.mode_set.poll():
bpy.ops.object.mode_set(mode='OBJECT')
if obj.mode == 'EDIT':
# The Object could not be set out of EDIT mode and therefore cannot be exported.
return False
finally:
# Always restore the original active Object and unlink obj from the Scene Collection if it had to be linked.
view_layer.objects.active = orig_active
if do_unlink_from_scene_collection:
scene.collection.objects.unlink(obj)
return True
# ##### UIDs code. #####
# ID class (mere int).