LibOverride: auto-run resync process on file reading.

`BKE_lib_override_library_main_resync` uses
`LIB_TAG_LIB_OVERRIDE_NEED_RESYNC` tags set by RNA override apply code,
and perform detection for the remaining cases (those were new overrides
need to be created for data that was not present before in the library).

And then it actually resync all needed local overrides.

source/blender/blenkernel/BKE_lib_override.h

@@ -61,6 +61,8 @@ void BKE_lib_override_library_copy(struct ID *dst_id,
 void BKE_lib_override_library_clear(struct IDOverrideLibrary *override, const bool do_id_user);
 void BKE_lib_override_library_free(struct IDOverrideLibrary **override, const bool do_id_user);
 
+bool BKE_lib_override_library_is_user_edited(struct ID *id);
+
 struct ID *BKE_lib_override_library_create_from_id(struct Main *bmain,
                                                    struct ID *reference_id,
                                                    const bool do_tagged_remap);
@@ -78,6 +80,10 @@ bool BKE_lib_override_library_resync(struct Main *bmain,
                                      struct Scene *scene,
                                      struct ViewLayer *view_layer,
                                      struct ID *id_root);
+void BKE_lib_override_library_main_resync(struct Main *bmain,
+                                          struct Scene *scene,
+                                          struct ViewLayer *view_layer);
+
 void BKE_lib_override_library_delete(struct Main *bmain, struct ID *id_root);
 
 struct IDOverrideLibraryProperty *BKE_lib_override_library_property_find(

source/blender/blenkernel/intern/blendfile.c

@@ -51,6 +51,7 @@
 #include "BKE_keyconfig.h"
 #include "BKE_layer.h"
 #include "BKE_lib_id.h"
+#include "BKE_lib_override.h"
 #include "BKE_main.h"
 #include "BKE_preferences.h"
 #include "BKE_report.h"
@@ -401,6 +402,13 @@ static void setup_app_data(bContext *C,
      * to recompute refcount for all local IDs too. */
     BKE_main_id_refcount_recompute(bmain, false);
   }
+
+  if (mode != LOAD_UNDO) {
+    BKE_lib_override_library_main_resync(
+        bmain,
+        curscene,
+        bfd->cur_view_layer ? bfd->cur_view_layer : BKE_view_layer_default_view(curscene));
+  }
 }
 
 static void setup_app_blend_file_data(bContext *C,

source/blender/blenkernel/intern/lib_override.c

@@ -213,6 +213,32 @@ static ID *lib_override_library_create_from(Main *bmain, ID *reference_id)
   return local_id;
 }
 
+/** Check if given ID has some override rules that actually indicate the user edited it.
+ *
+ * TODO: This could be simplified by storing a flag in IDOverrideLibrary during the diffing
+ * process? */
+bool BKE_lib_override_library_is_user_edited(struct ID *id)
+{
+  if (!ID_IS_OVERRIDE_LIBRARY(id)) {
+    return false;
+  }
+
+  LISTBASE_FOREACH (IDOverrideLibraryProperty *, op, &id->override_library->properties) {
+    LISTBASE_FOREACH (IDOverrideLibraryPropertyOperation *, opop, &op->operations) {
+      if ((opop->flag & IDOVERRIDE_LIBRARY_FLAG_IDPOINTER_MATCH_REFERENCE) != 0) {
+        continue;
+      }
+      if (opop->operation == IDOVERRIDE_LIBRARY_OP_NOOP) {
+        continue;
+      }
+      /* If an operation does not match the filters above, it is considered as a user-editing one,
+       * therefore this override is user-edited. */
+      return true;
+    }
+  }
+  return false;
+}
+
 /** Create an overridden local copy of linked reference. */
 ID *BKE_lib_override_library_create_from_id(Main *bmain,
                                             ID *reference_id,
@@ -379,7 +405,7 @@ typedef struct LibOverrideGroupTagData {
  *
  * Requires existing `Main.relations`.
  *
- * Note: this is typically called to complete `lib_override_linked_group_tag()`.
+ * NOTE: This is typically called to complete `lib_override_linked_group_tag()`.
  */
 static bool lib_override_hierarchy_dependencies_recursive_tag(LibOverrideGroupTagData *data)
 {
@@ -879,6 +905,8 @@ bool BKE_lib_override_library_resync(Main *bmain, Scene *scene, ViewLayer *view_
         ID *id_override_new = id->newid;
         ID *id_override_old = BLI_ghash_lookup(linkedref_to_old_override, id);
 
+        BLI_assert((id_override_new->tag & LIB_TAG_LIB_OVERRIDE_NEED_RESYNC) == 0);
+
         if (id_override_old != NULL) {
           /* Swap the names between old override ID and new one. */
           char id_name_buf[MAX_ID_NAME];
@@ -971,11 +999,23 @@ bool BKE_lib_override_library_resync(Main *bmain, Scene *scene, ViewLayer *view_
       }
       id->tag &= ~LIB_TAG_DOIT;
     }
-    /* Also cleanup old overrides that went missing in new linked data. */
+    /* Also deal with old overrides that went missing in new linked data. */
     else if (id->tag & LIB_TAG_MISSING && !ID_IS_LINKED(id)) {
       BLI_assert(ID_IS_OVERRIDE_LIBRARY(id));
-      id->tag |= LIB_TAG_DOIT;
-      id->tag &= ~LIB_TAG_MISSING;
+      if (!BKE_lib_override_library_is_user_edited(id)) {
+        /* If user never edited them, we can delete them. */
+        id->tag |= LIB_TAG_DOIT;
+        id->tag &= ~LIB_TAG_MISSING;
+        printf("%s: Old override %s is being deleted.\n", __func__, id->name);
+      }
+      else {
+        /* Otherwise, keep them, user needs to decide whether what to do with them. */
+        BLI_assert((id->tag & LIB_TAG_DOIT) == 0);
+        id_fake_user_set(id);
+        printf("%s: Old override %s is being kept around as it was user-edited.\n",
+               __func__,
+               id->name);
+      }
     }
   }
   FOREACH_MAIN_ID_END;
@@ -1002,6 +1042,191 @@ bool BKE_lib_override_library_resync(Main *bmain, Scene *scene, ViewLayer *view_
   return success;
 }
 
+/**
+ * Detect and handle required resync of overrides data, when relations between reference linked IDs
+ * have changed.
+ *
+ * This is a fairly complex and costly operation, typically it should be called after
+ * #BKE_lib_override_library_main_update, which would already detect and tag a lot of cases.
+ *
+ * This function will first detect the remaining cases requiring a resync (namely, either when an
+ * existing linked ID that did not require to be overridden before now would be, or when new IDs
+ * are added to the hierarchy).
+ *
+ * Then it will handle the resync of necessary IDs (through calls to
+ * #BKE_lib_override_library_resync).
+ */
+void BKE_lib_override_library_main_resync(Main *bmain, Scene *scene, ViewLayer *view_layer)
+{
+  BKE_main_relations_create(bmain, 0);
+  BKE_main_id_tag_all(bmain, LIB_TAG_DOIT, false);
+
+  /* Detect all linked data that would need to be overridden if we had to create an override from
+   * those used by current existing overrides. */
+
+  /* We check collections and objects first, as in most typical usecases those are the 'root' of an
+   * override hierarchy. */
+#if 0
+  LISTBASE_FOREACH (ID *, id, &bmain->collections) {
+    if (!ID_IS_OVERRIDE_LIBRARY_REAL(id)) {
+      continue;
+    }
+    if (id->tag & (LIB_TAG_DOIT | LIB_TAG_MISSING)) {
+      /* We already processed that ID as part of another ID's hierarchy. */
+      continue;
+    }
+
+    LibOverrideGroupTagData data = {.bmain = bmain,
+                                    .id_root = id->override_library->reference,
+                                    .tag = LIB_TAG_DOIT,
+                                    .missing_tag = LIB_TAG_MISSING};
+    lib_override_linked_group_tag(&data);
+    BKE_main_relations_tag_set(bmain, MAINIDRELATIONS_ENTRY_TAGS_PROCESSED, false);
+    lib_override_hierarchy_dependencies_recursive_tag(&data);
+    BKE_main_relations_tag_set(bmain, MAINIDRELATIONS_ENTRY_TAGS_PROCESSED, false);
+  }
+  LISTBASE_FOREACH (ID *, id, &bmain->objects) {
+    if (!ID_IS_OVERRIDE_LIBRARY_REAL(id)) {
+      continue;
+    }
+    if (id->tag & (LIB_TAG_DOIT | LIB_TAG_MISSING)) {
+      /* We already processed that ID as part of another ID's hierarchy. */
+      continue;
+    }
+
+    LibOverrideGroupTagData data = {.bmain = bmain,
+                                    .id_root = id->override_library->reference,
+                                    .tag = LIB_TAG_DOIT,
+                                    .missing_tag = LIB_TAG_MISSING};
+    lib_override_linked_group_tag(&data);
+    BKE_main_relations_tag_set(bmain, MAINIDRELATIONS_ENTRY_TAGS_PROCESSED, false);
+    lib_override_hierarchy_dependencies_recursive_tag(&data);
+    BKE_main_relations_tag_set(bmain, MAINIDRELATIONS_ENTRY_TAGS_PROCESSED, false);
+  }
+#endif
+  {
+    ID *id;
+    FOREACH_MAIN_ID_BEGIN (bmain, id) {
+      if (ELEM(GS(id->name), ID_GR, ID_OB)) {
+        //        continue;
+      }
+      if (!ID_IS_OVERRIDE_LIBRARY_REAL(id)) {
+        continue;
+      }
+      if (id->tag & (LIB_TAG_DOIT | LIB_TAG_MISSING)) {
+        /* We already processed that ID as part of another ID's hierarchy. */
+        continue;
+      }
+
+      LibOverrideGroupTagData data = {.bmain = bmain,
+                                      .id_root = id->override_library->reference,
+                                      .tag = LIB_TAG_DOIT,
+                                      .missing_tag = LIB_TAG_MISSING};
+      lib_override_linked_group_tag(&data);
+      BKE_main_relations_tag_set(bmain, MAINIDRELATIONS_ENTRY_TAGS_PROCESSED, false);
+      lib_override_hierarchy_dependencies_recursive_tag(&data);
+      BKE_main_relations_tag_set(bmain, MAINIDRELATIONS_ENTRY_TAGS_PROCESSED, false);
+    }
+    FOREACH_MAIN_ID_END;
+  }
+
+  /* Now check existing overrides, those needing resync will be the one either already tagged as
+   * such, or the one using linked data that is now tagged as needing override. */
+
+  {
+    ID *id;
+    FOREACH_MAIN_ID_BEGIN (bmain, id) {
+      if (!ID_IS_OVERRIDE_LIBRARY_REAL(id)) {
+        continue;
+      }
+
+      if (id->tag & LIB_TAG_LIB_OVERRIDE_NEED_RESYNC) {
+        printf("%s: ID %s was already tagged as needing resync.\n", __func__, id->name);
+        continue;
+      }
+
+      MainIDRelationsEntry *entry = BLI_ghash_lookup(bmain->relations->relations_from_pointers,
+                                                     id);
+      BLI_assert(entry != NULL);
+
+      for (MainIDRelationsEntryItem *entry_item = entry->to_ids; entry_item != NULL;
+           entry_item = entry_item->next) {
+        if (entry_item->usage_flag &
+            (IDWALK_CB_EMBEDDED | IDWALK_CB_LOOPBACK | IDWALK_CB_OVERRIDE_LIBRARY_REFERENCE)) {
+          continue;
+        }
+        ID *id_to = *entry_item->id_pointer.to;
+
+        /* Case where this ID pointer was to a linked ID, that now needs to be overridden. */
+        if (ID_IS_LINKED(id_to) && (id_to->tag & LIB_TAG_DOIT) != 0) {
+          id->tag |= LIB_TAG_LIB_OVERRIDE_NEED_RESYNC;
+          printf(
+              "%s: ID %s now tagged as needing resync because they use linked %s that now needs "
+              "to be overridden.\n",
+              __func__,
+              id->name,
+              id_to->name);
+          break;
+        }
+      }
+    }
+    FOREACH_MAIN_ID_END;
+  }
+
+  BKE_main_relations_free(bmain);
+  BKE_main_id_tag_all(bmain, LIB_TAG_DOIT, false);
+
+  bool do_loop = true;
+#if 0
+  while (do_loop) {
+    do_loop = false;
+    LISTBASE_FOREACH (ID *, id, &bmain->collections) {
+      if ((id->tag & LIB_TAG_LIB_OVERRIDE_NEED_RESYNC) == 0) {
+        continue;
+      }
+      do_loop = true;
+      printf("%s: Resynced %s, ", __func__, id->name);
+      const bool success = BKE_lib_override_library_resync(bmain, scene, view_layer, id);
+      printf("success: %d\n", success);
+      break;
+    }
+  }
+  do_loop = true;
+  while (do_loop) {
+    do_loop = false;
+    LISTBASE_FOREACH (ID *, id, &bmain->objects) {
+      if ((id->tag & LIB_TAG_LIB_OVERRIDE_NEED_RESYNC) == 0) {
+        continue;
+      }
+      do_loop = true;
+      printf("%s: Resynced %s, ", __func__, id->name);
+      const bool success = BKE_lib_override_library_resync(bmain, scene, view_layer, id);
+      printf("success: %d\n", success);
+      break;
+    }
+  }
+#endif
+  do_loop = true;
+  while (do_loop) {
+    do_loop = false;
+    ID *id;
+    FOREACH_MAIN_ID_BEGIN (bmain, id) {
+      if (ELEM(GS(id->name), ID_GR, ID_OB)) {
+        //        continue;
+      }
+      if ((id->tag & LIB_TAG_LIB_OVERRIDE_NEED_RESYNC) == 0) {
+        continue;
+      }
+      do_loop = true;
+      printf("%s: Resynced %s, ", __func__, id->name);
+      const bool success = BKE_lib_override_library_resync(bmain, scene, view_layer, id);
+      printf("success: %d\n", success);
+      break;
+    }
+    FOREACH_MAIN_ID_END;
+  }
+}
+
 /**
  * Advanced 'smart' function to delete library overrides (including their existing override
  * hierarchy) and remap their usages to their linked reference IDs.
@@ -1880,6 +2105,14 @@ void BKE_lib_override_library_main_unused_cleanup(struct Main *bmain)
   FOREACH_MAIN_ID_END;
 }
 
+static void lib_override_id_swap(Main *bmain, ID *id_local, ID *id_temp)
+{
+  BKE_lib_id_swap(bmain, id_local, id_temp);
+  /* We need to keep these tags from temp ID into orig one.
+   * ID swap does not swap most of ID data itself. */
+  id_local->tag |= (id_temp->tag & LIB_TAG_LIB_OVERRIDE_NEED_RESYNC);
+}
+
 /** Update given override from its reference (re-applying overridden properties). */
 void BKE_lib_override_library_update(Main *bmain, ID *local)
 {
@@ -1948,11 +2181,11 @@ void BKE_lib_override_library_update(Main *bmain, ID *local)
 
   /* This also transfers all pointers (memory) owned by local to tmp_id, and vice-versa.
    * So when we'll free tmp_id, we'll actually free old, outdated data from local. */
-  BKE_lib_id_swap(bmain, local, tmp_id);
+  lib_override_id_swap(bmain, local, tmp_id);
 
   if (local_key != NULL && tmp_key != NULL) {
     /* This is some kind of hard-coded 'always enforced override'. */
-    BKE_lib_id_swap(bmain, &local_key->id, &tmp_key->id);
+    lib_override_id_swap(bmain, &local_key->id, &tmp_key->id);
     tmp_key->id.flag |= (local_key->id.flag & LIB_EMBEDDED_DATA_LIB_OVERRIDE);
     /* The swap of local and tmp_id inverted those pointers, we need to redefine proper
      * relationships. */

source/blender/makesdna/DNA_ID.h

@@ -566,6 +566,11 @@ enum {
   /* RESET_AFTER_USE Used by undo system to tag unchanged IDs re-used from old Main (instead of
    * read from memfile). */
   LIB_TAG_UNDO_OLD_ID_REUSED = 1 << 19,
+
+  /**
+   * The data-block is a library override that needs re-sync to its linked reference.
+   */
+  LIB_TAG_LIB_OVERRIDE_NEED_RESYNC = 1 << 13,
 };
 
 /* Tag given ID for an update in all the dependency graphs. */

source/blender/makesrna/intern/rna_access_compare_override.c

@@ -57,7 +57,7 @@
  * Find the actual ID owner of the given \a ptr #PointerRNA, in override sense, and generate the
  * full rna path from it to given \a prop #PropertyRNA if \a rna_path is given.
  *
- * \note this is slightly different than 'generic' RNA 'id owner' as returned by
+ * \note This is slightly different than 'generic' RNA 'id owner' as returned by
  * #RNA_find_real_ID_and_path, since in overrides we also consider shape keys as embedded data, not
  * only root node trees and master collections.
  */
@@ -100,10 +100,6 @@ static ID *rna_property_override_property_real_id_owner(Main *bmain,
     }
   }
 
-  if (!ID_IS_OVERRIDE_LIBRARY_REAL(owner_id)) {
-    return NULL;
-  }
-
   if (r_rna_path == NULL) {
     return owner_id;
   }
@@ -1153,6 +1149,38 @@ void RNA_struct_override_apply(Main *bmain,
               ptr_storage, op->rna_path, &data_storage, &prop_storage, &data_item_storage);
         }
 
+        /* Check if an overridden ID pointer supposed to be in sync with linked data gets out of
+         * sync. */
+        if ((ptr_dst->owner_id->tag & LIB_TAG_LIB_OVERRIDE_NEED_RESYNC) == 0 &&
+            op->rna_prop_type == PROP_POINTER &&
+            (((IDOverrideLibraryPropertyOperation *)op->operations.first)->flag &
+             IDOVERRIDE_LIBRARY_FLAG_IDPOINTER_MATCH_REFERENCE) != 0) {
+          BLI_assert(ptr_src->owner_id ==
+                     rna_property_override_property_real_id_owner(bmain, &data_src, NULL, NULL));
+          BLI_assert(ptr_dst->owner_id ==
+                     rna_property_override_property_real_id_owner(bmain, &data_dst, NULL, NULL));
+
+          PointerRNA prop_ptr_src = RNA_property_pointer_get(&data_src, prop_src);
+          PointerRNA prop_ptr_dst = RNA_property_pointer_get(&data_dst, prop_dst);
+          ID *id_src = rna_property_override_property_real_id_owner(
+              bmain, &prop_ptr_src, NULL, NULL);
+          ID *id_dst = rna_property_override_property_real_id_owner(
+              bmain, &prop_ptr_dst, NULL, NULL);
+
+          BLI_assert(id_src == NULL || ID_IS_OVERRIDE_LIBRARY_REAL(id_src));
+
+          if (/* We might be in a case where id_dst has already been processed and its usages
+               * remapped to its new local override. In that case overrides and linked data are
+               * always properly matching. */
+              id_src != id_dst &&
+              /* If one of the pointers is NULL and not the other, or if linked reference ID of
+               * `id_src` is not `id_dst`,  we are in a non-matching case. */
+              (ELEM(NULL, id_src, id_dst) || id_src->override_library->reference != id_dst)) {
+            ptr_dst->owner_id->tag |= LIB_TAG_LIB_OVERRIDE_NEED_RESYNC;
+            printf("Local override %s detected as needing resync!\n", ptr_dst->owner_id->name);
+          }
+        }
+
         rna_property_override_apply_ex(bmain,
                                        &data_dst,
                                        &data_src,

source/blender/makesrna/intern/rna_rna.c

@@ -1238,6 +1238,8 @@ static bool rna_property_override_diff_propptr_validate_diffing(PointerRNA *prop
 
 /* Used for both Pointer and Collection properties. */
 static int rna_property_override_diff_propptr(Main *bmain,
+                                              ID *owner_id_a,
+                                              ID *owner_id_b,
                                               PointerRNA *propptr_a,
                                               PointerRNA *propptr_b,
                                               eRNACompareMode mode,
@@ -1359,6 +1361,17 @@ static int rna_property_override_diff_propptr(Main *bmain,
                * override is not matching its reference anymore. */
               opop->flag &= ~IDOVERRIDE_LIBRARY_FLAG_IDPOINTER_MATCH_REFERENCE;
             }
+            else if ((owner_id_a->tag & LIB_TAG_LIB_OVERRIDE_NEED_RESYNC) != 0 ||
+                     (owner_id_b->tag & LIB_TAG_LIB_OVERRIDE_NEED_RESYNC) != 0) {
+              /* In case one of the owner of the checked property is tagged as needing resync, do
+               * not change the 'match reference' status of its ID pointer properties overrides,
+               * since many non-matching ones are likely due to missing resync. */
+              printf(
+                  "%s: Not checking matching ID pointer properties, since owner %s is tagged as "
+                  "needing resync.\n",
+                  __func__,
+                  id_a->name);
+            }
             else if (id_a->override_library != NULL && id_a->override_library->reference == id_b) {
               opop->flag |= IDOVERRIDE_LIBRARY_FLAG_IDPOINTER_MATCH_REFERENCE;
             }
@@ -1778,6 +1791,8 @@ int rna_property_override_diff_default(Main *bmain,
         PointerRNA propptr_a = RNA_property_pointer_get(ptr_a, rawprop_a);
         PointerRNA propptr_b = RNA_property_pointer_get(ptr_b, rawprop_b);
         return rna_property_override_diff_propptr(bmain,
+                                                  ptr_a->owner_id,
+                                                  ptr_b->owner_id,
                                                   &propptr_a,
                                                   &propptr_b,
                                                   mode,
@@ -1934,6 +1949,8 @@ int rna_property_override_diff_default(Main *bmain,
           else if (is_id || is_valid_for_diffing) {
             if (equals || do_create) {
               const int eq = rna_property_override_diff_propptr(bmain,
+                                                                ptr_a->owner_id,
+                                                                ptr_b->owner_id,
                                                                 &iter_a.ptr,
                                                                 &iter_b.ptr,
                                                                 mode,