From d3ff88e5cfa44a84b15fd6f579ef493bc612764d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= <sybren@stuvel.eu>
Date: Tue, 27 Mar 2018 17:56:38 +0200
Subject: [PATCH] Also replace node types when key with underscore changed

Previously all keys starting with an underscore were ignored (so changes
to _created wouldn't count as "different"), but this clashes with saving
Markdown output to _xxx_html keys.
---
 pillar/api/utils/__init__.py |  6 ++++--
 pillar/cli/maintenance.py    | 11 +++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/pillar/api/utils/__init__.py b/pillar/api/utils/__init__.py
index 075d3f8a..4ff46af0 100644
--- a/pillar/api/utils/__init__.py
+++ b/pillar/api/utils/__init__.py
@@ -172,7 +172,7 @@ def doc_diff(doc1, doc2, *, falsey_is_equal=True, superkey: str = None):
 
     Yields changes as (key, value in doc1, value in doc2) tuples, where
     the value can also be the DoesNotExist class. Does not report changed
-    private keys (i.e. starting with underscores).
+    private keys (i.e. the standard Eve keys starting with underscores).
 
     Sub-documents (i.e. dicts) are recursed, and dot notation is used
     for the keys if changes are found.
@@ -181,6 +181,8 @@ def doc_diff(doc1, doc2, *, falsey_is_equal=True, superkey: str = None):
     function won't report differences between DoesNotExist, False, '', and 0.
     """
 
+    private_keys = {'_id', '_etag', '_deleted', '_updated', '_created'}
+
     def combine_key(some_key):
         """Combine this key with the superkey.
 
@@ -200,7 +202,7 @@ def doc_diff(doc1, doc2, *, falsey_is_equal=True, superkey: str = None):
 
     if isinstance(doc1, dict) and isinstance(doc2, dict):
         for key in set(doc1.keys()).union(set(doc2.keys())):
-            if isinstance(key, str) and key[0] == '_':
+            if key in private_keys:
                 continue
 
             val1 = doc1.get(key, DoesNotExist)
diff --git a/pillar/cli/maintenance.py b/pillar/cli/maintenance.py
index 5cb45f13..b6318824 100644
--- a/pillar/cli/maintenance.py
+++ b/pillar/cli/maintenance.py
@@ -484,7 +484,13 @@ def replace_pillar_node_type_schemas(project_url=None, all_projects=False, missi
     projects_collection = current_app.db()['projects']
     will_would = 'Will' if go else 'Would'
 
+    projects_changed = projects_seen = 0
+
     def handle_project(proj):
+        nonlocal projects_changed, projects_seen
+
+        projects_seen += 1
+
         orig_proj = copy.deepcopy(proj)
         proj_id = proj['_id']
         if 'url' not in proj:
@@ -526,6 +532,8 @@ def replace_pillar_node_type_schemas(project_url=None, all_projects=False, missi
                 proj_has_difference = True
             log.info('    %30r: %r → %r', key, val1, val2)
 
+        projects_changed += proj_has_difference
+
         if go and proj_has_difference:
             # Use Eve to PUT, so we have schema checking.
             db_proj = remove_private_keys(proj)
@@ -546,6 +554,9 @@ def replace_pillar_node_type_schemas(project_url=None, all_projects=False, missi
     if all_projects:
         for project in projects_collection.find({'_deleted': {'$ne': True}}):
             handle_project(project)
+        log.info('%s %d of %d projects',
+                 'Changed' if go else 'Would change',
+                 projects_changed, projects_seen)
         return
 
     if project_url: