Render attachments with shortcodes rather than slugs

The attachments should now be rendered using `{attachment slug}` instead
of `@[slug]`. The `link` attribute can be specified in the shortcode
(for attachments that support it), rather than in the attachment itself.

The attachment subdocument is now reduced to `{oid: File ObjectID}`, and
nodes without attachments should NOT have an `attachment` property at
all (previously it would be an empty dict). This makes querying for
nodes with/out attachments easier.

The CLI command `upgrade_attachment_schema` can do dry-run and remove
empty attachments:

- Added --go to actually perform the database changes.
- Remove empty attachments, so that a node either has one or more
  attachments or no attachments sub-document at all.

The CLI command `upgrade_attachment_usage` converts `@[slug]` to
`{attachment slug}`. It also takes into account 'link' and 'link_custom'
fields on the attachment. After conversion those fields are removed from
the attachment itself.

Simplified maintentance CLI commands that iterate over all projects:
I've moved the common approach (either run on one project or all of
them, skipping deleted ones, giving a message upon dry-run, and showing
duration of the command) to a new _db_projects() function. The new
function is now used by two recently-touched CLI commands; more of them
could be migrated to use this.

tests/test_api/test_cli.py

@@ -329,7 +329,7 @@ class UpgradeAttachmentSchemaTest(AbstractNodeReplacementTest):
         group_perms = self.add_group_permission_to_asset_node_type()
 
         with self.app.test_request_context():
-            upgrade_attachment_schema(self.proj['url'])
+            upgrade_attachment_schema(self.proj['url'], go=True)
 
         dbproj = self.fetch_project_from_db()
 

tests/test_cli/test_maintenance.py

@@ -1,6 +1,7 @@
 from bson import ObjectId
 
 from pillar.tests import AbstractPillarTest
+from pillar.tests import common_test_data as ctd
 
 
 class PurgeHomeProjectsTest(AbstractPillarTest):
@@ -33,3 +34,144 @@ class PurgeHomeProjectsTest(AbstractPillarTest):
             proj_coll = self.app.db('projects')
             self.assertEqual(True, proj_coll.find_one({'_id': ObjectId(home_a['_id'])})['_deleted'])
             self.assertEqual(True, proj_coll.find_one({'_id': ObjectId(home_b['_id'])})['_deleted'])
+
+
+class UpgradeAttachmentUsageTest(AbstractPillarTest):
+    def setUp(self, **kwargs):
+        super().setUp(**kwargs)
+        self.pid, self.uid = self.create_project_with_admin(user_id=24 * 'a')
+
+        with self.app.app_context():
+            files_coll = self.app.db('files')
+
+            res = files_coll.insert_one({
+                **ctd.EXAMPLE_FILE,
+                'project': self.pid,
+                'user': self.uid,
+            })
+            self.fid = res.inserted_id
+
+    def test_image_link(self):
+        with self.app.app_context():
+            nodes_coll = self.app.db('nodes')
+            res = nodes_coll.insert_one({
+                **ctd.EXAMPLE_NODE,
+                'picture': self.fid,
+                'project': self.pid,
+                'user': self.uid,
+                'description': "# Title\n\n@[slug0]\n@[slug1]\n@[slug2]\nEitje van Fabergé.",
+                'properties': {
+                    'status': 'published',
+                    'content_type': 'image',
+                    'file': self.fid,
+                    'attachments': {
+                        'slug0': {
+                            'oid': self.fid,
+                            'link': 'self',
+                        },
+                        'slug1': {
+                            'oid': self.fid,
+                            'link': 'custom',
+                            'link_custom': 'https://cloud.blender.org/',
+                        },
+                        'slug2': {
+                            'oid': self.fid,
+                        },
+                    }
+                }
+            })
+            nid = res.inserted_id
+
+        from pillar.cli.maintenance import upgrade_attachment_usage
+
+        with self.app.app_context():
+            upgrade_attachment_usage(proj_url=ctd.EXAMPLE_PROJECT['url'], go=True)
+            node = nodes_coll.find_one({'_id': nid})
+
+            self.assertEqual(
+                "# Title\n\n"
+                "{attachment 'slug0' link='self'}\n"
+                "{attachment 'slug1' link='https://cloud.blender.org/'}\n"
+                "{attachment 'slug2'}\n"
+                "Eitje van Fabergé.",
+                node['description'],
+                'The description should be updated')
+            self.assertEqual(
+                "<h1>Title</h1>\n"
+                "<!-- {attachment 'slug0' link='self'} -->\n"
+                "<!-- {attachment 'slug1' link='https://cloud.blender.org/'} -->\n"
+                "<!-- {attachment 'slug2'} -->\n"
+                "<p>Eitje van Fabergé.</p>\n",
+                node['_description_html'],
+                'The _description_html should be updated')
+
+            self.assertEqual(
+                {'slug0': {'oid': self.fid},
+                 'slug1': {'oid': self.fid},
+                 'slug2': {'oid': self.fid},
+                 },
+                node['properties']['attachments'],
+                'The link should have been removed from the attachment')
+
+    def test_post(self):
+        """This requires checking the dynamic schema of the node."""
+        with self.app.app_context():
+            nodes_coll = self.app.db('nodes')
+            res = nodes_coll.insert_one({
+                **ctd.EXAMPLE_NODE,
+                'node_type': 'post',
+                'project': self.pid,
+                'user': self.uid,
+                'picture': self.fid,
+                'description': "meh",
+                'properties': {
+                    'status': 'published',
+                    'content': "# Title\n\n@[slug0]\n@[slug1]\n@[slug2]\nEitje van Fabergé.",
+                    'attachments': {
+                        'slug0': {
+                            'oid': self.fid,
+                            'link': 'self',
+                        },
+                        'slug1': {
+                            'oid': self.fid,
+                            'link': 'custom',
+                            'link_custom': 'https://cloud.blender.org/',
+                        },
+                        'slug2': {
+                            'oid': self.fid,
+                        },
+                    }
+                }
+            })
+            nid = res.inserted_id
+
+        from pillar.cli.maintenance import upgrade_attachment_usage
+
+        with self.app.app_context():
+            upgrade_attachment_usage(proj_url=ctd.EXAMPLE_PROJECT['url'], go=True)
+            node = nodes_coll.find_one({'_id': nid})
+
+            self.assertEqual(
+                "# Title\n\n"
+                "{attachment 'slug0' link='self'}\n"
+                "{attachment 'slug1' link='https://cloud.blender.org/'}\n"
+                "{attachment 'slug2'}\n"
+                "Eitje van Fabergé.",
+                node['properties']['content'],
+                'The content should be updated')
+            self.assertEqual(
+                "<h1>Title</h1>\n"
+                "<!-- {attachment 'slug0' link='self'} -->\n"
+                "<!-- {attachment 'slug1' link='https://cloud.blender.org/'} -->\n"
+                "<!-- {attachment 'slug2'} -->\n"
+                "<p>Eitje van Fabergé.</p>\n",
+                node['properties']['_content_html'],
+                'The _content_html should be updated')
+
+            self.assertEqual(
+                {'slug0': {'oid': self.fid},
+                 'slug1': {'oid': self.fid},
+                 'slug2': {'oid': self.fid},
+                 },
+                node['properties']['attachments'],
+                'The link should have been removed from the attachment')