Cache Markdown'ed HTML in database

This is done via coercion rules. To cache the field 'content' in the
database, include this in your Eve schema:

    {'content': {'type': 'string', 'coerce': 'markdown'},
     '_content_html': {'type': 'string'}}

The `_content_html` field will be filled automatically when saving the
document via Eve.

To display the cached HTML, and fall back to display-time rendering if it
is not there, use `{{ document | markdowned('content') }}` in your template.

Still needs unit testing, a CLI command for regenerating the caches, and
a CLI command for migrating the node type definitions in existing projects.

pillar/api/custom_field_validation.py

@@ -1,10 +1,13 @@
 import logging
 
 from bson import ObjectId, tz_util
-from datetime import datetime, tzinfo
+from datetime import datetime
+import cerberus.errors
 from eve.io.mongo import Validator
 from flask import current_app
 
+import pillar.markdown
+
 log = logging.getLogger(__name__)
 
 
@@ -152,3 +155,52 @@ class ValidateCustomFields(Validator):
 
         if not isinstance(value, (bytes, bytearray)):
             self._error(field_name, f'wrong value type {type(value)}, expected bytes or bytearray')
+
+    def _validate_coerce(self, coerce, field: str, value):
+        """Override Cerberus' _validate_coerce method for richer features.
+
+        This now supports named coercion functions (available in Cerberus 1.0+)
+        and passes the field name to coercion functions as well.
+        """
+        if isinstance(coerce, str):
+            coerce = getattr(self, f'_normalize_coerce_{coerce}')
+
+        try:
+            return coerce(field, value)
+        except (TypeError, ValueError):
+            self._error(field, cerberus.errors.ERROR_COERCION_FAILED.format(field))
+
+    def _normalize_coerce_markdown(self, field: str, value):
+        """Render Markdown from this field into {field}_html.
+
+        The field name MUST NOT end in `_html`. The Markdown is read from this
+        field and the rendered HTML is written to the field `{field}_html`.
+        """
+        html = pillar.markdown.markdown(value)
+        field_name = pillar.markdown.cache_field_name(field)
+        self.current[field_name] = html
+        return value
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+
+    v = ValidateCustomFields()
+    v.schema = {
+        'foo': {'type': 'string', 'coerce': 'markdown'},
+        'foo_html': {'type': 'string'},
+        'nested': {
+            'type': 'dict',
+            'schema': {
+                'bar': {'type': 'string', 'coerce': 'markdown'},
+                'bar_html': {'type': 'string'},
+            }
+        }
+    }
+    print('Valid   :', v.validate({
+        'foo': '# Title\n\nHeyyyy',
+        'nested': {'bar': 'bhahaha'},
+    }))
+    print('Document:')
+    pprint(v.document)
+    print('Errors  :', v.errors)

pillar/api/eve_settings.py

@@ -155,7 +155,9 @@ organizations_schema = {
     'description': {
         'type': 'string',
         'maxlength': 256,
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     'website': {
         'type': 'string',
         'maxlength': 256,
@@ -290,7 +292,9 @@ nodes_schema = {
     },
     'description': {
         'type': 'string',
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     'picture': _file_embedded_schema,
     'order': {
         'type': 'integer',
@@ -535,7 +539,9 @@ projects_schema = {
     },
     'description': {
         'type': 'string',
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     # Short summary for the project
     'summary': {
         'type': 'string',

pillar/api/node_types/comment.py

@@ -2,16 +2,14 @@ node_type_comment = {
     'name': 'comment',
     'description': 'Comments for asset nodes, pages, etc.',
     'dyn_schema': {
-        # The actual comment content (initially Markdown format)
+        # The actual comment content
         'content': {
             'type': 'string',
             'minlength': 5,
             'required': True,
+            'coerce': 'markdown',
         },
-        # The converted-to-HTML content.
-        'content_html': {
-            'type': 'string',
-        },
+        '_content_html': {'type': 'string'},
         'status': {
             'type': 'string',
             'allowed': [

pillar/api/node_types/post.py

@@ -4,13 +4,14 @@ node_type_post = {
     'name': 'post',
     'description': 'A blog post, for any project',
     'dyn_schema': {
-        # The blogpost content (Markdown format)
         'content': {
             'type': 'string',
             'minlength': 5,
             'maxlength': 90000,
-            'required': True
+            'required': True,
+            'coerce': 'markdown',
         },
+        '_content_html': {'type': 'string'},
         'status': {
             'type': 'string',
             'allowed': [

pillar/api/nodes/__init__.py

@@ -378,30 +378,6 @@ def after_deleting_node(item):
     index.node_delete.delay(str(item['_id']))
 
 
-only_for_comments = only_for_node_type_decorator('comment')
-
-
-@only_for_comments
-def convert_markdown(node, original=None):
-    """Converts comments from Markdown to HTML.
-
-    Always does this on save, even when the original Markdown hasn't changed,
-    because our Markdown -> HTML conversion rules might have.
-    """
-
-    try:
-        content = node['properties']['content']
-    except KeyError:
-        node['properties']['content_html'] = ''
-    else:
-        node['properties']['content_html'] = pillar.markdown.markdown(content)
-
-
-def nodes_convert_markdown(nodes):
-    for node in nodes:
-        convert_markdown(node)
-
-
 only_for_textures = only_for_node_type_decorator('texture')
 
 
@@ -433,7 +409,6 @@ def setup_app(app, url_prefix):
     app.on_fetched_resource_nodes += before_returning_nodes
 
     app.on_replace_nodes += before_replacing_node
-    app.on_replace_nodes += convert_markdown
     app.on_replace_nodes += texture_sort_files
     app.on_replace_nodes += deduct_content_type
     app.on_replace_nodes += node_set_default_picture
@@ -442,11 +417,9 @@ def setup_app(app, url_prefix):
     app.on_insert_nodes += before_inserting_nodes
     app.on_insert_nodes += nodes_deduct_content_type
     app.on_insert_nodes += nodes_set_default_picture
-    app.on_insert_nodes += nodes_convert_markdown
     app.on_insert_nodes += textures_sort_files
     app.on_inserted_nodes += after_inserting_nodes
 
-    app.on_update_nodes += convert_markdown
     app.on_update_nodes += texture_sort_files
 
     app.on_delete_item_nodes += before_deleting_node

pillar/api/nodes/custom/comment.py

@@ -162,7 +162,7 @@ def edit_comment(user_id, node_id, patch):
         log.info('User %s edited comment %s', user_id, node_id)
 
     # Fetch the new content, so the client can show these without querying again.
-    node = nodes_coll.find_one(node_id, projection={'properties.content_html': 1})
+    node = nodes_coll.find_one(node_id, projection={'properties._content_html': 1})
     return status, node