Cache Markdown'ed HTML in database

This is done via coercion rules. To cache the field 'content' in the
database, include this in your Eve schema:

    {'content': {'type': 'string', 'coerce': 'markdown'},
     '_content_html': {'type': 'string'}}

The `_content_html` field will be filled automatically when saving the
document via Eve.

To display the cached HTML, and fall back to display-time rendering if it
is not there, use `{{ document | markdowned('content') }}` in your template.

Still needs unit testing, a CLI command for regenerating the caches, and
a CLI command for migrating the node type definitions in existing projects.

pillar/api/custom_field_validation.py

@@ -1,10 +1,13 @@
 import logging
 
 from bson import ObjectId, tz_util
-from datetime import datetime, tzinfo
+from datetime import datetime
+import cerberus.errors
 from eve.io.mongo import Validator
 from flask import current_app
 
+import pillar.markdown
+
 log = logging.getLogger(__name__)
 
 
@@ -152,3 +155,52 @@ class ValidateCustomFields(Validator):
 
         if not isinstance(value, (bytes, bytearray)):
             self._error(field_name, f'wrong value type {type(value)}, expected bytes or bytearray')
+
+    def _validate_coerce(self, coerce, field: str, value):
+        """Override Cerberus' _validate_coerce method for richer features.
+
+        This now supports named coercion functions (available in Cerberus 1.0+)
+        and passes the field name to coercion functions as well.
+        """
+        if isinstance(coerce, str):
+            coerce = getattr(self, f'_normalize_coerce_{coerce}')
+
+        try:
+            return coerce(field, value)
+        except (TypeError, ValueError):
+            self._error(field, cerberus.errors.ERROR_COERCION_FAILED.format(field))
+
+    def _normalize_coerce_markdown(self, field: str, value):
+        """Render Markdown from this field into {field}_html.
+
+        The field name MUST NOT end in `_html`. The Markdown is read from this
+        field and the rendered HTML is written to the field `{field}_html`.
+        """
+        html = pillar.markdown.markdown(value)
+        field_name = pillar.markdown.cache_field_name(field)
+        self.current[field_name] = html
+        return value
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+
+    v = ValidateCustomFields()
+    v.schema = {
+        'foo': {'type': 'string', 'coerce': 'markdown'},
+        'foo_html': {'type': 'string'},
+        'nested': {
+            'type': 'dict',
+            'schema': {
+                'bar': {'type': 'string', 'coerce': 'markdown'},
+                'bar_html': {'type': 'string'},
+            }
+        }
+    }
+    print('Valid   :', v.validate({
+        'foo': '# Title\n\nHeyyyy',
+        'nested': {'bar': 'bhahaha'},
+    }))
+    print('Document:')
+    pprint(v.document)
+    print('Errors  :', v.errors)

pillar/api/eve_settings.py

@@ -155,7 +155,9 @@ organizations_schema = {
     'description': {
         'type': 'string',
         'maxlength': 256,
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     'website': {
         'type': 'string',
         'maxlength': 256,
@@ -290,7 +292,9 @@ nodes_schema = {
     },
     'description': {
         'type': 'string',
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     'picture': _file_embedded_schema,
     'order': {
         'type': 'integer',
@@ -535,7 +539,9 @@ projects_schema = {
     },
     'description': {
         'type': 'string',
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     # Short summary for the project
     'summary': {
         'type': 'string',

pillar/api/node_types/comment.py

@@ -2,16 +2,14 @@ node_type_comment = {
     'name': 'comment',
     'description': 'Comments for asset nodes, pages, etc.',
     'dyn_schema': {
-        # The actual comment content (initially Markdown format)
+        # The actual comment content
         'content': {
             'type': 'string',
             'minlength': 5,
             'required': True,
+            'coerce': 'markdown',
         },
-        # The converted-to-HTML content.
-        'content_html': {
-            'type': 'string',
-        },
+        '_content_html': {'type': 'string'},
         'status': {
             'type': 'string',
             'allowed': [

pillar/api/node_types/post.py

@@ -4,13 +4,14 @@ node_type_post = {
     'name': 'post',
     'description': 'A blog post, for any project',
     'dyn_schema': {
-        # The blogpost content (Markdown format)
         'content': {
             'type': 'string',
             'minlength': 5,
             'maxlength': 90000,
-            'required': True
+            'required': True,
+            'coerce': 'markdown',
         },
+        '_content_html': {'type': 'string'},
         'status': {
             'type': 'string',
             'allowed': [

pillar/api/nodes/__init__.py

@@ -378,30 +378,6 @@ def after_deleting_node(item):
     index.node_delete.delay(str(item['_id']))
 
 
-only_for_comments = only_for_node_type_decorator('comment')
-
-
-@only_for_comments
-def convert_markdown(node, original=None):
-    """Converts comments from Markdown to HTML.
-
-    Always does this on save, even when the original Markdown hasn't changed,
-    because our Markdown -> HTML conversion rules might have.
-    """
-
-    try:
-        content = node['properties']['content']
-    except KeyError:
-        node['properties']['content_html'] = ''
-    else:
-        node['properties']['content_html'] = pillar.markdown.markdown(content)
-
-
-def nodes_convert_markdown(nodes):
-    for node in nodes:
-        convert_markdown(node)
-
-
 only_for_textures = only_for_node_type_decorator('texture')
 
 
@@ -433,7 +409,6 @@ def setup_app(app, url_prefix):
     app.on_fetched_resource_nodes += before_returning_nodes
 
     app.on_replace_nodes += before_replacing_node
-    app.on_replace_nodes += convert_markdown
     app.on_replace_nodes += texture_sort_files
     app.on_replace_nodes += deduct_content_type
     app.on_replace_nodes += node_set_default_picture
@@ -442,11 +417,9 @@ def setup_app(app, url_prefix):
     app.on_insert_nodes += before_inserting_nodes
     app.on_insert_nodes += nodes_deduct_content_type
     app.on_insert_nodes += nodes_set_default_picture
-    app.on_insert_nodes += nodes_convert_markdown
     app.on_insert_nodes += textures_sort_files
     app.on_inserted_nodes += after_inserting_nodes
 
-    app.on_update_nodes += convert_markdown
     app.on_update_nodes += texture_sort_files
 
     app.on_delete_item_nodes += before_deleting_node

pillar/api/nodes/custom/comment.py

@@ -162,7 +162,7 @@ def edit_comment(user_id, node_id, patch):
         log.info('User %s edited comment %s', user_id, node_id)
 
     # Fetch the new content, so the client can show these without querying again.
-    node = nodes_coll.find_one(node_id, projection={'properties.content_html': 1})
+    node = nodes_coll.find_one(node_id, projection={'properties._content_html': 1})
     return status, node
 
 

pillar/markdown.py

@@ -47,3 +47,11 @@ def markdown(s):
                              attributes=ALLOWED_ATTRIBUTES,
                              styles=ALLOWED_STYLES)
     return safe_html
+
+
+def cache_field_name(field_name: str) -> str:
+    """Return the field name containing the cached HTML.
+
+    See ValidateCustomFields._normalize_coerce_markdown().
+    """
+    return f'_{field_name}_html'

pillar/web/jinja.py

@@ -10,6 +10,7 @@ import flask_login
 import jinja2.filters
 import jinja2.utils
 import werkzeug.exceptions as wz_exceptions
+import pillarsdk
 
 import pillar.api.utils
 from pillar.web.utils import pretty_date
@@ -95,6 +96,12 @@ def do_pluralize(value, arg='s'):
 
 
 def do_markdown(s: typing.Optional[str]):
+    """Convert Markdown.
+
+    This filter is not preferred. Use {'coerce': 'markdown'} in the Eve schema
+    instead, to cache the HTML in the database, and use do_markdowned() to
+    fetch it.
+    """
     if s is None:
         return None
 
@@ -106,6 +113,35 @@ def do_markdown(s: typing.Optional[str]):
     return jinja2.utils.Markup(safe_html)
 
 
+def do_markdowned(document: typing.Union[dict, pillarsdk.Resource], field_name: str) -> str:
+    """Fetch pre-converted Markdown or render on the fly.
+
+    Use {'coerce': 'markdown'} in the Eve schema to cache the HTML in the
+    database and use do_markdowned() to fetch it in a safe way.
+
+    Jinja example: {{ node.properties | markdowned:'content' }}
+    """
+    if isinstance(document, pillarsdk.Resource):
+        document = document.to_dict()
+
+    if not document:
+        return ''
+
+    my_log = log.getChild('do_markdowned')
+
+    cache_field_name = pillar.markdown.cache_field_name(field_name)
+    my_log.debug('Getting %r', cache_field_name)
+
+    cached_html = document.get(cache_field_name)
+    if cached_html is not None:
+        my_log.debug('Cached HTML is %r', cached_html[:40])
+        return jinja2.utils.Markup(cached_html)
+
+    markdown_src = document.get(field_name)
+    my_log.debug('No cached HTML, rendering doc[%r]', field_name)
+    return do_markdown(markdown_src)
+
+
 def do_url_for_node(node_id=None, node=None):
     try:
         return url_for_node(node_id=node_id, node=node)
@@ -156,6 +192,7 @@ def setup_jinja_env(jinja_env, app_config: dict):
     jinja_env.filters['pluralize'] = do_pluralize
     jinja_env.filters['gravatar'] = pillar.api.utils.gravatar
     jinja_env.filters['markdown'] = do_markdown
+    jinja_env.filters['markdowned'] = do_markdowned
     jinja_env.filters['yesno'] = do_yesno
     jinja_env.filters['repr'] = repr
     jinja_env.filters['urljoin'] = functools.partial(urllib.parse.urljoin, allow_fragments=True)

pillar/web/nodes/custom/comments.py

@@ -80,7 +80,7 @@ def comment_edit(comment_id):
     return jsonify({
         'status': 'success',
         'data': {
-            'content_html': result.properties.content_html,
+            'content_html': result.properties['_content_html'],
         }})