Cache Markdown'ed HTML in database

This is done via coercion rules. To cache the field 'content' in the
database, include this in your Eve schema:

    {'content': {'type': 'string', 'coerce': 'markdown'},
     '_content_html': {'type': 'string'}}

The `_content_html` field will be filled automatically when saving the
document via Eve.

To display the cached HTML, and fall back to display-time rendering if it
is not there, use `{{ document | markdowned('content') }}` in your template.

Still needs unit testing, a CLI command for regenerating the caches, and
a CLI command for migrating the node type definitions in existing projects.

pillar/api/custom_field_validation.py

@@ -1,10 +1,13 @@
 import logging
 
 from bson import ObjectId, tz_util
-from datetime import datetime, tzinfo
+from datetime import datetime
+import cerberus.errors
 from eve.io.mongo import Validator
 from flask import current_app
 
+import pillar.markdown
+
 log = logging.getLogger(__name__)
 
 
@@ -152,3 +155,52 @@ class ValidateCustomFields(Validator):
 
         if not isinstance(value, (bytes, bytearray)):
             self._error(field_name, f'wrong value type {type(value)}, expected bytes or bytearray')
+
+    def _validate_coerce(self, coerce, field: str, value):
+        """Override Cerberus' _validate_coerce method for richer features.
+
+        This now supports named coercion functions (available in Cerberus 1.0+)
+        and passes the field name to coercion functions as well.
+        """
+        if isinstance(coerce, str):
+            coerce = getattr(self, f'_normalize_coerce_{coerce}')
+
+        try:
+            return coerce(field, value)
+        except (TypeError, ValueError):
+            self._error(field, cerberus.errors.ERROR_COERCION_FAILED.format(field))
+
+    def _normalize_coerce_markdown(self, field: str, value):
+        """Render Markdown from this field into {field}_html.
+
+        The field name MUST NOT end in `_html`. The Markdown is read from this
+        field and the rendered HTML is written to the field `{field}_html`.
+        """
+        html = pillar.markdown.markdown(value)
+        field_name = pillar.markdown.cache_field_name(field)
+        self.current[field_name] = html
+        return value
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+
+    v = ValidateCustomFields()
+    v.schema = {
+        'foo': {'type': 'string', 'coerce': 'markdown'},
+        'foo_html': {'type': 'string'},
+        'nested': {
+            'type': 'dict',
+            'schema': {
+                'bar': {'type': 'string', 'coerce': 'markdown'},
+                'bar_html': {'type': 'string'},
+            }
+        }
+    }
+    print('Valid   :', v.validate({
+        'foo': '# Title\n\nHeyyyy',
+        'nested': {'bar': 'bhahaha'},
+    }))
+    print('Document:')
+    pprint(v.document)
+    print('Errors  :', v.errors)

pillar/api/eve_settings.py

@@ -155,7 +155,9 @@ organizations_schema = {
     'description': {
         'type': 'string',
         'maxlength': 256,
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     'website': {
         'type': 'string',
         'maxlength': 256,
@@ -290,7 +292,9 @@ nodes_schema = {
     },
     'description': {
         'type': 'string',
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     'picture': _file_embedded_schema,
     'order': {
         'type': 'integer',
@@ -535,7 +539,9 @@ projects_schema = {
     },
     'description': {
         'type': 'string',
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     # Short summary for the project
     'summary': {
         'type': 'string',

pillar/api/node_types/comment.py

@@ -2,16 +2,14 @@ node_type_comment = {
     'name': 'comment',
     'description': 'Comments for asset nodes, pages, etc.',
     'dyn_schema': {
-        # The actual comment content (initially Markdown format)
+        # The actual comment content
         'content': {
             'type': 'string',
             'minlength': 5,
             'required': True,
+            'coerce': 'markdown',
         },
-        # The converted-to-HTML content.
+        '_content_html': {'type': 'string'},
-        'content_html': {
-            'type': 'string',
-        },
         'status': {
             'type': 'string',
             'allowed': [

pillar/api/node_types/post.py

@@ -4,13 +4,14 @@ node_type_post = {
     'name': 'post',
     'description': 'A blog post, for any project',
     'dyn_schema': {
-        # The blogpost content (Markdown format)
         'content': {
             'type': 'string',
             'minlength': 5,
             'maxlength': 90000,
-            'required': True
+            'required': True,
+            'coerce': 'markdown',
         },
+        '_content_html': {'type': 'string'},
         'status': {
             'type': 'string',
             'allowed': [

pillar/api/nodes/__init__.py

@@ -378,30 +378,6 @@ def after_deleting_node(item):
     index.node_delete.delay(str(item['_id']))
 
 
-only_for_comments = only_for_node_type_decorator('comment')
-
-
-@only_for_comments
-def convert_markdown(node, original=None):
-    """Converts comments from Markdown to HTML.
-
-    Always does this on save, even when the original Markdown hasn't changed,
-    because our Markdown -> HTML conversion rules might have.
-    """
-
-    try:
-        content = node['properties']['content']
-    except KeyError:
-        node['properties']['content_html'] = ''
-    else:
-        node['properties']['content_html'] = pillar.markdown.markdown(content)
-
-
-def nodes_convert_markdown(nodes):
-    for node in nodes:
-        convert_markdown(node)
-
-
 only_for_textures = only_for_node_type_decorator('texture')
 
 
@@ -433,7 +409,6 @@ def setup_app(app, url_prefix):
     app.on_fetched_resource_nodes += before_returning_nodes
 
     app.on_replace_nodes += before_replacing_node
-    app.on_replace_nodes += convert_markdown
     app.on_replace_nodes += texture_sort_files
     app.on_replace_nodes += deduct_content_type
     app.on_replace_nodes += node_set_default_picture
@@ -442,11 +417,9 @@ def setup_app(app, url_prefix):
     app.on_insert_nodes += before_inserting_nodes
     app.on_insert_nodes += nodes_deduct_content_type
     app.on_insert_nodes += nodes_set_default_picture
-    app.on_insert_nodes += nodes_convert_markdown
     app.on_insert_nodes += textures_sort_files
     app.on_inserted_nodes += after_inserting_nodes
 
-    app.on_update_nodes += convert_markdown
     app.on_update_nodes += texture_sort_files
 
     app.on_delete_item_nodes += before_deleting_node

pillar/api/nodes/custom/comment.py

@@ -162,7 +162,7 @@ def edit_comment(user_id, node_id, patch):
         log.info('User %s edited comment %s', user_id, node_id)
 
     # Fetch the new content, so the client can show these without querying again.
-    node = nodes_coll.find_one(node_id, projection={'properties.content_html': 1})
+    node = nodes_coll.find_one(node_id, projection={'properties._content_html': 1})
     return status, node
 
 

pillar/markdown.py

@@ -47,3 +47,11 @@ def markdown(s):
                              attributes=ALLOWED_ATTRIBUTES,
                              styles=ALLOWED_STYLES)
     return safe_html
+
+
+def cache_field_name(field_name: str) -> str:
+    """Return the field name containing the cached HTML.
+
+    See ValidateCustomFields._normalize_coerce_markdown().
+    """
+    return f'_{field_name}_html'

pillar/web/jinja.py

@@ -10,6 +10,7 @@ import flask_login
 import jinja2.filters
 import jinja2.utils
 import werkzeug.exceptions as wz_exceptions
+import pillarsdk
 
 import pillar.api.utils
 from pillar.web.utils import pretty_date
@@ -95,6 +96,12 @@ def do_pluralize(value, arg='s'):
 
 
 def do_markdown(s: typing.Optional[str]):
+    """Convert Markdown.
+
+    This filter is not preferred. Use {'coerce': 'markdown'} in the Eve schema
+    instead, to cache the HTML in the database, and use do_markdowned() to
+    fetch it.
+    """
     if s is None:
         return None
 
@@ -106,6 +113,35 @@ def do_markdown(s: typing.Optional[str]):
     return jinja2.utils.Markup(safe_html)
 
 
+def do_markdowned(document: typing.Union[dict, pillarsdk.Resource], field_name: str) -> str:
+    """Fetch pre-converted Markdown or render on the fly.
+
+    Use {'coerce': 'markdown'} in the Eve schema to cache the HTML in the
+    database and use do_markdowned() to fetch it in a safe way.
+
+    Jinja example: {{ node.properties | markdowned:'content' }}
+    """
+    if isinstance(document, pillarsdk.Resource):
+        document = document.to_dict()
+
+    if not document:
+        return ''
+
+    my_log = log.getChild('do_markdowned')
+
+    cache_field_name = pillar.markdown.cache_field_name(field_name)
+    my_log.debug('Getting %r', cache_field_name)
+
+    cached_html = document.get(cache_field_name)
+    if cached_html is not None:
+        my_log.debug('Cached HTML is %r', cached_html[:40])
+        return jinja2.utils.Markup(cached_html)
+
+    markdown_src = document.get(field_name)
+    my_log.debug('No cached HTML, rendering doc[%r]', field_name)
+    return do_markdown(markdown_src)
+
+
 def do_url_for_node(node_id=None, node=None):
     try:
         return url_for_node(node_id=node_id, node=node)
@@ -156,6 +192,7 @@ def setup_jinja_env(jinja_env, app_config: dict):
     jinja_env.filters['pluralize'] = do_pluralize
     jinja_env.filters['gravatar'] = pillar.api.utils.gravatar
     jinja_env.filters['markdown'] = do_markdown
+    jinja_env.filters['markdowned'] = do_markdowned
     jinja_env.filters['yesno'] = do_yesno
     jinja_env.filters['repr'] = repr
     jinja_env.filters['urljoin'] = functools.partial(urllib.parse.urljoin, allow_fragments=True)

pillar/web/nodes/custom/comments.py

@@ -80,7 +80,7 @@ def comment_edit(comment_id):
     return jsonify({
         'status': 'success',
         'data': {
-            'content_html': result.properties.content_html,
+            'content_html': result.properties['_content_html'],
         }})
 
 

requirements.txt

@@ -36,7 +36,7 @@ zencoder==0.6.5
 amqp==2.1.4
 billiard==3.5.0.2
 Flask-PyMongo==0.4.1
-Cerberus==0.9.2
+-e git+git@github.com:armadillica/cerberus.git@sybren-0.9#egg=Cerberus
 Events==0.2.2
 future==0.15.2
 html5lib==0.9999999

src/templates/nodes/custom/asset/file/view_embed.pug

@@ -18,7 +18,7 @@
 
 		| {% if node.description %}
 		.node-details-description#node-description
-			| {{ node.description | markdown }}
+			| {{ node | markdowned('description') }}
 		| {% endif %}
 
 		include ../../_node_details

src/templates/nodes/custom/asset/image/view_embed.pug

@@ -18,7 +18,7 @@
 
 		| {% if node.description %}
 		.node-details-description#node-description
-			| {{ node.description | markdown }}
+			| {{ node | markdowned('description') }}
 		| {% endif %}
 
 		include ../../_node_details
@@ -46,4 +46,3 @@ script.
 	});
 
 | {% endblock %}
-

src/templates/nodes/custom/asset/video/view_embed.pug

@@ -46,7 +46,7 @@
 
 		| {% if node.description %}
 		.node-details-description#node-description
-			| {{ node.description | markdown }}
+			| {{ node | markdowned('description') }}
 		| {% endif %}
 
 		include ../../_node_details

src/templates/nodes/custom/blog/_macros.pug

@@ -25,7 +25,7 @@ a.blog_index-header(href="{{ node.url }}")
 		| {{ node.name }}
 
 	.item-content
-		| {{ node.properties.content | markdown }}
+		| {{ node.properties | markdowned('content') }}
 
 | {% endmacro %}
 

src/templates/nodes/custom/comment/_macros.pug

@@ -11,7 +11,7 @@
 		.comment-body
 			p.comment-author {{ comment._user.full_name }}
 
-			span {{comment.properties.content_html | safe }}
+			span {{comment.properties._content_html | safe }}
 
 		// TODO: Markdown preview when editing
 

src/templates/nodes/custom/group/view_embed.pug

@@ -15,7 +15,7 @@
 
 	| {% if node.description %}
 	.node-details-description
-		| {{ node.description | markdown }}
+		| {{ node | markdowned('description') }}
 	| {% endif %}
 
 	section.node-children.group

src/templates/nodes/custom/group_hdri/view_embed.pug

@@ -12,7 +12,7 @@
 	| {% if node.description %}
 	section.node-row
 		.node-details-description
-			| {{ node.description | markdown }}
+			| {{ node | markdowned('description') }}
 	| {% endif %}
 
 	| {% if children %}

src/templates/nodes/custom/group_texture/view_embed.pug

@@ -12,7 +12,7 @@
 	| {% if node.description %}
 	section.node-row
 		.node-details-description
-			| {{ node.description | markdown }}
+			| {{ node | markdowned('description') }}
 	| {% endif %}
 
 	| {% if children %}

src/templates/nodes/custom/hdri/view_embed.pug

@@ -93,7 +93,7 @@
 
 		| {% if node.description %}
 		.node-details-description#node-description
-			| {{ node.description | markdown }}
+			| {{ node | markdowned('description') }}
 		| {% endif %}
 
 		| {% if node.properties.license_notes %}

src/templates/nodes/custom/page/view_embed.pug

@@ -17,7 +17,7 @@
 
 		| {% if node.description %}
 		.node-details-description#node-description
-			| {{ node.description | markdown }}
+			| {{ node | markdowned('description') }}
 		| {% endif %}
 
 		.node-details-meta.footer

src/templates/projects/view_embed.pug

@@ -39,7 +39,7 @@
 
 		| {% if project.description %}
 		.node-details-description
-			| {{ project.description | markdown }}
+			| {{ project | markdowned('description') }}
 		| {% endif %}
 
 		| {# Until we implement urls for pages
@@ -84,9 +84,9 @@
 							a.title(href="{{ url_for_node(node=n) }}") {{ n.name }}
 							p.description(href="{{ url_for_node(node=n) }}")
 								| {% if n.node_type == 'post' %}
-								| {{ n.properties.content | markdown | striptags | truncate(140, end="... <small>read more</small>") | safe | hide_none }}
+								| {{ n.properties | markdowned('content') | striptags | truncate(140, end="... <small>read more</small>") | safe | hide_none }}
 								| {% else %}
-								| {{ n.description | markdown | striptags | truncate(140, end="... <small>read more</small>") | safe | hide_none }}
+								| {{ n | markdowned('description') | striptags | truncate(140, end="... <small>read more</small>") | safe | hide_none }}
 								| {% endif %}
 							span.details
 								span.what {% if n.properties.content_type %}{{ n.properties.content_type | undertitle }}{% else %}{{ n.node_type | undertitle }}{% endif %} ·

tests/test_api/test_markdown.py

@@ -0,0 +1,53 @@
+import copy
+
+from pillar.tests import AbstractPillarTest
+from pillar.tests import common_test_data as ctd
+
+
+class CoerceMarkdownTest(AbstractPillarTest):
+    def test_node_description(self):
+        from pillar.markdown import markdown
+        pid, uid = self.create_project_with_admin(24 * 'a')
+        self.create_valid_auth_token(uid, 'token-a')
+        node = {
+            'node_type': 'group',
+            'name': 'Test group',
+            'description': '# Title\n\nThis is content.',
+            'properties': {},
+            'project': pid,
+            'user': uid,
+        }
+
+        created_data = self.post('/api/nodes', json=node, expected_status=201,
+                                 auth_token='token-a').json()
+        node_id = created_data['_id']
+
+        json_node = self.get(f'/api/nodes/{node_id}', auth_token='token-a').json()
+        self.assertEqual(markdown(node['description']), json_node['_description_html'])
+
+    def test_project_description(self):
+        from pillar.markdown import markdown
+        from pillar.api.utils import remove_private_keys
+
+        uid = self.create_user(24 * 'a', token='token-a')
+
+        # Go through Eve to create the project.
+        proj = {
+            **ctd.EXAMPLE_PROJECT,
+            'description': '# Title\n\nThis is content.',
+            'user': uid,
+        }
+        proj.pop('picture_header')
+        proj.pop('picture_square')
+        proj.pop('permissions')
+
+        r, _, _, status = self.app.post_internal('projects', remove_private_keys(proj))
+        self.assertEqual(201, status, f'failed because {r}')
+
+        pid = r['_id']
+
+        json_proj = self.get(f'/api/projects/{pid}', auth_token='token-a').json()
+        json_proj.pop('node_types', None)  # just to make it easier to print
+        import pprint
+        pprint.pprint(json_proj)
+        self.assertEqual(markdown(proj['description']), json_proj['_description_html'])