Shortcodes for YouTube and iframes

Added shortcodes 2.5.0 as dependency; Earlier versions corrupted
non-ASCII characters, see
https://github.com/dmulholland/shortcodes/issues/6

The rendered elements have a `shortcode` CSS class.

The YouTube shortcode supports various ways to refer to a video:

    - `{youtube VideoID}`
    - `{youtube youtube.com or youtu.be URL}`

URLs containing an '=' should be quoted, or otherwise the shortcodes
library will parse it as "key=value" pair.

The IFrame shortcode supports the `cap` and `nocap` attributes. `cap`
indicates the required capability the user should have in order to
render the tag. If `nocap` is given, its contents are shown as a message
to users who do not have this tag; without it, the iframe is silently
hidden.

`{iframe src='https://source' cap='subscriber' nocap='Subscribe to view'}`

Merged test code + added HTML class for shortcode iframes
This commit is contained in:
Sybren A. Stüvel 2018-03-26 11:58:09 +02:00
parent 0841d52dd1
commit f4e0b9185b
9 changed files with 443 additions and 23 deletions

View File

@ -6,6 +6,8 @@ This is for user-generated stuff, like comments.
import bleach
import CommonMark
from . import shortcodes
ALLOWED_TAGS = [
'a',
'abbr',
@ -40,12 +42,14 @@ ALLOWED_STYLES = [
]
def markdown(s):
tainted_html = CommonMark.commonmark(s)
def markdown(s: str) -> str:
commented_shortcodes = shortcodes.comment_shortcodes(s)
tainted_html = CommonMark.commonmark(commented_shortcodes)
safe_html = bleach.clean(tainted_html,
tags=ALLOWED_TAGS,
attributes=ALLOWED_ATTRIBUTES,
styles=ALLOWED_STYLES)
styles=ALLOWED_STYLES,
strip_comments=False)
return safe_html

204
pillar/shortcodes.py Normal file
View File

@ -0,0 +1,204 @@
"""Shortcode rendering.
Shortcodes are little snippets between square brackets, which can be rendered
into HTML. Markdown passes such snippets unchanged to its HTML output, so this
module assumes its input is HTML-with-shortcodes.
See mulholland.xyz/docs/shortcodes/.
{iframe src='http://hey' has-cap='subscriber'}
NOTE: nested braces fail, so something like {shortcode abc='{}'} is not
supported.
NOTE: only single-line shortcodes are supported for now, due to the need to
pass them though Markdown unscathed.
"""
import html as html_module # I want to be able to use the name 'html' in local scope.
import logging
import re
import typing
import urllib.parse
import shortcodes
_parser: shortcodes.Parser = None
_commented_parser: shortcodes.Parser = None
log = logging.getLogger(__name__)
def shortcode(name: str):
"""Class decorator for shortcodes."""
def decorator(cls):
assert hasattr(cls, '__call__'), '@shortcode should be used on callables.'
if isinstance(cls, type):
instance = cls()
else:
instance = cls
shortcodes.register(name)(instance)
return cls
return decorator
@shortcode('test')
class Test:
def __call__(self,
context: typing.Any,
content: str,
pargs: typing.List[str],
kwargs: typing.Dict[str, str]) -> str:
"""Just for testing.
"{test abc='def'}" "<dl><dt>test</dt><dt>abc</dt><dd>def</dd></dl>"
"""
parts = ['<dl><dt>test</dt>']
e = html_module.escape
parts.extend([
f'<dt>{e(key)}</dt><dd>{e(value)}</dd>' for key, value in kwargs.items()
])
parts.append('</dl>')
return ''.join(parts)
@shortcode('youtube')
class YouTube:
log = log.getChild('YouTube')
def video_id(self, url: str) -> str:
"""Find the video ID from a YouTube URL.
:raise ValueError: when the ID cannot be determined.
"""
if re.fullmatch(r'[a-zA-Z0-9_\-]+', url):
return url
try:
parts = urllib.parse.urlparse(url)
if parts.netloc == 'youtu.be':
return parts.path.split('/')[1]
if parts.netloc in {'www.youtube.com', 'youtube.com'}:
if parts.path.startswith('/embed/'):
return parts.path.split('/')[2]
if parts.path.startswith('/watch'):
qs = urllib.parse.parse_qs(parts.query)
return qs['v'][0]
except (ValueError, IndexError, KeyError) as ex:
pass
raise ValueError(f'Unable to parse YouTube URL {url!r}')
def __call__(self,
context: typing.Any,
content: str,
pargs: typing.List[str],
kwargs: typing.Dict[str, str]) -> str:
"""Embed a YouTube video.
The first parameter must be the YouTube video ID or URL. The width and
height can be passed in the equally named keyword arguments.
"""
width = kwargs.get('width', '560')
height = kwargs.get('height', '315')
# Figure out the embed URL for the video.
try:
youtube_src = pargs[0]
except IndexError:
return html_module.escape('{youtube missing YouTube ID/URL}')
try:
youtube_id = self.video_id(youtube_src)
except ValueError as ex:
return html_module.escape('{youtube %s}' % "; ".join(ex.args))
if not youtube_id:
return html_module.escape('{youtube invalid YouTube ID/URL}')
src = f'https://www.youtube.com/embed/{youtube_id}?rel=0'
html = f'<iframe class="shortcode youtube" width="{width}" height="{height}" src="{src}"' \
f' frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>'
return html
@shortcode('iframe')
def iframe(context: typing.Any,
content: str,
pargs: typing.List[str],
kwargs: typing.Dict[str, str]) -> str:
"""Show an iframe to users with the required capability.
kwargs:
- 'cap': Capability required for viewing.
- others: Turned into attributes for the iframe element.
"""
import xml.etree.ElementTree as ET
from pillar.auth import current_user
cap = kwargs.pop('cap', None)
if not cap:
return html_module.escape('{iframe missing cap="somecap"}')
nocap = kwargs.pop('nocap', '')
if not current_user.has_cap(cap):
if not nocap:
return ''
html = html_module.escape(nocap)
return f'<p class="shortcode nocap">{html}</p>'
kwargs['class'] = f'shortcode {kwargs.get("class", "")}'.strip()
element = ET.Element('iframe', kwargs)
html = ET.tostring(element, encoding='unicode', method='html', short_empty_elements=True)
return html
def _get_parser() -> typing.Tuple[shortcodes.Parser, shortcodes.Parser]:
"""Return the shortcodes parser, create it if necessary."""
global _parser, _commented_parser
if _parser is None:
start, end = '{}'
_parser = shortcodes.Parser(start, end)
_commented_parser = shortcodes.Parser(f'<!-- {start}', f'{end} -->')
return _parser, _commented_parser
def render_commented(text: str, context: typing.Any = None) -> str:
"""Parse and render HTML-commented shortcodes.
Expects shortcodes like "<!-- {shortcode abc='def'} -->", as output by
escape_html().
"""
_, parser = _get_parser()
# TODO(Sybren): catch exceptions and handle those gracefully in the response.
try:
return parser.parse(text, context)
except shortcodes.InvalidTagError as ex:
return html_module.escape('{%s}' % ex)
except shortcodes.RenderingError as ex:
return html_module.escape('{unable to render tag: %s}' % str(ex.__cause__ or ex))
def render(text: str, context: typing.Any = None) -> str:
"""Parse and render shortcodes."""
parser, _ = _get_parser()
# TODO(Sybren): catch exceptions and handle those gracefully in the response.
return parser.parse(text, context)
def comment_shortcodes(html: str) -> str:
"""Escape shortcodes in HTML comments.
This is required to pass the shortcodes as-is through Markdown. Render the
shortcodes afterwards with render_commented().
>>> comment_shortcodes("text\\n{shortcode abc='def'}\\n")
"text\\n<!-- {shortcode abc='def'} -->\\n"
"""
parser, _ = _get_parser()
return parser.regex.sub(r'<!-- \g<0> -->', html)

View File

@ -101,6 +101,8 @@ def do_markdown(s: typing.Optional[str]):
This filter is not preferred. Use {'coerce': 'markdown'} in the Eve schema
instead, to cache the HTML in the database, and use do_markdowned() to
fetch it.
Jinja example: {{ node.properties.content | markdown }}
"""
if s is None:
return None
@ -113,33 +115,47 @@ def do_markdown(s: typing.Optional[str]):
return jinja2.utils.Markup(safe_html)
def do_markdowned(document: typing.Union[dict, pillarsdk.Resource], field_name: str) -> str:
def _get_markdowned_html(document: typing.Union[dict, pillarsdk.Resource], field_name: str) -> str:
"""Fetch pre-converted Markdown or render on the fly.
Use {'coerce': 'markdown'} in the Eve schema to cache the HTML in the
database and use do_markdowned() to fetch it in a safe way.
Jinja example: {{ node.properties | markdowned:'content' }}
Places shortcode tags `{...}` between HTML comments, so that they pass
through the Markdown parser as-is.
"""
if isinstance(document, pillarsdk.Resource):
document = document.to_dict()
if not document:
# If it's empty, we don't care what it is.
return ''
my_log = log.getChild('do_markdowned')
assert isinstance(document, dict), \
f'document should be dict or pillarsdk.Resource, not {document!r}'
cache_field_name = pillar.markdown.cache_field_name(field_name)
my_log.debug('Getting %r', cache_field_name)
html = document.get(cache_field_name)
if html is None:
markdown_src = document.get(field_name) or ''
html = pillar.markdown.markdown(markdown_src)
return html
cached_html = document.get(cache_field_name)
if cached_html is not None:
my_log.debug('Cached HTML is %r', cached_html[:40])
return jinja2.utils.Markup(cached_html)
markdown_src = document.get(field_name)
my_log.debug('No cached HTML, rendering doc[%r]', field_name)
return do_markdown(markdown_src)
def do_markdowned(document: typing.Union[dict, pillarsdk.Resource], field_name: str) \
-> jinja2.utils.Markup:
"""Render Markdown and shortcodes.
Use {'coerce': 'markdown'} in the Eve schema to cache the HTML in the
database and use do_markdowned() to fetch it in a safe way.
Jinja example: {{ node.properties | markdowned('content') }}
The value of `document` is sent as context to the shortcodes render
function.
"""
from pillar import shortcodes
html = _get_markdowned_html(document, field_name)
html = shortcodes.render_commented(html, context=document)
return jinja2.utils.Markup(html)
def do_url_for_node(node_id=None, node=None):

View File

@ -28,6 +28,7 @@ python-dateutil==2.5.3
rauth==0.7.3
raven[flask]==6.3.0
redis==2.10.5
shortcodes==2.5.0
WebOb==1.5.0
wheel==0.29.0
zencoder==0.6.5

View File

@ -54,6 +54,7 @@ setuptools.setup(
'Pillow>=2.8.1',
'requests>=2.9.1',
'rsa>=3.3',
'shortcodes>=2.5', # 2.4.0 and earlier corrupted unicode
'zencoder>=0.6.5',
'bcrypt>=2.0.0',
'blinker>=1.4',

View File

@ -1217,3 +1217,10 @@ button, .btn
&.active
opacity: 1
p.shortcode.nocap
padding: 0.6em 3em
font-size: .8em
color: $color-text-dark-primary
background-color: $color-background-dark
border-radius: 3px

View File

@ -1,12 +1,9 @@
import copy
from pillar.tests import AbstractPillarTest
from pillar.tests import common_test_data as ctd
class CoerceMarkdownTest(AbstractPillarTest):
def test_node_description(self):
from pillar.markdown import markdown
pid, uid = self.create_project_with_admin(24 * 'a')
self.create_valid_auth_token(uid, 'token-a')
node = {
@ -23,10 +20,10 @@ class CoerceMarkdownTest(AbstractPillarTest):
node_id = created_data['_id']
json_node = self.get(f'/api/nodes/{node_id}', auth_token='token-a').json()
self.assertEqual(markdown(node['description']), json_node['_description_html'])
self.assertEqual('<h1>Title</h1>\n<p>This is content.</p>\n',
json_node['_description_html'])
def test_project_description(self):
from pillar.markdown import markdown
from pillar.api.utils import remove_private_keys
uid = self.create_user(24 * 'a', token='token-a')
@ -50,4 +47,25 @@ class CoerceMarkdownTest(AbstractPillarTest):
json_proj.pop('node_types', None) # just to make it easier to print
import pprint
pprint.pprint(json_proj)
self.assertEqual(markdown(proj['description']), json_proj['_description_html'])
self.assertEqual('<h1>Title</h1>\n<p>This is content.</p>\n',
json_proj['_description_html'])
def test_comment_shortcodes(self):
pid, uid = self.create_project_with_admin(24 * 'a')
self.create_valid_auth_token(uid, 'token-a')
node = {
'node_type': 'group',
'name': 'Test group',
'description': '# Title\n\n{test a="b"}',
'properties': {},
'project': pid,
'user': uid,
}
created_data = self.post('/api/nodes', json=node, expected_status=201,
auth_token='token-a').json()
node_id = created_data['_id']
json_node = self.get(f'/api/nodes/{node_id}', auth_token='token-a').json()
expect = '<h1>Title</h1>\n<!-- {test a="b"} -->\n'
self.assertEqual(expect, json_node['_description_html'])

157
tests/test_shortcodes.py Normal file
View File

@ -0,0 +1,157 @@
import unittest
from pillar.tests import AbstractPillarTest
class EscapeHTMLTest(unittest.TestCase):
def test_simple(self):
from pillar.shortcodes import comment_shortcodes
self.assertEqual(
"text\\n<!-- {shortcode abc='def'} -->\\n",
comment_shortcodes("text\\n{shortcode abc='def'}\\n")
)
def test_double_tags(self):
from pillar.shortcodes import comment_shortcodes
self.assertEqual(
"text\\n<!-- {shortcode abc='def'} -->hey<!-- {othercode} -->\\n",
comment_shortcodes("text\\n{shortcode abc='def'}hey{othercode}\\n")
)
class DegenerateTest(unittest.TestCase):
def test_degenerate_cases(self):
from pillar.shortcodes import render
self.assertEqual('', render(''))
with self.assertRaises(TypeError):
render(None)
class DemoTest(unittest.TestCase):
def test_demo(self):
from pillar.shortcodes import render
self.assertEqual('<dl><dt>test</dt></dl>', render('{test}'))
self.assertEqual('<dl><dt>test</dt><dt>a</dt><dd>b</dd></dl>', render('{test a="b"}'))
def test_unicode(self):
from pillar.shortcodes import render
self.assertEqual('<dl><dt>test</dt><dt>ü</dt><dd>é</dd></dl>', render('{test ü="é"}'))
class YouTubeTest(unittest.TestCase):
def test_missing(self):
from pillar.shortcodes import render
self.assertEqual('{youtube missing YouTube ID/URL}', render('{youtube}'))
def test_invalid(self):
from pillar.shortcodes import render
self.assertEqual(
'{youtube Unable to parse YouTube URL &#x27;https://attacker.com/&#x27;}',
render('{youtube https://attacker.com/}')
)
def test_id(self):
from pillar.shortcodes import render
self.assertEqual(
'<iframe class="shortcode youtube" width="560" height="315" '
'src="https://www.youtube.com/embed/ABCDEF?rel=0" frameborder="0" '
'allow="autoplay; encrypted-media" allowfullscreen></iframe>',
render('{youtube ABCDEF}')
)
def test_embed_url(self):
from pillar.shortcodes import render
self.assertEqual(
'<iframe class="shortcode youtube" width="560" height="315" '
'src="https://www.youtube.com/embed/ABCDEF?rel=0" frameborder="0" '
'allow="autoplay; encrypted-media" allowfullscreen></iframe>',
render('{youtube http://youtube.com/embed/ABCDEF}')
)
def test_youtu_be(self):
from pillar.shortcodes import render
self.assertEqual(
'<iframe class="shortcode youtube" width="560" height="315" '
'src="https://www.youtube.com/embed/NwVGvcIrNWA?rel=0" frameborder="0" '
'allow="autoplay; encrypted-media" allowfullscreen></iframe>',
render('{youtube https://youtu.be/NwVGvcIrNWA}')
)
def test_watch(self):
from pillar.shortcodes import render
self.assertEqual(
'<iframe class="shortcode youtube" width="560" height="315" '
'src="https://www.youtube.com/embed/NwVGvcIrNWA?rel=0" frameborder="0" '
'allow="autoplay; encrypted-media" allowfullscreen></iframe>',
render('{youtube "https://www.youtube.com/watch?v=NwVGvcIrNWA"}')
)
def test_width_height(self):
from pillar.shortcodes import render
self.assertEqual(
'<iframe class="shortcode youtube" width="5" height="3" '
'src="https://www.youtube.com/embed/NwVGvcIrNWA?rel=0" frameborder="0" '
'allow="autoplay; encrypted-media" allowfullscreen></iframe>',
render('{youtube "https://www.youtube.com/watch?v=NwVGvcIrNWA" width=5 height="3"}')
)
class IFrameTest(AbstractPillarTest):
def test_missing_cap(self):
from pillar.shortcodes import render
self.assertEqual('{iframe missing cap=&quot;somecap&quot;}', render('{iframe}'))
def test_user_no_cap(self):
from pillar.shortcodes import render
with self.app.app_context():
# Anonymous user, so no subscriber capability.
self.assertEqual('', render('{iframe cap=subscriber}'))
self.assertEqual('', render('{iframe cap="subscriber"}'))
self.assertEqual(
'<p class="shortcode nocap">Aðeins áskrifendur hafa aðgang að þessu efni.</p>',
render('{iframe'
' cap="subscriber"'
' nocap="Aðeins áskrifendur hafa aðgang að þessu efni."}'))
def test_user_has_cap(self):
from pillar.shortcodes import render
roles = {'demo'}
uid = self.create_user(roles=roles)
with self.app.app_context():
self.login_api_as(uid, roles=roles)
self.assertEqual('<iframe class="shortcode"></iframe>',
render('{iframe cap=subscriber}'))
self.assertEqual('<iframe class="shortcode"></iframe>',
render('{iframe cap="subscriber"}'))
self.assertEqual('<iframe class="shortcode"></iframe>',
render('{iframe cap="subscriber" nocap="x"}'))
def test_attributes(self):
from pillar.shortcodes import render
roles = {'demo'}
uid = self.create_user(roles=roles)
md = '{iframe cap=subscriber zzz=xxx class="bigger" ' \
'src="https://docs.python.org/3/library/xml.etree.elementtree.html#functions"}'
expect = '<iframe class="shortcode bigger"' \
' src="https://docs.python.org/3/library/xml.etree.elementtree.html#functions"' \
' zzz="xxx">' \
'</iframe>'
with self.app.app_context():
self.login_api_as(uid, roles=roles)
self.assertEqual(expect, render(md))

View File

@ -23,9 +23,21 @@ class MarkdownTest(unittest.TestCase):
def test_markdowned(self):
from pillar.web import jinja
self.assertEqual(None, jinja.do_markdowned({'eek': None}, 'eek'))
self.assertEqual('', jinja.do_markdowned({'eek': None}, 'eek'))
self.assertEqual('<p>ook</p>\n', jinja.do_markdowned({'eek': 'ook'}, 'eek'))
self.assertEqual('<p>ook</p>\n', jinja.do_markdowned(
{'eek': 'ook', '_eek_html': None}, 'eek'))
self.assertEqual('prerendered', jinja.do_markdowned(
{'eek': 'ook', '_eek_html': 'prerendered'}, 'eek'))
def test_markdowned_with_shortcodes(self):
from pillar.web import jinja
self.assertEqual(
'<dl><dt>test</dt><dt>a</dt><dd>b</dd><dt>c</dt><dd>d</dd></dl>\n',
jinja.do_markdowned({'eek': '{test a="b" c="d"}'}, 'eek'))
self.assertEqual(
'<h1>Title</h1>\n<p>Before</p>\n'
'<dl><dt>test</dt><dt>a</dt><dd>b</dd><dt>c</dt><dd>d</dd></dl>\n',
jinja.do_markdowned({'eek': '# Title\n\nBefore\n{test a="b" c="d"}'}, 'eek'))