Embed iframe shortcodes into container

Use the @capcheck decorator on any shortcode that should support
this. Currently used by iframe and youtube.

pillar/__init__.py

@@ -184,7 +184,6 @@ class PillarServer(BlinkerCompatibleEve):
 
         if not self.config.get('STATIC_FILE_HASH'):
             self.log.warning('STATIC_FILE_HASH is empty, generating random one')
-            f = open('/data/git/blender-cloud/config_local.py', 'a')
             h = re.sub(r'[_.~-]', '', secrets.token_urlsafe())[:8]
             self.config['STATIC_FILE_HASH'] = h
 

pillar/api/blender_id.py

@@ -47,13 +47,6 @@ def store_subclient_token():
                     'subclient_user_id': str(db_user['_id'])}), status
 
 
-def blender_id_endpoint():
-    """Gets the endpoint for the authentication API. If the env variable
-    is defined, it's possible to override the (default) production address.
-    """
-    return current_app.config['BLENDER_ID_ENDPOINT'].rstrip('/')
-
-
 def validate_create_user(blender_id_user_id, token, oauth_subclient_id):
     """Validates a user against Blender ID, creating the user in our database.
 
@@ -121,13 +114,13 @@ def validate_token(user_id, token, oauth_subclient_id):
         # We only want to accept Blender Cloud tokens.
         payload['client_id'] = current_app.config['OAUTH_CREDENTIALS']['blender-id']['id']
 
-    url = '{0}/u/validate_token'.format(blender_id_endpoint())
+    url = '{0}/u/validate_token'.format(current_app.config['BLENDER_ID_ENDPOINT'])
     log.debug('POSTing to %r', url)
 
     # Retry a few times when POSTing to BlenderID fails.
     # Source: http://stackoverflow.com/a/15431343/875379
     s = requests.Session()
-    s.mount(blender_id_endpoint(), HTTPAdapter(max_retries=5))
+    s.mount(current_app.config['BLENDER_ID_ENDPOINT'], HTTPAdapter(max_retries=5))
 
     # POST to Blender ID, handling errors as negative verification results.
     try:
@@ -225,7 +218,7 @@ def fetch_blenderid_user() -> dict:
 
     my_log = log.getChild('fetch_blenderid_user')
 
-    bid_url = '%s/api/user' % blender_id_endpoint()
+    bid_url = '%s/api/user' % current_app.config['BLENDER_ID_ENDPOINT']
     my_log.debug('Fetching user info from %s', bid_url)
 
     credentials = current_app.config['OAUTH_CREDENTIALS']['blender-id']
@@ -270,7 +263,7 @@ def setup_app(app, url_prefix):
 def switch_user_url(next_url: str) -> str:
     from urllib.parse import quote
 
-    base_url = '%s/switch' % blender_id_endpoint()
+    base_url = '%s/switch' % current_app.config['BLENDER_ID_ENDPOINT']
     if next_url:
         return '%s?next=%s' % (base_url, quote(next_url))
     return base_url

pillar/api/node_types/__init__.py

@@ -40,6 +40,51 @@ attachments_embedded_schema = {
     },
 }
 
+# TODO (fsiddi) reference this schema in all node_types that allow ratings
+ratings_embedded_schema = {
+    'type': 'dict',
+    # Total count of positive ratings (updated at every rating action)
+    'schema': {
+        'positive': {
+            'type': 'integer',
+        },
+        # Total count of negative ratings (updated at every rating action)
+        'negative': {
+            'type': 'integer',
+        },
+        # Collection of ratings, keyed by user
+        'ratings': {
+            'type': 'list',
+            'schema': {
+                'type': 'dict',
+                'schema': {
+                    'user': {
+                        'type': 'objectid',
+                        'data_relation': {
+                            'resource': 'users',
+                            'field': '_id',
+                            'embeddable': False
+                        }
+                    },
+                    'is_positive': {
+                        'type': 'boolean'
+                    },
+                    # Weight of the rating based on user rep and the context.
+                    # Currently we have the following weights:
+                    # - 1 auto null
+                    # - 2 manual null
+                    # - 3 auto valid
+                    # - 4 manual valid
+                    'weight': {
+                        'type': 'integer'
+                    }
+                }
+            }
+        },
+        'hot': {'type': 'float'},
+    },
+}
+
 # Import after defining the common embedded schemas, to prevent dependency cycles.
 from pillar.api.node_types.asset import node_type_asset
 from pillar.api.node_types.blog import node_type_blog

pillar/api/utils/rating.py

@@ -0,0 +1,87 @@
+# These functions come from Reddit
+# https://github.com/reddit/reddit/blob/master/r2/r2/lib/db/_sorts.pyx
+
+# Additional resources
+# http://www.redditblog.com/2009/10/reddits-new-comment-sorting-system.html
+# http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
+# http://amix.dk/blog/post/19588
+
+from datetime import datetime, timezone
+from math import log
+from math import sqrt
+
+epoch = datetime(1970, 1, 1, 0, 0, 0, 0, timezone.utc)
+
+
+def epoch_seconds(date):
+    """Returns the number of seconds from the epoch to date."""
+    td = date - epoch
+    return td.days * 86400 + td.seconds + (float(td.microseconds) / 1000000)
+
+
+def score(ups, downs):
+    return ups - downs
+
+
+def hot(ups, downs, date):
+    """The hot formula. Reddit's hot ranking uses the logarithm function to
+    weight the first votes higher than the rest.
+    The first 10 upvotes have the same weight as the next 100 upvotes which
+    have the same weight as the next 1000, etc.
+
+    Dillo authors: we modified the formula to give more weight to negative
+    votes when an entry is controversial.
+
+    TODO: make this function more dynamic so that different defaults can be
+    specified depending on the item that is being rated.
+    """
+
+    s = score(ups, downs)
+    order = log(max(abs(s), 1), 10)
+    sign = 1 if s > 0 else -1 if s < 0 else 0
+    seconds = epoch_seconds(date) - 1134028003
+    base_hot = round(sign * order + seconds / 45000, 7)
+
+    if downs > 1:
+        rating_delta = 100 * (downs - ups) / downs
+        if rating_delta < 25:
+            # The post is controversial
+            return base_hot
+        base_hot = base_hot - (downs * 6)
+
+    return base_hot
+
+
+def _confidence(ups, downs):
+    n = ups + downs
+
+    if n == 0:
+        return 0
+
+    z = 1.0 #1.0 = 85%, 1.6 = 95%
+    phat = float(ups) / n
+    return sqrt(phat+z*z/(2*n)-z*((phat*(1-phat)+z*z/(4*n))/n))/(1+z*z/n)
+
+
+def confidence(ups, downs):
+    if ups + downs == 0:
+        return 0
+    else:
+        return _confidence(ups, downs)
+
+
+def update_hot(document):
+    """Update the hotness of a document given its current ratings.
+
+    We expect the document to implement the ratings_embedded_schema in
+    a 'ratings' property.
+    """
+
+    dt = document['_created']
+    dt = dt.replace(tzinfo=timezone.utc)
+
+    document['properties']['ratings']['hot'] = hot(
+        document['properties']['ratings']['positive'],
+        document['properties']['ratings']['negative'],
+        dt,
+    )

pillar/auth/oauth.py

@@ -131,16 +131,15 @@ class BlenderIdSignIn(OAuthSignIn):
     def __init__(self):
         super().__init__()
 
-        base_url = current_app.config['OAUTH_CREDENTIALS']['blender-id'].get(
+        base_url = current_app.config['BLENDER_ID_ENDPOINT']
-            'base_url', 'https://www.blender.org/id/')
 
         self.service = OAuth2Service(
             name='blender-id',
             client_id=self.consumer_id,
             client_secret=self.consumer_secret,
-            authorize_url='%soauth/authorize' % base_url,
+            authorize_url='%s/oauth/authorize' % base_url,
-            access_token_url='%soauth/token' % base_url,
+            access_token_url='%s/oauth/token' % base_url,
-            base_url='%sapi/' % base_url
+            base_url='%s/api/' % base_url
         )
 
     def authorize(self):

pillar/config.py

@@ -32,7 +32,7 @@ SECRET_KEY = ''
 AUTH_TOKEN_HMAC_KEY = b''
 
 # Authentication settings
-BLENDER_ID_ENDPOINT = 'http://blender-id:8000/'
+BLENDER_ID_ENDPOINT = 'http://id.local:8000'
 
 CDN_USE_URL_SIGNING = True
 CDN_SERVICE_DOMAIN_PROTOCOL = 'https'
@@ -124,9 +124,8 @@ BLENDER_ID_USER_INFO_TOKEN = '-set-in-config-local-'
 # Example entry:
 # OAUTH_CREDENTIALS = {
 #    'blender-id': {
-#        'id': 'CLOUD-OF-SNOWFLAKES-43',
+#        'id': 'CLOUD-OF-SNOWFLAKES-42',
 #        'secret': 'thesecret',
-#        'base_url': 'http://blender-id:8000/'
 #     }
 # }
 # OAuth providers are defined in pillar.auth.oauth

pillar/markdown.py

@@ -45,11 +45,15 @@ ALLOWED_STYLES = [
 def markdown(s: str) -> str:
     commented_shortcodes = shortcodes.comment_shortcodes(s)
     tainted_html = CommonMark.commonmark(commented_shortcodes)
-    safe_html = bleach.clean(tainted_html,
+
-                             tags=ALLOWED_TAGS,
+    # Create a Cleaner that supports parsing of bare links (see filters).
+    cleaner = bleach.Cleaner(tags=ALLOWED_TAGS,
                              attributes=ALLOWED_ATTRIBUTES,
                              styles=ALLOWED_STYLES,
-                             strip_comments=False)
+                             strip_comments=False,
+                             filters=[bleach.linkifier.LinkifyFilter])
+
+    safe_html = cleaner.clean(tainted_html)
     return safe_html
 
 

pillar/shortcodes.py

@@ -33,18 +33,57 @@ log = logging.getLogger(__name__)
 def shortcode(name: str):
     """Class decorator for shortcodes."""
 
-    def decorator(cls):
+    def decorator(decorated):
-        assert hasattr(cls, '__call__'), '@shortcode should be used on callables.'
+        assert hasattr(decorated, '__call__'), '@shortcode should be used on callables.'
-        if isinstance(cls, type):
+        if isinstance(decorated, type):
-            instance = cls()
+            as_callable = decorated()
         else:
-            instance = cls
+            as_callable = decorated
-        shortcodes.register(name)(instance)
+        shortcodes.register(name)(as_callable)
-        return cls
+        return decorated
 
     return decorator
 
 
+class capcheck:
+    """Decorator for shortcodes.
+
+    On call, check for capabilities before calling the function. If the user does not
+    have a capability, display a message insdead of the content.
+
+    kwargs:
+        - 'cap': Capability required for viewing.
+        - 'nocap': Optional, text shown when the user does not have this capability.
+        - others: Passed to the decorated shortcode.
+    """
+
+    def __init__(self, decorated):
+        assert hasattr(decorated, '__call__'), '@capcheck should be used on callables.'
+        if isinstance(decorated, type):
+            as_callable = decorated()
+        else:
+            as_callable = decorated
+        self.decorated = as_callable
+
+    def __call__(self,
+                 context: typing.Any,
+                 content: str,
+                 pargs: typing.List[str],
+                 kwargs: typing.Dict[str, str]) -> str:
+        from pillar.auth import current_user
+
+        cap = kwargs.pop('cap', '')
+        if cap:
+            nocap = kwargs.pop('nocap', '')
+            if not current_user.has_cap(cap):
+                if not nocap:
+                    return ''
+                html = html_module.escape(nocap)
+                return f'<p class="shortcode nocap">{html}</p>'
+
+        return self.decorated(context, content, pargs, kwargs)
+
+
 @shortcode('test')
 class Test:
     def __call__(self,
@@ -68,6 +107,7 @@ class Test:
 
 
 @shortcode('youtube')
+@capcheck
 class YouTube:
     log = log.getChild('YouTube')
 
@@ -123,12 +163,16 @@ class YouTube:
             return html_module.escape('{youtube invalid YouTube ID/URL}')
 
         src = f'https://www.youtube.com/embed/{youtube_id}?rel=0'
-        html = f'<iframe class="shortcode youtube" width="{width}" height="{height}" src="{src}"' \
+        iframe_tag = f'<iframe class="shortcode youtube embed-responsive-item" width="{width}"' \
-               f' frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>'
+                     f' height="{height}" src="{src}" frameborder="0" allow="autoplay; encrypted-media"' \
+                     f' allowfullscreen></iframe>'
+        # Embed the iframe in a container, to allow easier styling
+        html = f'<div class="embed-responsive embed-responsive-16by9">{iframe_tag}</div>'
         return html
 
 
 @shortcode('iframe')
+@capcheck
 def iframe(context: typing.Any,
            content: str,
            pargs: typing.List[str],
@@ -140,16 +184,6 @@ def iframe(context: typing.Any,
         - others: Turned into attributes for the iframe element.
     """
     import xml.etree.ElementTree as ET
-    from pillar.auth import current_user
-
-    cap = kwargs.pop('cap', '')
-    if cap:
-        nocap = kwargs.pop('nocap', '')
-        if not current_user.has_cap(cap):
-            if not nocap:
-                return ''
-            html = html_module.escape(nocap)
-            return f'<p class="shortcode nocap">{html}</p>'
 
     kwargs['class'] = f'shortcode {kwargs.get("class", "")}'.strip()
     element = ET.Element('iframe', kwargs)

pillar/tests/config_testing.py

@@ -1,6 +1,6 @@
 """Flask configuration file for unit testing."""
 
-BLENDER_ID_ENDPOINT = 'http://127.0.0.1:8001'  # nonexistant server, no trailing slash!
+BLENDER_ID_ENDPOINT = 'http://id.local:8001'  # Non existant server
 
 SERVER_NAME = 'localhost'
 PILLAR_SERVER_ENDPOINT = 'http://localhost/api/'
@@ -26,7 +26,6 @@ OAUTH_CREDENTIALS = {
     'blender-id': {
         'id': 'blender-id-app-id',
         'secret': 'blender-id–secret',
-        'base_url': 'http://blender-id:8000/'
     },
     'facebook': {
         'id': 'fb-app-id',

pillar/web/nodes/forms.py

@@ -4,7 +4,7 @@ from datetime import datetime
 from datetime import date
 import pillarsdk
 from flask import current_app
-from flask_wtf import Form
+from flask_wtf import FlaskForm
 from wtforms import StringField
 from wtforms import DateField
 from wtforms import SelectField
@@ -110,7 +110,7 @@ def get_node_form(node_type):
     :param node_type: Describes the node type via dyn_schema, form_schema and
     parent
     """
-    class ProceduralForm(Form):
+    class ProceduralForm(FlaskForm):
         pass
 
     parent_prop = node_type['parent']

pillar/web/projects/forms.py

@@ -1,4 +1,4 @@
-from flask_wtf import Form
+from flask_wtf import FlaskForm
 from wtforms import StringField
 from wtforms import BooleanField
 from wtforms import HiddenField
@@ -12,7 +12,7 @@ from pillar.web import system_util
 from pillar.web.utils.forms import FileSelectField, JSONRequired
 
 
-class ProjectForm(Form):
+class ProjectForm(FlaskForm):
     project_id = HiddenField('project_id', validators=[DataRequired()])
     name = StringField('Name', validators=[DataRequired()])
     url = StringField('Url', validators=[DataRequired()])
@@ -32,7 +32,7 @@ class ProjectForm(Form):
     picture_square = FileSelectField('Picture square', file_format='image')
 
     def validate(self):
-        rv = Form.validate(self)
+        rv = FlaskForm.validate(self)
         if not rv:
             return False
 
@@ -54,7 +54,7 @@ class ProjectForm(Form):
         return True
 
 
-class NodeTypeForm(Form):
+class NodeTypeForm(FlaskForm):
     project_id = HiddenField('project_id', validators=[DataRequired()])
     name = StringField('Name', validators=[DataRequired()])
     parent = StringField('Parent')

pillar/web/system_util.py

@@ -12,14 +12,6 @@ from pillar.sdk import FlaskInternalApi
 log = logging.getLogger(__name__)
 
 
-def blender_id_endpoint():
-    """Gets the endpoint for the authentication API. If the env variable
-    is defined, it's possible to override the (default) production address.
-    """
-    return os.environ.get('BLENDER_ID_ENDPOINT',
-                          "https://www.blender.org/id").rstrip('/')
-
-
 def pillar_server_endpoint():
     """Gets the endpoint for the authentication API. If the env variable
     is defined, we will use the one from the config object.

pillar/web/users/forms.py

@@ -1,5 +1,5 @@
 from flask_login import current_user
-from flask_wtf import Form
+from flask_wtf import FlaskForm
 from pillar.web import system_util
 from pillarsdk.users import User
 
@@ -14,7 +14,7 @@ from wtforms.validators import Regexp
 import wtforms.validators as wtvalid
 
 
-class UserLoginForm(Form):
+class UserLoginForm(FlaskForm):
     username = StringField('Username', validators=[DataRequired()])
     password = PasswordField('Password', validators=[DataRequired()])
     remember_me = BooleanField('Remember Me')
@@ -23,7 +23,7 @@ class UserLoginForm(Form):
         super(UserLoginForm, self).__init__(csrf_enabled=False, *args, **kwargs)
 
 
-class UserProfileForm(Form):
+class UserProfileForm(FlaskForm):
     username = StringField('Username', validators=[DataRequired(), Length(
         min=3, max=128, message="Min. 3, max. 128 chars please"), Regexp(
         r'^[\w.@+-]+$', message="Please do not use spaces")])
@@ -52,7 +52,7 @@ class UserProfileForm(Form):
         return True
 
 
-class UserSettingsEmailsForm(Form):
+class UserSettingsEmailsForm(FlaskForm):
     choices = [
         (1, 'Keep me updated with Blender Cloud news.'),
         (0, 'Do not mail me news update.')]
@@ -74,7 +74,7 @@ class RolesField(SelectMultipleField):
         return current_app.user_roles
 
 
-class UserEditForm(Form):
+class UserEditForm(FlaskForm):
     roles = RolesField('Roles')
     email = StringField(
         validators=[wtvalid.DataRequired(), wtvalid.Email()],

requirements.txt

@@ -5,7 +5,7 @@ attrs==16.2.0
 algoliasearch==1.12.0
 bcrypt==3.1.3
 blinker==1.4
-bleach==1.4.3
+bleach==2.1.3
 celery[redis]==4.0.2
 CommonMark==0.7.2
 elasticsearch==6.1.1
@@ -40,7 +40,7 @@ Flask-PyMongo==0.4.1
 -e git+https://github.com/armadillica/cerberus.git@sybren-0.9#egg=Cerberus
 Events==0.2.2
 future==0.15.2
-html5lib==0.9999999
+html5lib==0.99999999
 googleapis-common-protos==1.1.0
 itsdangerous==0.24
 Jinja2==2.9.6

src/styles/_utils.sass

@@ -405,6 +405,10 @@
 			bottom: 25px
 			top: 25px
 
+		&.emoji
+			display: inline-block
+			padding: initial
+
 	h2
 		margin-bottom: 15px
 
@@ -443,11 +447,12 @@
 
 		li
 			margin-bottom: 7px
+
 			img
 				display: block
 				padding:
-					top: 25px
 					bottom: 10px
+					top: 25px
 
 			ul, ul li ul
 				margin-top: 15px
@@ -455,10 +460,13 @@
 
 	code, kbd, pre, samp
 		background-color: rgba($color-primary, .05)
-		color: $color-primary
+		color: darken($color-primary, 15%)
 		font-size: inherit
 		white-space: pre-line
 
+		code
+			background-color: transparent
+
 	kbd
 		border:
 			color: rgba($color-primary, .33)

src/templates/nodes/custom/blog/_macros.pug

@@ -3,7 +3,7 @@
 | {% macro render_blog_post(node, project=None, pages=None) %}
 | {% if node.picture %}
 a.blog_index-header(href="{{ node.url }}")
-	img(src="{{ node.picture.thumbnail('l', api=api) }}")
+	img(src="{{ node.picture.thumbnail('h', api=api) }}")
 | {% endif %}
 | {% if project and project._id != config.MAIN_PROJECT_ID %}
 | {{ projectmacros.render_secondary_navigation(project, pages=pages) }}

src/templates/nodes/custom/comment/list_embed_base.pug

@@ -39,7 +39,7 @@
 						a(
 							title="{{ _('Handy guide of Markdown syntax') }}",
 							target="_blank",
-							href="https://guides.github.com/features/mastering-markdown/")
+							href="http://commonmark.org/help/")
 							span {{ _('markdown cheatsheet') }}
 			| {% endblock can_post_comment %}
 

src/templates/nodes/custom/page/view_embed.pug

@@ -1,24 +1,24 @@
 | {% extends 'projects/landing.html' %}
 
 | {% block body %}
+
 | {% if node.picture %}
 header
-	img.header(src="{{ node.picture.thumbnail('l', api=api) }}")
+	.jumbotron.jumbotron-fluid(
+		style="background-image: url('{{ node.picture.thumbnail('h', api=api) }}'); background-position: 50% 50%;")
 | {% endif  %}
+
+| {# Secondary Navigation #}
 | {% block navbar_secondary %}
 | {{ super() }}
 | {% endblock navbar_secondary %}
-#node-container
-	#node-overlay
 
-	section.node-details-container.page
+.container.landing
-
+	section.node-details-container.project
-		.node-details-header
+		.node-details-title.container
-			.node-title#node-title
+			h1 {{node.name}}
-				| {{node.name}}
 
 			| {% if node.description %}
-		.node-details-description#node-description
 			| {{ node | markdowned('description') }}
 			| {% endif %}
 

src/templates/projects/_macros.pug

@@ -1,36 +1,41 @@
 | {% macro render_secondary_navigation(project, pages=None) %}
-nav.navbar-secondary
+.container.navbar-secondary
-	.navbar-container
+	ul.nav.justify-content-left
-		nav.collapse.navbar-collapse
+		li.nav-item
-			ul.nav.navbar-nav.navbar-right
+			a.nav-link.nav-title(
-				li
-					a.navbar-item(
 				href="{{ url_for('projects.view', project_url=project.url) }}",
-						title="{{ project.name }} Homepage")
+				title="{{ project.name }} Homepage") {{ project.name }}
-						span
+		li.nav-item
-							b {{ project.name }}
+			a.nav-link(
-				li
-					a.navbar-item(
 				href="{{ url_for('main.project_blog', project_url=project.url) }}",
-					title="Project Blog",
+				class="{% if title == 'updates' %}active{% endif %}") Updates
-					class="{% if category == 'blog' %}active{% endif %}")
-						span Blog
 		| {% if pages %}
-				| {% for p in pages %}
+		| {% for page in pages %}
-				li
+		li.nav-item
-					a.navbar-item(
+			a.nav-link(
-					href="{{ url_for('projects.view_node', project_url=project.url, node_id=p._id) }}",
+				href="{{ url_for('projects.view_node', project_url=project.url, node_id=page._id) }}",
-					title="{{ p.name }}",
+				class="{% if title == 'updates' %}active{% endif %}") {{ page.name }}
-					class="{% if category == 'page' %}active{% endif %}")
-						span {{ p.name }}
 		| {% endfor %}
 		| {% endif %}
+		li.nav-item
+			a.nav-link(
+				href="/projects/gallery.html",
+				class="{% if title == 'gallery' %}active{% endif %}") Gallery
+		li.nav-item
+			a.nav-link(
+				href="#",
+				class="{% if title == 'assets' %}active{% endif %}") Assets
 		| {% if project.nodes_featured %}
-				li
+		| {# In some cases featured_nodes might might be embedded #}
-					a.navbar-item(
+		| {% if '_id' in project.nodes_featured[0] %}
-					href="{{ url_for('projects.view_node', project_url=project.url, node_id=project.nodes_featured[0]) }}",
+		| {% set featured_node_id=project.nodes_featured[0]._id %}
-					title="Explore {{ project.name }}",
+		| {% else %}
-					class="{% if category == 'blog' %}active{% endif %}")
+		| {% set featured_node_id=project.nodes_featured[0] %}
-						span Explore
 		| {% endif %}
+		| {% endif %}
+		li.nav-item
+			a.nav-link(
+			href="{{ url_for('projects.view_node', project_url=project.url, node_id=featured_node_id) }}",
+			title="Explore {{ project.name }}") Dashboard
+
 | {% endmacro %}

tests/test_api/test_cerberus.py

@@ -0,0 +1,199 @@
+"""Test that what we feed to Cerberus actually works.
+
+This'll help us upgrade to new versions of Cerberus.
+"""
+
+import unittest
+from pillar.tests import AbstractPillarTest
+
+from bson import ObjectId
+
+
+class CerberusCanaryTest(unittest.TestCase):
+
+    def _canary_test(self, validator):
+        groups_schema = {'name': {'type': 'string', 'required': True}}
+
+        # On error, validate_schema() raises ValidationError
+        validator.validate_schema(groups_schema)
+
+        # On error, validate() returns False
+        self.assertTrue(validator.validate({'name': 'je moeder'}, groups_schema))
+        self.assertFalse(validator.validate({'je moeder': 'op je hoofd'}, groups_schema))
+
+    def test_canary(self):
+        import cerberus
+
+        validator = cerberus.Validator()
+        self._canary_test(validator)
+
+    def test_our_validator_simple(self):
+        from pillar.api import custom_field_validation
+
+        validator = custom_field_validation.ValidateCustomFields()
+        self._canary_test(validator)
+
+
+class ValidationTest(AbstractPillarTest):
+    def setUp(self):
+        super().setUp()
+
+        from pillar.api import custom_field_validation
+
+        self.validator = custom_field_validation.ValidateCustomFields()
+        self.user_id = ObjectId(8 * 'abc')
+        self.ensure_user_exists(self.user_id, 'Tést Üsâh')
+
+    def assertValid(self, document, schema):
+        with self.app.app_context():
+            is_valid = self.validator.validate(document, schema)
+        self.assertTrue(is_valid, f'errors: {self.validator.errors}')
+
+    def assertInvalid(self, document, schema):
+        with self.app.app_context():
+            is_valid = self.validator.validate(document, schema)
+        self.assertFalse(is_valid)
+
+
+class ProjectValidationTest(ValidationTest):
+
+    def test_empty(self):
+        from pillar.api.eve_settings import projects_schema
+        self.assertInvalid({}, projects_schema)
+
+    def test_simple_project(self):
+        from pillar.api.eve_settings import projects_schema
+
+        project = {
+            'name': 'Té Ærhüs',
+            'user': self.user_id,
+            'category': 'assets',
+            'is_private': False,
+            'status': 'published',
+        }
+
+        self.assertValid(project, projects_schema)
+
+    def test_with_node_types(self):
+        from pillar.api.eve_settings import projects_schema
+        from pillar.api import node_types
+
+        project = {
+            'name': 'Té Ærhüs',
+            'user': self.user_id,
+            'category': 'assets',
+            'is_private': False,
+            'status': 'published',
+            'node_types': [node_types.node_type_asset,
+                           node_types.node_type_comment]
+        }
+
+        self.assertValid(project, projects_schema)
+
+
+class NodeValidationTest(ValidationTest):
+    def setUp(self):
+        super().setUp()
+        self.pid, self.project = self.ensure_project_exists()
+
+    def test_empty(self):
+        from pillar.api.eve_settings import nodes_schema
+        self.assertInvalid({}, nodes_schema)
+
+    def test_asset(self):
+        from pillar.api.eve_settings import nodes_schema
+
+        file_id, _ = self.ensure_file_exists()
+
+        node = {
+            'name': '"The Harmless Prototype™"',
+            'project': self.pid,
+            'node_type': 'asset',
+            'properties': {
+                'status': 'published',
+                'content_type': 'image',
+                'file': file_id,
+            },
+            'user': self.user_id,
+            'short_code': 'ABC333',
+        }
+        self.assertValid(node, nodes_schema)
+
+    def test_asset_invalid_properties(self):
+        from pillar.api.eve_settings import nodes_schema
+
+        file_id, _ = self.ensure_file_exists()
+
+        node = {
+            'name': '"The Harmless Prototype™"',
+            'project': self.pid,
+            'node_type': 'asset',
+            'properties': {
+                'status': 'invalid-status',
+                'content_type': 'image',
+                'file': file_id,
+            },
+            'user': self.user_id,
+            'short_code': 'ABC333',
+        }
+        self.assertInvalid(node, nodes_schema)
+
+    def test_comment(self):
+        from pillar.api.eve_settings import nodes_schema
+
+        file_id, _ = self.ensure_file_exists()
+
+        node = {
+            'name': '"The Harmless Prototype™"',
+            'project': self.pid,
+            'node_type': 'asset',
+            'properties': {
+                'status': 'published',
+                'content_type': 'image',
+                'file': file_id,
+            },
+            'user': self.user_id,
+            'short_code': 'ABC333',
+        }
+        node_id = self.create_node(node)
+
+        comment = {
+            'name': 'comment on some node',
+            'project': self.pid,
+            'node_type': 'comment',
+            'properties': {
+                'content': 'this is a comment',
+                'status': 'published',
+            },
+            'parent': node_id,
+        }
+        self.assertValid(comment, nodes_schema)
+
+
+class IPRangeValidatorTest(ValidationTest):
+    schema = {'iprange': {'type': 'iprange', 'required': True}}
+
+    def assertValid(self, document, schema=None):
+        return super().assertValid(document, schema or self.schema)
+
+    def assertInvalid(self, document, schema=None):
+        return super().assertInvalid(document, schema or self.schema)
+
+    def test_ipv6(self):
+        self.assertValid({'iprange': '2a03:b0c0:0:1010::8fe:6ef1'})
+        self.assertValid({'iprange': '0:0:0:0:0:ffff:102:304'})
+        self.assertValid({'iprange': '2a03:b0c0:0:1010::8fe:6ef1/120'})
+        self.assertValid({'iprange': 'ff06::/8'})
+        self.assertValid({'iprange': '::/8'})
+        self.assertValid({'iprange': '::/1'})
+        self.assertValid({'iprange': '::1/128'})
+        self.assertValid({'iprange': '::'})
+        self.assertInvalid({'iprange': '::/0'})
+        self.assertInvalid({'iprange': 'barbled'})
+
+    def test_ipv4(self):
+        self.assertValid({'iprange': '1.2.3.4'})
+        self.assertValid({'iprange': '1.2.3.4/24'})
+        self.assertValid({'iprange': '127.0.0.0/8'})
+        self.assertInvalid({'iprange': '127.0.0.0/0'})
+        self.assertInvalid({'iprange': 'garbled'})

tests/test_api/test_oauth.py

@@ -12,7 +12,7 @@ class OAuthTests(AbstractPillarTest):
 
         oauth_provider = OAuthSignIn.get_provider('blender-id')
         self.assertIsInstance(oauth_provider, BlenderIdSignIn)
-        self.assertEqual(oauth_provider.service.base_url, 'http://blender-id:8000/api/')
+        self.assertEqual(oauth_provider.service.base_url, 'http://id.local:8001/api/')
 
     def test_provider_not_implemented(self):
         from pillar.auth.oauth import OAuthSignIn, ProviderNotImplemented
@@ -46,11 +46,11 @@ class OAuthTests(AbstractPillarTest):
     def test_provider_callback_happy(self):
         from pillar.auth.oauth import OAuthSignIn
 
-        responses.add(responses.POST, 'http://blender-id:8000/oauth/token',
+        responses.add(responses.POST, 'http://id.local:8001/oauth/token',
                       json={'access_token': 'successful-token'},
                       status=200)
 
-        responses.add(responses.GET, 'http://blender-id:8000/api/user',
+        responses.add(responses.GET, 'http://id.local:8001/api/user',
                       json={'id': '7',
                             'email': 'harry@blender.org'},
                       status=200)

tests/test_api/test_utils.py

@@ -184,3 +184,31 @@ class NodeSetattrTest(unittest.TestCase):
 
         node_setattr(node, 'b.complex', {None: 5})
         self.assertEqual({'b': {'complex': {None: 5}}}, node)
+
+
+class TestRating(unittest.TestCase):
+    def test_hotness(self):
+        """We expect the sorted values to reflect the original order in the
+        list.
+        """
+        from datetime import datetime, timezone
+        from pillar.api.utils.rating import hot
+        t = datetime(2017, 2, 11, 0, 0, 0, 0, timezone.utc)
+        y = datetime(2017, 2, 10, 0, 0, 0, 0, timezone.utc)
+        w = datetime(2017, 2, 5, 0, 0, 0, 0, timezone.utc)
+        cases = [
+            (hot(1, 8, t), 'today super bad'),
+            (hot(0, 3, t), 'today slightly worse'),
+            (hot(0, 2, y), 'yesterday bad'),
+            (hot(0, 2, t), 'today bad'),
+            (hot(4, 4, w), 'last week controversial'),
+            (hot(7, 1, w), 'last week very good'),
+            (hot(5, 1, y), 'yesterday medium'),
+            (hot(5, 0, y), 'yesterday good'),
+            (hot(7, 1, y), 'yesterday very good'),
+            (hot(4, 4, t), 'today controversial'),
+            (hot(7, 1, t), 'today very good'),
+        ]
+        sorted_by_hot = sorted(cases, key=lambda tup: tup[0])
+        for idx, t in enumerate(sorted_by_hot):
+            self.assertEqual(cases[idx][0], t[0])

tests/test_shortcodes.py

@@ -40,7 +40,7 @@ class DemoTest(unittest.TestCase):
         self.assertEqual('<dl><dt>test</dt><dt>ü</dt><dd>é</dd></dl>', render('{test ü="é"}'))
 
 
-class YouTubeTest(unittest.TestCase):
+class YouTubeTest(AbstractPillarTest):
     def test_missing(self):
         from pillar.shortcodes import render
 
@@ -104,6 +104,19 @@ class YouTubeTest(unittest.TestCase):
             render('{youtube "https://www.youtube.com/watch?v=NwVGvcIrNWA" width=5 height="3"}')
         )
 
+    def test_user_no_cap(self):
+        from pillar.shortcodes import render
+
+        with self.app.app_context():
+            # Anonymous user, so no subscriber capability.
+            self.assertEqual('', render('{youtube ABCDEF cap=subscriber}'))
+            self.assertEqual('', render('{youtube ABCDEF cap="subscriber"}'))
+            self.assertEqual(
+                '<p class="shortcode nocap">Aðeins áskrifendur hafa aðgang að þessu efni.</p>',
+                render('{youtube ABCDEF'
+                       ' cap="subscriber"'
+                       ' nocap="Aðeins áskrifendur hafa aðgang að þessu efni."}'))
+
 
 class IFrameTest(AbstractPillarTest):
     def test_missing_cap(self):

tests/test_web/test_jinja.py

@@ -11,8 +11,8 @@ class MarkdownTest(unittest.TestCase):
     def test_bleached(self):
         from pillar.web import jinja
 
-        self.assertEqual('<script>alert("hey");<script>',
+        self.assertEqual('<script>alert("hey");</script>',
-                         jinja.do_markdown('<script>alert("hey");<script>').strip())
+                         jinja.do_markdown('<script>alert("hey");</script>').strip())
 
     def test_degenerate(self):
         from pillar.web import jinja