T53890: Improving static content serving

Static files are now served with an 8-character hash before the last
extension. For example, `tutti.min.js` is now served as
`tutti.min.abcd1234.js`. When doing a request the hash is removed before
serving the static file.

The hash must be 8 characters long, and is taken from STATIC_FILE_HASH.
It is up to the deployment to change this configuration variable
whenever static files change. This forces browsers that download newly
deployed HTML to also refresh the dependencies (most importantly
JS/CSS).

For this to work, the URL must be built with `url_for('static_xxx',
filename='/path/to/file')`. The 'static' module still returns regular,
hashless URLs.

pillar/__init__.py

@@ -101,6 +101,7 @@ class PillarServer(BlinkerCompatibleEve):
         self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
         self.log.info('Creating new instance from %r', self.app_root)
 
+        self._config_url_map()
         self._config_auth_token_hmac_key()
         self._config_tempdirs()
         self._config_git()
@@ -171,6 +172,19 @@ class PillarServer(BlinkerCompatibleEve):
         if self.config['DEBUG']:
             log.info('Pillar starting, debug=%s', self.config['DEBUG'])
 
+    def _config_url_map(self):
+        """Extend Flask url_map with our own converters."""
+        import secrets, re
+        from . import flask_extra
+
+        if not self.config.get('STATIC_FILE_HASH'):
+            self.log.warning('STATIC_FILE_HASH is empty, generating random one')
+            f = open('/data/git/blender-cloud/config_local.py', 'a')
+            h = re.sub(r'[_.~-]', '', secrets.token_urlsafe())[:8]
+            self.config['STATIC_FILE_HASH'] = h
+
+        self.url_map.converters['hashed_path'] = flask_extra.HashedPathConverter
+
     def _config_auth_token_hmac_key(self):
         """Load AUTH_TOKEN_HMAC_KEY, falling back to SECRET_KEY."""
 
@@ -529,7 +543,7 @@ class PillarServer(BlinkerCompatibleEve):
         from pillar.web.staticfile import PillarStaticFile
 
         view_func = PillarStaticFile.as_view(endpoint_name, static_folder=static_folder)
-        self.add_url_rule('%s/<path:filename>' % url_prefix, view_func=view_func)
+        self.add_url_rule(f'{url_prefix}/<hashed_path:filename>', view_func=view_func)
 
     def process_extensions(self):
         """This is about Eve extensions, not Pillar extensions."""

pillar/config.py

@@ -247,3 +247,11 @@ SMTP_TIMEOUT = 30  # timeout in seconds, https://docs.python.org/3/library/smtpl
 MAIL_RETRY = 180  # in seconds, delay until trying to send an email again.
 MAIL_DEFAULT_FROM_NAME = 'Blender Cloud'
 MAIL_DEFAULT_FROM_ADDR = 'cloudsupport@localhost'
+
+SEND_FILE_MAX_AGE_DEFAULT = 3600 * 24 * 365  # seconds
+
+# MUST be 8 characters long, see pillar.flask_extra.HashedPathConverter
+# Intended to be changed for every deploy. If it is empty, a random hash will
+# be used. Note that this causes extra traffic, since every time the process
+# restarts the URLs will be different.
+STATIC_FILE_HASH = ''

pillar/flask_extra.py

@@ -1,5 +1,34 @@
+import re
 import functools
+
 import flask
+import werkzeug.routing
+
+
+class HashedPathConverter(werkzeug.routing.PathConverter):
+    """Allows for files `xxx.yyy.js` to be served as `xxx.yyy.abc123.js`.
+
+    The hash code is placed before the last extension.
+    """
+    weight = 300
+    # Hash length is hard-coded to 8 characters for now.
+    hash_re = re.compile(r'\.([a-zA-Z0-9]{8})(?=\.[^.]+$)')
+
+    @functools.lru_cache(maxsize=1024)
+    def to_python(self, from_url: str) -> str:
+        return self.hash_re.sub('', from_url)
+
+    @functools.lru_cache(maxsize=1024)
+    def to_url(self, filepath: str) -> str:
+        try:
+            dotidx = filepath.rindex('.')
+        except ValueError:
+            # Happens when there is no dot. Very unlikely.
+            return filepath
+
+        current_hash = flask.current_app.config['STATIC_FILE_HASH']
+        before, after = filepath[:dotidx], filepath[dotidx:]
+        return f'{before}.{current_hash}{after}'
 
 
 def add_response_headers(headers: dict):

pillar/tests/config_testing.py

@@ -42,3 +42,6 @@ ELASTIC_INDICES = {
     'NODE': 'test_nodes',
     'USER': 'test_users',
 }
+
+# MUST be 8 characters long, see pillar.flask_extra.HashedPathConverter
+STATIC_FILE_HASH = 'abcd1234'

pillar/web/staticfile.py

@@ -1,12 +1,21 @@
 """Static file handling"""
+import logging
 
 import flask
 import flask.views
 
+log = logging.getLogger(__name__)
+
 
 class PillarStaticFile(flask.views.MethodView):
     def __init__(self, static_folder):
         self.static_folder = static_folder
 
     def get(self, filename):
+        log.debug('Request file %s/%s', self.static_folder, filename)
         return flask.send_from_directory(self.static_folder, filename)
+        return flask.send_from_directory(
+            self.static_folder, filename,
+            conditional=True,
+            add_etags=True,
+        )

src/templates/layout.pug

@@ -39,8 +39,8 @@ html(lang="en")
 
 			loadCSS( "//fonts.googleapis.com/css?family=Roboto:300,400" );
 
-		script(src="{{ url_for('static_pillar', filename='assets/js/markdown.min.js', v=17320171) }}")
+		script(src="{{ url_for('static_pillar', filename='assets/js/markdown.min.js') }}")
-		script(src="{{ url_for('static_pillar', filename='assets/js/tutti.min.js', v=17320171) }}")
+		script(src="{{ url_for('static_pillar', filename='assets/js/tutti.min.js') }}")
 
 		link(href="{{ url_for('static', filename='assets/img/favicon.png') }}", rel="shortcut icon")
 		link(href="{{ url_for('static', filename='assets/img/apple-touch-icon-precomposed.png') }}", rel="icon apple-touch-icon-precomposed", sizes="192x192")
@@ -50,12 +50,12 @@ html(lang="en")
 		| {% block head %}{% endblock %}
 
 		| {% block css %}
-		link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css', v=17320171) }}", rel="stylesheet")
+		link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css') }}", rel="stylesheet")
-		link(href="{{ url_for('static_pillar', filename='assets/css/base.css', v=17320171) }}", rel="stylesheet")
+		link(href="{{ url_for('static_pillar', filename='assets/css/base.css') }}", rel="stylesheet")
 		| {% if title == 'blog' %}
-		link(href="{{ url_for('static_pillar', filename='assets/css/blog.css', v=17320171) }}", rel="stylesheet")
+		link(href="{{ url_for('static_pillar', filename='assets/css/blog.css') }}", rel="stylesheet")
 		| {% else %}
-		link(href="{{ url_for('static_pillar', filename='assets/css/main.css', v=17320171) }}", rel="stylesheet")
+		link(href="{{ url_for('static_pillar', filename='assets/css/main.css') }}", rel="stylesheet")
 		| {% endif %}
 		| {% endblock %}
 
@@ -81,7 +81,7 @@ html(lang="en")
 			| {% endblock footer %}
 		| {% endblock footer_container%}
 
-		script(src="{{ url_for('static_pillar', filename='assets/js/vendor/jquery.bootstrap-3.3.7.min.js', v=17320171) }}")
+		script(src="{{ url_for('static_pillar', filename='assets/js/vendor/jquery.bootstrap-3.3.7.min.js') }}")
 
 
 		| {% block footer_scripts_pre %}{% endblock %}

src/templates/nodes/custom/blog/index.pug

@@ -5,7 +5,7 @@
 
 | {% block css %}
 | {{ super() }}
-link(href="{{ url_for('static_pillar', filename='assets/css/blog.css', v=17320171) }}", rel="stylesheet")
+link(href="{{ url_for('static_pillar', filename='assets/css/blog.css') }}", rel="stylesheet")
 | {% endblock %}
 
 | {% block project_context %}

src/templates/nodes/custom/post/view.pug

@@ -21,7 +21,7 @@ meta(property="og:image", content="{{ node.picture.thumbnail('l', api=api) }}")
 
 | {% block css %}
 | {{ super() }}
-link(href="{{ url_for('static_pillar', filename='assets/css/blog.css', v=17320171) }}", rel="stylesheet")
+link(href="{{ url_for('static_pillar', filename='assets/css/blog.css') }}", rel="stylesheet")
 | {% endblock %}
 
 | {% block project_context %}

src/templates/nodes/search.pug

@@ -2,9 +2,9 @@
 | {% block page_title %}Search{% if project %} {{ project.name }}{% endif %}{% endblock %}
 
 | {% block head %}
-script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js', v=9112017) }}")
+script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js') }}")
-script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js', v=9112017) }}")
+script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js') }}")
-script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js', v=9112017) }}")
+script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js') }}")
 | {% endblock %}
 
 | {% block og %}

src/templates/projects/view.pug

@@ -66,20 +66,20 @@ meta(property="og:url", content="{{url_for('projects.view', project_url=project.
 | {% endblock %}
 
 | {% block head %}
-link(href="{{ url_for('static_pillar', filename='assets/jstree/themes/default/style.min.css', v=9112017) }}", rel="stylesheet")
+link(href="{{ url_for('static_pillar', filename='assets/jstree/themes/default/style.min.css') }}", rel="stylesheet")
 | {% if node %}
 link(rel="amphtml", href="{{ url_for('nodes.view', node_id=node._id, _external=True, format='amp') }}")
 | {% endif %}
 
-script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js', v=9112017) }}")
+script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js') }}")
-script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js', v=9112017) }}")
+script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js') }}")
-script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js', v=9112017) }}")
+script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js') }}")
 | {% endblock %}
 
 | {% block css %}
-link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css', v=9112017) }}", rel="stylesheet")
+link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css') }}", rel="stylesheet")
-link(href="{{ url_for('static_pillar', filename='assets/css/base.css', v=9112017) }}", rel="stylesheet")
+link(href="{{ url_for('static_pillar', filename='assets/css/base.css') }}", rel="stylesheet")
-link(href="{{ url_for('static_pillar', filename='assets/css/project-main.css', v=9112017) }}", rel="stylesheet")
+link(href="{{ url_for('static_pillar', filename='assets/css/project-main.css') }}", rel="stylesheet")
 | {% endblock %}
 
 | {% block body %}
@@ -280,7 +280,7 @@ link(href="{{ url_for('static_pillar', filename='assets/css/project-main.css', v
 
 | {% if project.has_method('PUT') %}
 | {# JS containing the Edit, Add, Featured, and Move functions #}
-script(type="text/javascript", src="{{ url_for('static_pillar', filename='assets/js/project-edit.min.js', v=9112017) }}")
+script(type="text/javascript", src="{{ url_for('static_pillar', filename='assets/js/project-edit.min.js') }}")
 | {% endif %}
 
 script.

src/templates/projects/view_theatre.pug

@@ -29,8 +29,8 @@ li
 | {% endblock %}
 
 | {% block css %}
-link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css', v=171020161) }}", rel="stylesheet")
+link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css') }}", rel="stylesheet")
-link(href="{{ url_for('static_pillar', filename='assets/css/theatre.css', v=171020161) }}", rel="stylesheet")
+link(href="{{ url_for('static_pillar', filename='assets/css/theatre.css') }}", rel="stylesheet")
 | {% endblock %}
 
 | {% block body %}

tests/test_flask_extra.py

@@ -2,6 +2,8 @@ import unittest
 
 import flask
 
+from pillar.tests import AbstractPillarTest
+
 
 class FlaskExtraTest(unittest.TestCase):
     def test_vary_xhr(self):
@@ -84,3 +86,25 @@ class EnsureSchemaTest(unittest.TestCase):
             self.assertEqual('/some/path/only', pillar.flask_extra.ensure_schema('/some/path/only'))
             self.assertEqual('https://hostname/path',
                              pillar.flask_extra.ensure_schema('//hostname/path'))
+
+
+class HashedPathConverterTest(AbstractPillarTest):
+    def test_to_python(self):
+        from pillar.flask_extra import HashedPathConverter
+
+        hpc = HashedPathConverter({})
+        self.assertEqual('/path/to/file.min.js', hpc.to_python('/path/to/file.min.abcd1234.js'))
+        self.assertEqual('/path/to/file.js', hpc.to_python('/path/to/file.abcd1234.js'))
+        self.assertEqual('/path/to/file', hpc.to_python('/path/to/file'))
+        self.assertEqual('', hpc.to_python(''))
+
+    def test_to_url(self):
+        from pillar.flask_extra import HashedPathConverter
+
+        hpc = HashedPathConverter({})
+
+        with self.app.app_context():
+            self.assertEqual('/path/to/file.min.abcd1234.js', hpc.to_url('/path/to/file.min.js'))
+            self.assertEqual('/path/to/file.abcd1234.js', hpc.to_url('/path/to/file.js'))
+            self.assertEqual('/path/to/file', hpc.to_url('/path/to/file'))
+            self.assertEqual('', hpc.to_url(''))