T53890: Improving static content serving

Static files are now served with an 8-character hash before the last
extension. For example, `tutti.min.js` is now served as
`tutti.min.abcd1234.js`. When doing a request the hash is removed before
serving the static file.

The hash must be 8 characters long, and is taken from STATIC_FILE_HASH.
It is up to the deployment to change this configuration variable
whenever static files change. This forces browsers that download newly
deployed HTML to also refresh the dependencies (most importantly
JS/CSS).

For this to work, the URL must be built with `url_for('static_xxx',
filename='/path/to/file')`. The 'static' module still returns regular,
hashless URLs.

pillar/__init__.py

@@ -101,6 +101,7 @@ class PillarServer(BlinkerCompatibleEve):
         self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
         self.log.info('Creating new instance from %r', self.app_root)
 
+        self._config_url_map()
         self._config_auth_token_hmac_key()
         self._config_tempdirs()
         self._config_git()
@@ -171,6 +172,19 @@ class PillarServer(BlinkerCompatibleEve):
         if self.config['DEBUG']:
             log.info('Pillar starting, debug=%s', self.config['DEBUG'])
 
+    def _config_url_map(self):
+        """Extend Flask url_map with our own converters."""
+        import secrets, re
+        from . import flask_extra
+
+        if not self.config.get('STATIC_FILE_HASH'):
+            self.log.warning('STATIC_FILE_HASH is empty, generating random one')
+            f = open('/data/git/blender-cloud/config_local.py', 'a')
+            h = re.sub(r'[_.~-]', '', secrets.token_urlsafe())[:8]
+            self.config['STATIC_FILE_HASH'] = h
+
+        self.url_map.converters['hashed_path'] = flask_extra.HashedPathConverter
+
     def _config_auth_token_hmac_key(self):
         """Load AUTH_TOKEN_HMAC_KEY, falling back to SECRET_KEY."""
 
@@ -529,7 +543,7 @@ class PillarServer(BlinkerCompatibleEve):
         from pillar.web.staticfile import PillarStaticFile
 
         view_func = PillarStaticFile.as_view(endpoint_name, static_folder=static_folder)
-        self.add_url_rule('%s/<path:filename>' % url_prefix, view_func=view_func)
+        self.add_url_rule(f'{url_prefix}/<hashed_path:filename>', view_func=view_func)
 
     def process_extensions(self):
         """This is about Eve extensions, not Pillar extensions."""

pillar/config.py

@@ -247,3 +247,11 @@ SMTP_TIMEOUT = 30  # timeout in seconds, https://docs.python.org/3/library/smtpl
 MAIL_RETRY = 180  # in seconds, delay until trying to send an email again.
 MAIL_DEFAULT_FROM_NAME = 'Blender Cloud'
 MAIL_DEFAULT_FROM_ADDR = 'cloudsupport@localhost'
+
+SEND_FILE_MAX_AGE_DEFAULT = 3600 * 24 * 365  # seconds
+
+# MUST be 8 characters long, see pillar.flask_extra.HashedPathConverter
+# Intended to be changed for every deploy. If it is empty, a random hash will
+# be used. Note that this causes extra traffic, since every time the process
+# restarts the URLs will be different.
+STATIC_FILE_HASH = ''

pillar/flask_extra.py

@@ -1,5 +1,34 @@
+import re
 import functools
+
 import flask
+import werkzeug.routing
+
+
+class HashedPathConverter(werkzeug.routing.PathConverter):
+    """Allows for files `xxx.yyy.js` to be served as `xxx.yyy.abc123.js`.
+
+    The hash code is placed before the last extension.
+    """
+    weight = 300
+    # Hash length is hard-coded to 8 characters for now.
+    hash_re = re.compile(r'\.([a-zA-Z0-9]{8})(?=\.[^.]+$)')
+
+    @functools.lru_cache(maxsize=1024)
+    def to_python(self, from_url: str) -> str:
+        return self.hash_re.sub('', from_url)
+
+    @functools.lru_cache(maxsize=1024)
+    def to_url(self, filepath: str) -> str:
+        try:
+            dotidx = filepath.rindex('.')
+        except ValueError:
+            # Happens when there is no dot. Very unlikely.
+            return filepath
+
+        current_hash = flask.current_app.config['STATIC_FILE_HASH']
+        before, after = filepath[:dotidx], filepath[dotidx:]
+        return f'{before}.{current_hash}{after}'
 
 
 def add_response_headers(headers: dict):

pillar/tests/config_testing.py

@@ -42,3 +42,6 @@ ELASTIC_INDICES = {
     'NODE': 'test_nodes',
     'USER': 'test_users',
 }
+
+# MUST be 8 characters long, see pillar.flask_extra.HashedPathConverter
+STATIC_FILE_HASH = 'abcd1234'

pillar/web/staticfile.py

@@ -1,12 +1,21 @@
 """Static file handling"""
+import logging
 
 import flask
 import flask.views
 
+log = logging.getLogger(__name__)
+
 
 class PillarStaticFile(flask.views.MethodView):
     def __init__(self, static_folder):
         self.static_folder = static_folder
 
     def get(self, filename):
+        log.debug('Request file %s/%s', self.static_folder, filename)
         return flask.send_from_directory(self.static_folder, filename)
+        return flask.send_from_directory(
+            self.static_folder, filename,
+            conditional=True,
+            add_etags=True,
+        )