archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

T53890: Improving static content serving

Browse Source 
Static files are now served with an 8-character hash before the last
extension. For example, `tutti.min.js` is now served as
`tutti.min.abcd1234.js`. When doing a request the hash is removed before
serving the static file.

The hash must be 8 characters long, and is taken from STATIC_FILE_HASH.
It is up to the deployment to change this configuration variable
whenever static files change. This forces browsers that download newly
deployed HTML to also refresh the dependencies (most importantly
JS/CSS).

For this to work, the URL must be built with `url_for('static_xxx',
filename='/path/to/file')`. The 'static' module still returns regular,
hashless URLs.
This commit is contained in:
Sybren A. Stüvel 2018-03-23 16:34:33 +01:00
parent 0cf45c0d78
commit 12272750c3
12 changed files with 110 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
pillar/__init__.py
View File

@ -101,6 +101,7 @@ class PillarServer(BlinkerCompatibleEve):
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.log.info('Creating new instance from %r', self.app_root)
self._config_url_map()
self._config_auth_token_hmac_key()
self._config_tempdirs()
self._config_git()
@ -171,6 +172,19 @@ class PillarServer(BlinkerCompatibleEve):
if self.config['DEBUG']:
log.info('Pillar starting, debug=%s', self.config['DEBUG'])
def _config_url_map(self):
"""Extend Flask url_map with our own converters."""
import secrets, re
from . import flask_extra
if not self.config.get('STATIC_FILE_HASH'):
self.log.warning('STATIC_FILE_HASH is empty, generating random one')
f = open('/data/git/blender-cloud/config_local.py', 'a')
h = re.sub(r'[_.~-]', '', secrets.token_urlsafe())[:8]
self.config['STATIC_FILE_HASH'] = h
self.url_map.converters['hashed_path'] = flask_extra.HashedPathConverter
def _config_auth_token_hmac_key(self):
"""Load AUTH_TOKEN_HMAC_KEY, falling back to SECRET_KEY."""
@ -529,7 +543,7 @@ class PillarServer(BlinkerCompatibleEve):
from pillar.web.staticfile import PillarStaticFile
view_func = PillarStaticFile.as_view(endpoint_name, static_folder=static_folder)
self.add_url_rule('%s/<path:filename>' % url_prefix, view_func=view_func)
self.add_url_rule(f'{url_prefix}/<hashed_path:filename>', view_func=view_func)
def process_extensions(self):
"""This is about Eve extensions, not Pillar extensions."""

8
pillar/config.py
View File

@ -247,3 +247,11 @@ SMTP_TIMEOUT = 30 # timeout in seconds, https://docs.python.org/3/library/smtpl
MAIL_RETRY = 180 # in seconds, delay until trying to send an email again.
MAIL_DEFAULT_FROM_NAME = 'Blender Cloud'
MAIL_DEFAULT_FROM_ADDR = 'cloudsupport@localhost'
SEND_FILE_MAX_AGE_DEFAULT = 3600 * 24 * 365 # seconds
# MUST be 8 characters long, see pillar.flask_extra.HashedPathConverter
# Intended to be changed for every deploy. If it is empty, a random hash will
# be used. Note that this causes extra traffic, since every time the process
# restarts the URLs will be different.
STATIC_FILE_HASH = ''

29
pillar/flask_extra.py
View File

@ -1,5 +1,34 @@
import re
import functools
import flask
import werkzeug.routing
class HashedPathConverter(werkzeug.routing.PathConverter):
"""Allows for files `xxx.yyy.js` to be served as `xxx.yyy.abc123.js`.
The hash code is placed before the last extension.
"""
weight = 300
# Hash length is hard-coded to 8 characters for now.
hash_re = re.compile(r'\.([a-zA-Z0-9]{8})(?=\.[^.]+$)')
@functools.lru_cache(maxsize=1024)
def to_python(self, from_url: str) -> str:
return self.hash_re.sub('', from_url)
@functools.lru_cache(maxsize=1024)
def to_url(self, filepath: str) -> str:
try:
dotidx = filepath.rindex('.')
except ValueError:
# Happens when there is no dot. Very unlikely.
return filepath
current_hash = flask.current_app.config['STATIC_FILE_HASH']
before, after = filepath[:dotidx], filepath[dotidx:]
return f'{before}.{current_hash}{after}'
def add_response_headers(headers: dict):

3
pillar/tests/config_testing.py
View File

@ -42,3 +42,6 @@ ELASTIC_INDICES = {
'NODE': 'test_nodes',
'USER': 'test_users',
}
# MUST be 8 characters long, see pillar.flask_extra.HashedPathConverter
STATIC_FILE_HASH = 'abcd1234'

9
pillar/web/staticfile.py
View File

@ -1,12 +1,21 @@
"""Static file handling"""
import logging
import flask
import flask.views
log = logging.getLogger(__name__)
class PillarStaticFile(flask.views.MethodView):
def __init__(self, static_folder):
self.static_folder = static_folder
def get(self, filename):
log.debug('Request file %s/%s', self.static_folder, filename)
return flask.send_from_directory(self.static_folder, filename)
return flask.send_from_directory(
self.static_folder, filename,
conditional=True,
add_etags=True,
)

14
src/templates/layout.pug
View File

@ -39,8 +39,8 @@ html(lang="en")
loadCSS( "//fonts.googleapis.com/css?family=Roboto:300,400" );
script(src="{{ url_for('static_pillar', filename='assets/js/markdown.min.js', v=17320171) }}")
script(src="{{ url_for('static_pillar', filename='assets/js/tutti.min.js', v=17320171) }}")
script(src="{{ url_for('static_pillar', filename='assets/js/markdown.min.js') }}")
script(src="{{ url_for('static_pillar', filename='assets/js/tutti.min.js') }}")
link(href="{{ url_for('static', filename='assets/img/favicon.png') }}", rel="shortcut icon")
link(href="{{ url_for('static', filename='assets/img/apple-touch-icon-precomposed.png') }}", rel="icon apple-touch-icon-precomposed", sizes="192x192")
@ -50,12 +50,12 @@ html(lang="en")
| {% block head %}{% endblock %}
| {% block css %}
link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css', v=17320171) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/base.css', v=17320171) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css') }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/base.css') }}", rel="stylesheet")
| {% if title == 'blog' %}
link(href="{{ url_for('static_pillar', filename='assets/css/blog.css', v=17320171) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/blog.css') }}", rel="stylesheet")
| {% else %}
link(href="{{ url_for('static_pillar', filename='assets/css/main.css', v=17320171) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/main.css') }}", rel="stylesheet")
| {% endif %}
| {% endblock %}
@ -81,7 +81,7 @@ html(lang="en")
| {% endblock footer %}
| {% endblock footer_container%}
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/jquery.bootstrap-3.3.7.min.js', v=17320171) }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/jquery.bootstrap-3.3.7.min.js') }}")
| {% block footer_scripts_pre %}{% endblock %}

2
src/templates/nodes/custom/blog/index.pug
View File

@ -5,7 +5,7 @@
| {% block css %}
| {{ super() }}
link(href="{{ url_for('static_pillar', filename='assets/css/blog.css', v=17320171) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/blog.css') }}", rel="stylesheet")
| {% endblock %}
| {% block project_context %}

2
src/templates/nodes/custom/post/view.pug
View File

@ -21,7 +21,7 @@ meta(property="og:image", content="{{ node.picture.thumbnail('l', api=api) }}")
| {% block css %}
| {{ super() }}
link(href="{{ url_for('static_pillar', filename='assets/css/blog.css', v=17320171) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/blog.css') }}", rel="stylesheet")
| {% endblock %}
| {% block project_context %}

6
src/templates/nodes/search.pug
View File

@ -2,9 +2,9 @@
| {% block page_title %}Search{% if project %} {{ project.name }}{% endif %}{% endblock %}
| {% block head %}
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js', v=9112017) }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js', v=9112017) }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js', v=9112017) }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js') }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js') }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js') }}")
| {% endblock %}
| {% block og %}

16
src/templates/projects/view.pug
View File

@ -66,20 +66,20 @@ meta(property="og:url", content="{{url_for('projects.view', project_url=project.
| {% endblock %}
| {% block head %}
link(href="{{ url_for('static_pillar', filename='assets/jstree/themes/default/style.min.css', v=9112017) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/jstree/themes/default/style.min.css') }}", rel="stylesheet")
| {% if node %}
link(rel="amphtml", href="{{ url_for('nodes.view', node_id=node._id, _external=True, format='amp') }}")
| {% endif %}
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js', v=9112017) }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js', v=9112017) }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js', v=9112017) }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js') }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js') }}")
script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js') }}")
| {% endblock %}
| {% block css %}
link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css', v=9112017) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/base.css', v=9112017) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/project-main.css', v=9112017) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css') }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/base.css') }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/project-main.css') }}", rel="stylesheet")
| {% endblock %}
| {% block body %}
@ -280,7 +280,7 @@ link(href="{{ url_for('static_pillar', filename='assets/css/project-main.css', v
| {% if project.has_method('PUT') %}
| {# JS containing the Edit, Add, Featured, and Move functions #}
script(type="text/javascript", src="{{ url_for('static_pillar', filename='assets/js/project-edit.min.js', v=9112017) }}")
script(type="text/javascript", src="{{ url_for('static_pillar', filename='assets/js/project-edit.min.js') }}")
| {% endif %}
script.

4
src/templates/projects/view_theatre.pug
View File

@ -29,8 +29,8 @@ li
| {% endblock %}
| {% block css %}
link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css', v=171020161) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/theatre.css', v=171020161) }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css') }}", rel="stylesheet")
link(href="{{ url_for('static_pillar', filename='assets/css/theatre.css') }}", rel="stylesheet")
| {% endblock %}
| {% block body %}

24
tests/test_flask_extra.py
View File

@ -2,6 +2,8 @@ import unittest
import flask
from pillar.tests import AbstractPillarTest
class FlaskExtraTest(unittest.TestCase):
def test_vary_xhr(self):
@ -84,3 +86,25 @@ class EnsureSchemaTest(unittest.TestCase):
self.assertEqual('/some/path/only', pillar.flask_extra.ensure_schema('/some/path/only'))
self.assertEqual('https://hostname/path',
pillar.flask_extra.ensure_schema('//hostname/path'))
class HashedPathConverterTest(AbstractPillarTest):
def test_to_python(self):
from pillar.flask_extra import HashedPathConverter
hpc = HashedPathConverter({})
self.assertEqual('/path/to/file.min.js', hpc.to_python('/path/to/file.min.abcd1234.js'))
self.assertEqual('/path/to/file.js', hpc.to_python('/path/to/file.abcd1234.js'))
self.assertEqual('/path/to/file', hpc.to_python('/path/to/file'))
self.assertEqual('', hpc.to_python(''))
def test_to_url(self):
from pillar.flask_extra import HashedPathConverter
hpc = HashedPathConverter({})
with self.app.app_context():
self.assertEqual('/path/to/file.min.abcd1234.js', hpc.to_url('/path/to/file.min.js'))
self.assertEqual('/path/to/file.abcd1234.js', hpc.to_url('/path/to/file.js'))
self.assertEqual('/path/to/file', hpc.to_url('/path/to/file'))
self.assertEqual('', hpc.to_url(''))