diff --git a/pillar/__init__.py b/pillar/__init__.py index ba78e0da..0eecdbe0 100644 --- a/pillar/__init__.py +++ b/pillar/__init__.py @@ -101,6 +101,7 @@ class PillarServer(BlinkerCompatibleEve): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.log.info('Creating new instance from %r', self.app_root) + self._config_url_map() self._config_auth_token_hmac_key() self._config_tempdirs() self._config_git() @@ -171,6 +172,19 @@ class PillarServer(BlinkerCompatibleEve): if self.config['DEBUG']: log.info('Pillar starting, debug=%s', self.config['DEBUG']) + def _config_url_map(self): + """Extend Flask url_map with our own converters.""" + import secrets, re + from . import flask_extra + + if not self.config.get('STATIC_FILE_HASH'): + self.log.warning('STATIC_FILE_HASH is empty, generating random one') + f = open('/data/git/blender-cloud/config_local.py', 'a') + h = re.sub(r'[_.~-]', '', secrets.token_urlsafe())[:8] + self.config['STATIC_FILE_HASH'] = h + + self.url_map.converters['hashed_path'] = flask_extra.HashedPathConverter + def _config_auth_token_hmac_key(self): """Load AUTH_TOKEN_HMAC_KEY, falling back to SECRET_KEY.""" @@ -529,7 +543,7 @@ class PillarServer(BlinkerCompatibleEve): from pillar.web.staticfile import PillarStaticFile view_func = PillarStaticFile.as_view(endpoint_name, static_folder=static_folder) - self.add_url_rule('%s/' % url_prefix, view_func=view_func) + self.add_url_rule(f'{url_prefix}/', view_func=view_func) def process_extensions(self): """This is about Eve extensions, not Pillar extensions.""" diff --git a/pillar/config.py b/pillar/config.py index 6489b14b..9c2f49a1 100644 --- a/pillar/config.py +++ b/pillar/config.py @@ -247,3 +247,11 @@ SMTP_TIMEOUT = 30 # timeout in seconds, https://docs.python.org/3/library/smtpl MAIL_RETRY = 180 # in seconds, delay until trying to send an email again. MAIL_DEFAULT_FROM_NAME = 'Blender Cloud' MAIL_DEFAULT_FROM_ADDR = 'cloudsupport@localhost' + +SEND_FILE_MAX_AGE_DEFAULT = 3600 * 24 * 365 # seconds + +# MUST be 8 characters long, see pillar.flask_extra.HashedPathConverter +# Intended to be changed for every deploy. If it is empty, a random hash will +# be used. Note that this causes extra traffic, since every time the process +# restarts the URLs will be different. +STATIC_FILE_HASH = '' diff --git a/pillar/flask_extra.py b/pillar/flask_extra.py index d2eda1b9..eb091638 100644 --- a/pillar/flask_extra.py +++ b/pillar/flask_extra.py @@ -1,5 +1,34 @@ +import re import functools + import flask +import werkzeug.routing + + +class HashedPathConverter(werkzeug.routing.PathConverter): + """Allows for files `xxx.yyy.js` to be served as `xxx.yyy.abc123.js`. + + The hash code is placed before the last extension. + """ + weight = 300 + # Hash length is hard-coded to 8 characters for now. + hash_re = re.compile(r'\.([a-zA-Z0-9]{8})(?=\.[^.]+$)') + + @functools.lru_cache(maxsize=1024) + def to_python(self, from_url: str) -> str: + return self.hash_re.sub('', from_url) + + @functools.lru_cache(maxsize=1024) + def to_url(self, filepath: str) -> str: + try: + dotidx = filepath.rindex('.') + except ValueError: + # Happens when there is no dot. Very unlikely. + return filepath + + current_hash = flask.current_app.config['STATIC_FILE_HASH'] + before, after = filepath[:dotidx], filepath[dotidx:] + return f'{before}.{current_hash}{after}' def add_response_headers(headers: dict): diff --git a/pillar/tests/config_testing.py b/pillar/tests/config_testing.py index 34d7c017..2954683e 100644 --- a/pillar/tests/config_testing.py +++ b/pillar/tests/config_testing.py @@ -42,3 +42,6 @@ ELASTIC_INDICES = { 'NODE': 'test_nodes', 'USER': 'test_users', } + +# MUST be 8 characters long, see pillar.flask_extra.HashedPathConverter +STATIC_FILE_HASH = 'abcd1234' diff --git a/pillar/web/staticfile.py b/pillar/web/staticfile.py index 118a0da3..c0a86c7d 100644 --- a/pillar/web/staticfile.py +++ b/pillar/web/staticfile.py @@ -1,12 +1,21 @@ """Static file handling""" +import logging import flask import flask.views +log = logging.getLogger(__name__) + class PillarStaticFile(flask.views.MethodView): def __init__(self, static_folder): self.static_folder = static_folder def get(self, filename): + log.debug('Request file %s/%s', self.static_folder, filename) return flask.send_from_directory(self.static_folder, filename) + return flask.send_from_directory( + self.static_folder, filename, + conditional=True, + add_etags=True, + ) diff --git a/src/templates/layout.pug b/src/templates/layout.pug index b398faa8..e8f439e0 100644 --- a/src/templates/layout.pug +++ b/src/templates/layout.pug @@ -39,8 +39,8 @@ html(lang="en") loadCSS( "//fonts.googleapis.com/css?family=Roboto:300,400" ); - script(src="{{ url_for('static_pillar', filename='assets/js/markdown.min.js', v=17320171) }}") - script(src="{{ url_for('static_pillar', filename='assets/js/tutti.min.js', v=17320171) }}") + script(src="{{ url_for('static_pillar', filename='assets/js/markdown.min.js') }}") + script(src="{{ url_for('static_pillar', filename='assets/js/tutti.min.js') }}") link(href="{{ url_for('static', filename='assets/img/favicon.png') }}", rel="shortcut icon") link(href="{{ url_for('static', filename='assets/img/apple-touch-icon-precomposed.png') }}", rel="icon apple-touch-icon-precomposed", sizes="192x192") @@ -50,12 +50,12 @@ html(lang="en") | {% block head %}{% endblock %} | {% block css %} - link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css', v=17320171) }}", rel="stylesheet") - link(href="{{ url_for('static_pillar', filename='assets/css/base.css', v=17320171) }}", rel="stylesheet") + link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css') }}", rel="stylesheet") + link(href="{{ url_for('static_pillar', filename='assets/css/base.css') }}", rel="stylesheet") | {% if title == 'blog' %} - link(href="{{ url_for('static_pillar', filename='assets/css/blog.css', v=17320171) }}", rel="stylesheet") + link(href="{{ url_for('static_pillar', filename='assets/css/blog.css') }}", rel="stylesheet") | {% else %} - link(href="{{ url_for('static_pillar', filename='assets/css/main.css', v=17320171) }}", rel="stylesheet") + link(href="{{ url_for('static_pillar', filename='assets/css/main.css') }}", rel="stylesheet") | {% endif %} | {% endblock %} @@ -81,7 +81,7 @@ html(lang="en") | {% endblock footer %} | {% endblock footer_container%} - script(src="{{ url_for('static_pillar', filename='assets/js/vendor/jquery.bootstrap-3.3.7.min.js', v=17320171) }}") + script(src="{{ url_for('static_pillar', filename='assets/js/vendor/jquery.bootstrap-3.3.7.min.js') }}") | {% block footer_scripts_pre %}{% endblock %} diff --git a/src/templates/nodes/custom/blog/index.pug b/src/templates/nodes/custom/blog/index.pug index 71c8dccc..6d502a1b 100644 --- a/src/templates/nodes/custom/blog/index.pug +++ b/src/templates/nodes/custom/blog/index.pug @@ -5,7 +5,7 @@ | {% block css %} | {{ super() }} -link(href="{{ url_for('static_pillar', filename='assets/css/blog.css', v=17320171) }}", rel="stylesheet") +link(href="{{ url_for('static_pillar', filename='assets/css/blog.css') }}", rel="stylesheet") | {% endblock %} | {% block project_context %} diff --git a/src/templates/nodes/custom/post/view.pug b/src/templates/nodes/custom/post/view.pug index a9634729..28426b8f 100644 --- a/src/templates/nodes/custom/post/view.pug +++ b/src/templates/nodes/custom/post/view.pug @@ -21,7 +21,7 @@ meta(property="og:image", content="{{ node.picture.thumbnail('l', api=api) }}") | {% block css %} | {{ super() }} -link(href="{{ url_for('static_pillar', filename='assets/css/blog.css', v=17320171) }}", rel="stylesheet") +link(href="{{ url_for('static_pillar', filename='assets/css/blog.css') }}", rel="stylesheet") | {% endblock %} | {% block project_context %} diff --git a/src/templates/nodes/search.pug b/src/templates/nodes/search.pug index c87ae9d4..90978ef8 100644 --- a/src/templates/nodes/search.pug +++ b/src/templates/nodes/search.pug @@ -2,9 +2,9 @@ | {% block page_title %}Search{% if project %} {{ project.name }}{% endif %}{% endblock %} | {% block head %} -script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js', v=9112017) }}") -script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js', v=9112017) }}") -script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js', v=9112017) }}") +script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js') }}") +script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js') }}") +script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js') }}") | {% endblock %} | {% block og %} diff --git a/src/templates/projects/view.pug b/src/templates/projects/view.pug index 66a12e41..6626636a 100644 --- a/src/templates/projects/view.pug +++ b/src/templates/projects/view.pug @@ -66,20 +66,20 @@ meta(property="og:url", content="{{url_for('projects.view', project_url=project. | {% endblock %} | {% block head %} -link(href="{{ url_for('static_pillar', filename='assets/jstree/themes/default/style.min.css', v=9112017) }}", rel="stylesheet") +link(href="{{ url_for('static_pillar', filename='assets/jstree/themes/default/style.min.css') }}", rel="stylesheet") | {% if node %} link(rel="amphtml", href="{{ url_for('nodes.view', node_id=node._id, _external=True, format='amp') }}") | {% endif %} -script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js', v=9112017) }}") -script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js', v=9112017) }}") -script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js', v=9112017) }}") +script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-6.2.8.min.js') }}") +script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-ga-0.4.2.min.js') }}") +script(src="{{ url_for('static_pillar', filename='assets/js/vendor/videojs-hotkeys-0.2.20.min.js') }}") | {% endblock %} | {% block css %} -link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css', v=9112017) }}", rel="stylesheet") -link(href="{{ url_for('static_pillar', filename='assets/css/base.css', v=9112017) }}", rel="stylesheet") -link(href="{{ url_for('static_pillar', filename='assets/css/project-main.css', v=9112017) }}", rel="stylesheet") +link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css') }}", rel="stylesheet") +link(href="{{ url_for('static_pillar', filename='assets/css/base.css') }}", rel="stylesheet") +link(href="{{ url_for('static_pillar', filename='assets/css/project-main.css') }}", rel="stylesheet") | {% endblock %} | {% block body %} @@ -280,7 +280,7 @@ link(href="{{ url_for('static_pillar', filename='assets/css/project-main.css', v | {% if project.has_method('PUT') %} | {# JS containing the Edit, Add, Featured, and Move functions #} -script(type="text/javascript", src="{{ url_for('static_pillar', filename='assets/js/project-edit.min.js', v=9112017) }}") +script(type="text/javascript", src="{{ url_for('static_pillar', filename='assets/js/project-edit.min.js') }}") | {% endif %} script. diff --git a/src/templates/projects/view_theatre.pug b/src/templates/projects/view_theatre.pug index c022ac8c..376d8c94 100644 --- a/src/templates/projects/view_theatre.pug +++ b/src/templates/projects/view_theatre.pug @@ -29,8 +29,8 @@ li | {% endblock %} | {% block css %} -link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css', v=171020161) }}", rel="stylesheet") -link(href="{{ url_for('static_pillar', filename='assets/css/theatre.css', v=171020161) }}", rel="stylesheet") +link(href="{{ url_for('static_pillar', filename='assets/css/font-pillar.css') }}", rel="stylesheet") +link(href="{{ url_for('static_pillar', filename='assets/css/theatre.css') }}", rel="stylesheet") | {% endblock %} | {% block body %} diff --git a/tests/test_flask_extra.py b/tests/test_flask_extra.py index 9ecdd7b7..016595c8 100644 --- a/tests/test_flask_extra.py +++ b/tests/test_flask_extra.py @@ -2,6 +2,8 @@ import unittest import flask +from pillar.tests import AbstractPillarTest + class FlaskExtraTest(unittest.TestCase): def test_vary_xhr(self): @@ -84,3 +86,25 @@ class EnsureSchemaTest(unittest.TestCase): self.assertEqual('/some/path/only', pillar.flask_extra.ensure_schema('/some/path/only')) self.assertEqual('https://hostname/path', pillar.flask_extra.ensure_schema('//hostname/path')) + + +class HashedPathConverterTest(AbstractPillarTest): + def test_to_python(self): + from pillar.flask_extra import HashedPathConverter + + hpc = HashedPathConverter({}) + self.assertEqual('/path/to/file.min.js', hpc.to_python('/path/to/file.min.abcd1234.js')) + self.assertEqual('/path/to/file.js', hpc.to_python('/path/to/file.abcd1234.js')) + self.assertEqual('/path/to/file', hpc.to_python('/path/to/file')) + self.assertEqual('', hpc.to_python('')) + + def test_to_url(self): + from pillar.flask_extra import HashedPathConverter + + hpc = HashedPathConverter({}) + + with self.app.app_context(): + self.assertEqual('/path/to/file.min.abcd1234.js', hpc.to_url('/path/to/file.min.js')) + self.assertEqual('/path/to/file.abcd1234.js', hpc.to_url('/path/to/file.js')) + self.assertEqual('/path/to/file', hpc.to_url('/path/to/file')) + self.assertEqual('', hpc.to_url(''))