Embed iframe shortcodes into container

Use the @capcheck decorator on any shortcode that should support
this. Currently used by iframe and youtube.

.gitignore

@@ -22,6 +22,8 @@ profile.stats
 *.css.map
 *.js.map
 
+/translations/*/LC_MESSAGES/*.mo
+
 pillar/web/static/assets/css/*.css
 pillar/web/static/assets/js/*.min.js
 pillar/web/static/storage/

README.md

@@ -0,0 +1,78 @@
+Pillar
+======
+
+This is the latest iteration on the Attract project. We are building a unified
+framework called Pillar. Pillar will combine Blender Cloud and Attract. You
+can see Pillar in action on the [Blender Cloud](https://cloud.bender.org).
+
+## Custom fonts
+
+The icons on the website are drawn using a custom font, stored in
+[pillar/web/static/font](pillar/web/static/font).
+This font is generated via [Fontello](http://fontello.com/) by uploading
+[pillar/web/static/font/config.json](pillar/web/static/font/config.json).
+
+Note that we only use the WOFF and WOFF2 formats, and discard the others
+supplied by Fontello.
+
+After replacing the font files & `config.json`, edit the Fontello-supplied
+`font.css` to remove all font formats except `woff` and `woff2`. Then upload
+it to [css2sass](http://css2sass.herokuapp.com/) to convert it to SASS, and
+place it in [src/styles/font-pillar.sass](src/styles/font-pillar.sass).
+
+Don't forget to Gulp!
+
+
+## Installation
+
+Make sure your /data directory exists and is writable by the current user.
+Alternatively, provide a `pillar/config_local.py` that changes the relevant
+settings.
+
+```
+git clone git@git.blender.org:pillar-python-sdk.git ../pillar-python-sdk
+pip install -e ../pillar-python-sdk
+pip install -U -r requirements.txt
+pip install -e .
+```
+
+## HDRi viewer
+
+The HDRi viewer uses [Google VRView](https://github.com/googlevr/vrview). To upgrade,
+get those files:
+
+* [three.min.js](https://raw.githubusercontent.com/googlevr/vrview/master/build/three.min.js)
+* [embed.min.js](https://raw.githubusercontent.com/googlevr/vrview/master/build/embed.min.js)
+* [loading.gif](https://raw.githubusercontent.com/googlevr/vrview/master/images/loading.gif)
+
+and place them in `pillar/web/static/assets/vrview`. Replace `images/loading.gif` in `embed.min.js` with `static/pillar/assets/vrview/loading.gif`.
+
+You may also want to compare their
+[index.html](https://raw.githubusercontent.com/googlevr/vrview/master/index.html) to our
+`src/templates/vrview.pug`.
+
+When on a HDRi page with the viewer embedded, use this JavaScript code to find the current
+yaw: `vrview_window.contentWindow.yaw()`. This can be passed as `default_yaw` parameter to
+the iframe.
+
+## Celery
+
+Pillar requires [Celery](http://www.celeryproject.org/) for background task processing. This in
+turn requires a backend and a broker, for which the default Pillar configuration uses Redis and
+RabbitMQ.
+
+You can run the Celery Worker using `manage.py celery worker`.
+
+Find other Celery operations with the `manage.py celery` command.
+
+## Translations
+
+If the language you want to support doesn't exist, you need to run: `translations init es_AR`.
+
+Every time a new string is marked for translation you need to update the entire catalog: `translations update`
+
+And once more strings are translated, you need to compile the translations: `translations compile`
+
+*To mark strings strings for translations in Python scripts you need to
+wrap them with the `flask_babel.gettext` function.
+For .pug templates wrap them with `_()`.*

backup-db.sh

@@ -1,3 +1,3 @@
 #!/bin/bash -ex
 
-mongodump -h localhost:27018 -d cloud --out dump/$(date +'%Y-%m-%d-%H%M') --excludeCollection tokens
+mongodump -h localhost:27018 -d cloud --out dump/$(date +'%Y-%m-%d-%H%M') --excludeCollection tokens --excludeCollection flamenco_task_logs

deploy.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-echo
-echo "==========================================================================="
-echo "Dummy deploy script for people with a 'git pp' alias to push to production."
-echo "Run deploy script on your server project."
-echo "When done, press [ENTER] to stop this script."
-read dummy

gulpfile.js

@@ -1,26 +1,34 @@
 var argv         = require('minimist')(process.argv.slice(2));
 var autoprefixer = require('gulp-autoprefixer');
+var cache        = require('gulp-cached');
 var chmod        = require('gulp-chmod');
 var concat       = require('gulp-concat');
-var gulp         = require('gulp');
+var git          = require('gulp-git');
 var gulpif       = require('gulp-if');
-var jade         = require('gulp-jade');
+var gulp         = require('gulp');
 var livereload   = require('gulp-livereload');
 var plumber      = require('gulp-plumber');
+var pug          = require('gulp-pug');
 var rename       = require('gulp-rename');
 var sass         = require('gulp-sass');
 var sourcemaps   = require('gulp-sourcemaps');
 var uglify       = require('gulp-uglify');
-var cache        = require('gulp-cached');
 
 var enabled = {
     uglify: argv.production,
     maps: argv.production,
-    failCheck: argv.production,
+    failCheck: !argv.production,
     prettyPug: !argv.production,
-    liveReload: !argv.production
+    cachify: !argv.production,
+    cleanup: argv.production,
 };
 
+var destination = {
+    css: 'pillar/web/static/assets/css',
+    pug: 'pillar/web/templates',
+    js: 'pillar/web/static/assets/js',
+}
+
 
 /* CSS */
 gulp.task('styles', function() {
@@ -32,21 +40,21 @@ gulp.task('styles', function() {
             ))
         .pipe(autoprefixer("last 3 versions"))
         .pipe(gulpif(enabled.maps, sourcemaps.write(".")))
-        .pipe(gulp.dest('pillar/web/static/assets/css'))
-        .pipe(gulpif(enabled.liveReload, livereload()));
+        .pipe(gulp.dest(destination.css))
+        .pipe(gulpif(argv.livereload, livereload()));
 });
 
 
-/* Templates - Jade */
+/* Templates - Pug */
 gulp.task('templates', function() {
-    gulp.src('src/templates/**/*.jade')
+    gulp.src('src/templates/**/*.pug')
         .pipe(gulpif(enabled.failCheck, plumber()))
-        .pipe(cache('templating'))
-        .pipe(jade({
+        .pipe(gulpif(enabled.cachify, cache('templating')))
+        .pipe(pug({
             pretty: enabled.prettyPug
         }))
-        .pipe(gulp.dest('pillar/web/templates/'))
-        .pipe(gulpif(enabled.liveReload, livereload()));
+        .pipe(gulp.dest(destination.pug))
+        .pipe(gulpif(argv.livereload, livereload()));
 });
 
 
@@ -54,14 +62,14 @@ gulp.task('templates', function() {
 gulp.task('scripts', function() {
     gulp.src('src/scripts/*.js')
         .pipe(gulpif(enabled.failCheck, plumber()))
-        .pipe(cache('scripting'))
+        .pipe(gulpif(enabled.cachify, cache('scripting')))
         .pipe(gulpif(enabled.maps, sourcemaps.init()))
         .pipe(gulpif(enabled.uglify, uglify()))
         .pipe(rename({suffix: '.min'}))
         .pipe(gulpif(enabled.maps, sourcemaps.write(".")))
         .pipe(chmod(644))
-        .pipe(gulp.dest('pillar/web/static/assets/js/'))
-        .pipe(gulpif(enabled.liveReload, livereload()));
+        .pipe(gulp.dest(destination.js))
+        .pipe(gulpif(argv.livereload, livereload()));
 });
 
 
@@ -75,8 +83,8 @@ gulp.task('scripts_concat_tutti', function() {
         .pipe(gulpif(enabled.uglify, uglify()))
         .pipe(gulpif(enabled.maps, sourcemaps.write(".")))
         .pipe(chmod(644))
-        .pipe(gulp.dest('pillar/web/static/assets/js/'))
-        .pipe(gulpif(enabled.liveReload, livereload()));
+        .pipe(gulp.dest(destination.js))
+        .pipe(gulpif(argv.livereload, livereload()));
 });
 
 gulp.task('scripts_concat_markdown', function() {
@@ -87,8 +95,8 @@ gulp.task('scripts_concat_markdown', function() {
         .pipe(gulpif(enabled.uglify, uglify()))
         .pipe(gulpif(enabled.maps, sourcemaps.write(".")))
         .pipe(chmod(644))
-        .pipe(gulp.dest('pillar/web/static/assets/js/'))
-        .pipe(gulpif(enabled.liveReload, livereload()));
+        .pipe(gulp.dest(destination.js))
+        .pipe(gulpif(argv.livereload, livereload()));
 });
 
 
@@ -100,12 +108,33 @@ gulp.task('watch',function() {
     }
 
     gulp.watch('src/styles/**/*.sass',['styles']);
-    gulp.watch('src/templates/**/*.jade',['templates']);
+    gulp.watch('src/templates/**/*.pug',['templates']);
     gulp.watch('src/scripts/*.js',['scripts']);
     gulp.watch('src/scripts/tutti/**/*.js',['scripts_concat_tutti']);
     gulp.watch('src/scripts/markdown/**/*.js',['scripts_concat_markdown']);
 });
 
+// Erases all generated files in output directories.
+gulp.task('cleanup', function() {
+    var paths = [];
+    for (attr in destination) {
+        paths.push(destination[attr]);
+    }
+
+    git.clean({ args: '-f -X ' + paths.join(' ') }, function (err) {
+        if(err) throw err;
+    });
+
+});
+
 
 // Run 'gulp' to build everything at once
-gulp.task('default', ['styles', 'templates', 'scripts', 'scripts_concat_tutti', 'scripts_concat_markdown']);
+var tasks = [];
+if (enabled.cleanup) tasks.push('cleanup');
+gulp.task('default', tasks.concat([
+    'styles',
+    'templates',
+    'scripts',
+    'scripts_concat_tutti',
+    'scripts_concat_markdown',
+]));

package.json

@@ -1,11 +1,11 @@
 {
 	"name": "pillar",
+	"license": "GPL-2.0+",
+	"author": "Blender Institute",
 	"repository": {
 		"type": "git",
 		"url": "https://github.com/armadillica/pillar.git"
 	},
-  "author": "Blender Institute",
-  "license": "GPL",
 	"devDependencies": {
 		"gulp": "~3.9.1",
 		"gulp-autoprefixer": "~2.3.1",
@@ -13,9 +13,10 @@
 		"gulp-chmod": "~1.3.0",
 		"gulp-concat": "~2.6.0",
 		"gulp-if": "^2.0.1",
-    "gulp-jade": "~1.1.0",
+		"gulp-git": "~2.4.2",
 		"gulp-livereload": "~3.8.1",
 		"gulp-plumber": "~1.1.0",
+		"gulp-pug": "~3.2.0",
 		"gulp-rename": "~1.2.2",
 		"gulp-sass": "~2.3.1",
 		"gulp-sourcemaps": "~1.6.0",

pillar/__init__.py

@@ -1,20 +1,37 @@
 """Pillar server."""
 
 import collections
+import contextlib
 import copy
 import json
 import logging
 import logging.config
 import subprocess
 import tempfile
+import typing
 import os
 import os.path
+import pathlib
 
 import jinja2
 from eve import Eve
 import flask
-from flask import render_template, request
+from flask import g, render_template, request
+from flask_babel import Babel, gettext as _
 from flask.templating import TemplateNotFound
+import pymongo.database
+from werkzeug.local import LocalProxy
+
+
+# Declare pillar.current_app before importing other Pillar modules.
+def _get_current_app():
+    """Returns the current application."""
+
+    return flask.current_app
+
+
+current_app: 'PillarServer' = LocalProxy(_get_current_app)
+"""the current app, annotated as PillarServer"""
 
 from pillar.api import custom_field_validation
 from pillar.api.utils import authentication
@@ -23,6 +40,8 @@ import pillar.web.jinja
 from . import api
 from . import web
 from . import auth
+from . import sentry_extra
+import pillar.api.organizations
 
 empty_settings = {
     # Use a random URL prefix when booting Eve, to ensure that any
@@ -33,15 +52,49 @@ empty_settings = {
 }
 
 
-class PillarServer(Eve):
+class ConfigurationMissingError(SystemExit):
+    """Raised when a vital configuration key is missing.
+
+    Causes Python to exit.
+    """
+
+
+class BlinkerCompatibleEve(Eve):
+    """Workaround for https://github.com/pyeve/eve/issues/1087"""
+
+    def __getattr__(self, name):
+        if name in {"im_self", "im_func"}:
+            raise AttributeError("type object '%s' has no attribute '%s'" %
+                                 (self.__class__.__name__, name))
+        return super().__getattr__(name)
+
+
+class PillarServer(BlinkerCompatibleEve):
     def __init__(self, app_root, **kwargs):
+        from .extension import PillarExtension
+        from celery import Celery
+        from flask_wtf.csrf import CSRFProtect
+
         kwargs.setdefault('validator', custom_field_validation.ValidateCustomFields)
         super(PillarServer, self).__init__(settings=empty_settings, **kwargs)
 
         # mapping from extension name to extension object.
-        self.pillar_extensions = collections.OrderedDict()
+        map_type = typing.MutableMapping[str, PillarExtension]
+        self.pillar_extensions: map_type = collections.OrderedDict()
         self.pillar_extensions_template_paths = []  # list of paths
 
+        # The default roles Pillar uses. Will probably all move to extensions at some point.
+        self._user_roles: typing.Set[str] = {
+            'demo', 'admin', 'subscriber', 'homeproject',
+            'protected', 'org-subscriber', 'video-encoder',
+            'service', 'badger', 'svner',
+        }
+        self._user_roles_indexable: typing.Set[str] = {'demo', 'admin', 'subscriber'}
+
+        # Mapping from role name to capabilities given to that role.
+        self._user_caps: typing.MutableMapping[str, typing.FrozenSet[str]] = \
+            collections.defaultdict(frozenset)
+
         self.app_root = os.path.abspath(app_root)
         self._load_flask_config()
         self._config_logging()
@@ -49,9 +102,13 @@ class PillarServer(Eve):
         self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
         self.log.info('Creating new instance from %r', self.app_root)
 
+        self._config_url_map()
+        self._config_auth_token_hmac_key()
         self._config_tempdirs()
         self._config_git()
-        self._config_bugsnag()
+
+        self.sentry: typing.Optional[sentry_extra.PillarSentry] = None
+        self._config_sentry()
         self._config_google_cloud_storage()
 
         self.algolia_index_users = None
@@ -69,15 +126,37 @@ class PillarServer(Eve):
                                          'api', 'eve_settings.py')
         # self.settings = self.config['EVE_SETTINGS_PATH']
         self.load_config()
+        self._validate_config()
 
         # Configure authentication
         self.login_manager = auth.config_login_manager(self)
-        self.oauth_blender_id = auth.config_oauth_login(self)
 
         self._config_caching()
 
+        self._config_translations()
+
+        # Celery itself is configured after all extensions have loaded.
+        self.celery: Celery = None
+
+        self.org_manager = pillar.api.organizations.OrgManager()
+
         self.before_first_request(self.setup_db_indices)
 
+        # Make CSRF protection available to the application. By default it is
+        # disabled on all endpoints. More info at WTF_CSRF_CHECK_DEFAULT in config.py
+        self.csrf = CSRFProtect(self)
+
+    def _validate_config(self):
+        if not self.config.get('SECRET_KEY'):
+            raise ConfigurationMissingError('SECRET_KEY configuration key is missing')
+
+        server_name = self.config.get('SERVER_NAME')
+        if not server_name:
+            raise ConfigurationMissingError('SERVER_NAME configuration key is missing, should be a '
+                                            'FQDN with TLD')
+        if server_name != 'localhost' and '.' not in server_name:
+            raise ConfigurationMissingError('SERVER_NAME should contain a FQDN with TLD')
+
     def _load_flask_config(self):
         # Load configuration from different sources, to make it easy to override
         # settings with secrets, as well as for development & testing.
@@ -98,6 +177,30 @@ class PillarServer(Eve):
         if self.config['DEBUG']:
             log.info('Pillar starting, debug=%s', self.config['DEBUG'])
 
+    def _config_url_map(self):
+        """Extend Flask url_map with our own converters."""
+        import secrets, re
+        from . import flask_extra
+
+        if not self.config.get('STATIC_FILE_HASH'):
+            self.log.warning('STATIC_FILE_HASH is empty, generating random one')
+            h = re.sub(r'[_.~-]', '', secrets.token_urlsafe())[:8]
+            self.config['STATIC_FILE_HASH'] = h
+
+        self.url_map.converters['hashed_path'] = flask_extra.HashedPathConverter
+
+    def _config_auth_token_hmac_key(self):
+        """Load AUTH_TOKEN_HMAC_KEY, falling back to SECRET_KEY."""
+
+        hmac_key = self.config.get('AUTH_TOKEN_HMAC_KEY')
+        if not hmac_key:
+            self.log.warning('AUTH_TOKEN_HMAC_KEY not set, falling back to SECRET_KEY')
+            hmac_key = self.config['AUTH_TOKEN_HMAC_KEY'] = self.config['SECRET_KEY']
+
+        if isinstance(hmac_key, str):
+            self.log.warning('Converting AUTH_TOKEN_HMAC_KEY to bytes')
+            self.config['AUTH_TOKEN_HMAC_KEY'] = hmac_key.encode('utf8')
+
     def _config_tempdirs(self):
         storage_dir = self.config['STORAGE_DIR']
         if not os.path.exists(storage_dir):
@@ -123,25 +226,18 @@ class PillarServer(Eve):
             self.config['GIT_REVISION'] = 'unknown'
         self.log.info('Git revision %r', self.config['GIT_REVISION'])
 
-    def _config_bugsnag(self):
-        # Configure Bugsnag
-        if self.config.get('TESTING') or not self.config.get('BUGSNAG_API_KEY'):
-            self.log.info('Bugsnag NOT configured.')
+    def _config_sentry(self):
+        # TODO(Sybren): keep Sentry unconfigured when running CLI commands.
+        sentry_dsn = self.config.get('SENTRY_CONFIG', {}).get('dsn')
+        if self.config.get('TESTING') or sentry_dsn in {'', '-set-in-config-local-'}:
+            self.log.warning('Sentry NOT configured.')
+            self.sentry = None
             return
 
-        import bugsnag
-        from bugsnag.flask import handle_exceptions
-        from bugsnag.handlers import BugsnagHandler
-
-        bugsnag.configure(
-            api_key=self.config['BUGSNAG_API_KEY'],
-            project_root="/data/git/pillar/pillar",
-        )
-        handle_exceptions(self)
-
-        bs_handler = BugsnagHandler()
-        bs_handler.setLevel(logging.ERROR)
-        self.log.addHandler(bs_handler)
+        self.sentry = sentry_extra.PillarSentry(
+            self, logging=True, level=logging.WARNING,
+            logging_exclusions=('werkzeug',))
+        self.log.debug('Sentry setup complete')
 
     def _config_google_cloud_storage(self):
         # Google Cloud project
@@ -149,17 +245,17 @@ class PillarServer(Eve):
             os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = \
                 self.config['GCLOUD_APP_CREDENTIALS']
         except KeyError:
-            raise SystemExit('GCLOUD_APP_CREDENTIALS configuration is missing')
+            raise ConfigurationMissingError('GCLOUD_APP_CREDENTIALS configuration is missing')
 
         # Storage backend (GCS)
         try:
             os.environ['GCLOUD_PROJECT'] = self.config['GCLOUD_PROJECT']
         except KeyError:
-            raise SystemExit('GCLOUD_PROJECT configuration value is missing')
+            raise ConfigurationMissingError('GCLOUD_PROJECT configuration value is missing')
 
     def _config_algolia(self):
         # Algolia search
-        if self.config['SEARCH_BACKEND'] != 'algolia':
+        if 'algolia' not in self.config['SEARCH_BACKENDS']:
             return
 
         from algoliasearch import algoliasearch
@@ -173,8 +269,13 @@ class PillarServer(Eve):
     def _config_encoding_backend(self):
         # Encoding backend
         if self.config['ENCODING_BACKEND'] != 'zencoder':
+            self.log.warning('Encoding backend %r not supported, no video encoding possible!',
+                             self.config['ENCODING_BACKEND'])
             return
 
+        self.log.info('Setting up video encoding backend %r',
+                      self.config['ENCODING_BACKEND'])
+
         from zencoder import Zencoder
         self.encoding_service_client = Zencoder(self.config['ZENCODER_API_KEY'])
 
@@ -182,6 +283,74 @@ class PillarServer(Eve):
         from flask_cache import Cache
         self.cache = Cache(self)
 
+    def set_languages(self, translations_folder: pathlib.Path):
+        """Set the supported languages based on translations folders
+
+        English is an optional language included by default, since we will
+        never have a translations folder for it.
+        """
+        self.default_locale = self.config['DEFAULT_LOCALE']
+        self.config['BABEL_DEFAULT_LOCALE'] = self.default_locale
+
+        # Determine available languages.
+        languages = list()
+
+        # The available languages will be determined based on available
+        # translations in the //translations/ folder. The exception is (American) English
+        # since all the text is originally in English already.
+        # That said, if rare occasions we may want to never show
+        # the site in English.
+
+        if self.config['SUPPORT_ENGLISH']:
+            languages.append('en_US')
+
+        base_path = pathlib.Path(self.app_root) / 'translations'
+
+        if not base_path.is_dir():
+            self.log.debug('Project has no translations folder: %s', base_path)
+        else:
+            languages.extend(i.name for i in base_path.iterdir() if i.is_dir())
+
+        # Use set for quicker lookup
+        self.languages = set(languages)
+
+        self.log.info('Available languages: %s' % ', '.join(self.languages))
+
+    def _config_translations(self):
+        """
+        Initialize translations variable.
+
+        The BABEL_TRANSLATION_DIRECTORIES has the folder for the compiled
+        translations files. It uses ; separation for the extension folders.
+        """
+        self.log.info('Configure translations')
+        translations_path = pathlib.Path(__file__).parents[1].joinpath('translations')
+
+        self.config['BABEL_TRANSLATION_DIRECTORIES'] = str(translations_path)
+        babel = Babel(self)
+
+        self.set_languages(translations_path)
+
+        # get_locale() is registered as a callback for locale selection.
+        # That prevents the function from being garbage collected.
+        @babel.localeselector
+        def get_locale() -> str:
+            """
+            Callback runs before each request to give us a chance to choose the
+            language to use when producing its response.
+
+            We set g.locale to be able to access it from the template pages.
+            We still need to return it explicitly, since this function is
+            called as part of the babel translation framework.
+
+            We are using the 'Accept-Languages' header to match the available
+            translations with the user supported languages.
+            """
+            locale = request.accept_languages.best_match(
+                self.languages, self.default_locale)
+            g.locale = locale
+            return locale
+
     def load_extension(self, pillar_extension, url_prefix):
         from .extension import PillarExtension
 
@@ -200,12 +369,15 @@ class PillarServer(Eve):
         self.pillar_extensions[pillar_extension.name] = pillar_extension
 
         # Load extension Flask configuration
-        for key, value in pillar_extension.flask_config():
+        for key, value in pillar_extension.flask_config().items():
             self.config.setdefault(key, value)
 
         # Load extension blueprint(s)
         for blueprint in pillar_extension.blueprints():
             if blueprint.url_prefix:
+                if not url_prefix:
+                    # If we registered the extension with url_prefix=None
+                    url_prefix = ''
                 blueprint_prefix = url_prefix + blueprint.url_prefix
             else:
                 blueprint_prefix = url_prefix
@@ -238,6 +410,19 @@ class PillarServer(Eve):
 
             self.config['DOMAIN'].update(eve_settings['DOMAIN'])
 
+        # Configure the extension translations
+        trpath = pillar_extension.translations_path
+        if not trpath:
+            self.log.debug('Extension %s does not have a translations folder',
+                           pillar_extension.name)
+            return
+
+        self.log.info('Extension %s: adding translations path %s',
+                      pillar_extension.name, trpath)
+
+        # Babel requires semi-colon string separation
+        self.config['BABEL_TRANSLATION_DIRECTORIES'] += ';' + str(trpath)
+
     def _config_jinja_env(self):
         # Start with the extensions...
         paths_list = [
@@ -260,7 +445,15 @@ class PillarServer(Eve):
         custom_jinja_loader = jinja2.ChoiceLoader(paths_list)
         self.jinja_loader = custom_jinja_loader
 
-        pillar.web.jinja.setup_jinja_env(self.jinja_env)
+        pillar.web.jinja.setup_jinja_env(self.jinja_env, self.config)
+
+        # Register context processors from extensions
+        for ext in self.pillar_extensions.values():
+            if not ext.has_context_processor:
+                continue
+
+            self.log.debug('Registering context processor for %s', ext.name)
+            self.context_processor(ext.context_processor)
 
     def _config_static_dirs(self):
         # Setup static folder for the instanced app
@@ -279,13 +472,87 @@ class PillarServer(Eve):
                                                'static_%s' % name,
                                                ext.static_path)
 
+    def _config_celery(self):
+        from celery import Celery
+
+        self.log.info('Configuring Celery')
+
+        # Pillar-defined Celery task modules:
+        celery_task_modules = [
+            'pillar.celery.tasks',
+            'pillar.celery.search_index_tasks',
+            'pillar.celery.file_link_tasks',
+            'pillar.celery.email_tasks',
+        ]
+
+        # Allow Pillar extensions from defining their own Celery tasks.
+        for extension in self.pillar_extensions.values():
+            celery_task_modules.extend(extension.celery_task_modules)
+
+        self.celery = Celery(
+            'pillar.celery',
+            backend=self.config['CELERY_BACKEND'],
+            broker=self.config['CELERY_BROKER'],
+            include=celery_task_modules,
+            task_track_started=True,
+            result_expires=3600,
+        )
+
+        # This configures the Celery task scheduler in such a way that we don't
+        # have to import the pillar.celery.XXX modules. Remember to run
+        # 'manage.py celery beat' too, otherwise those will never run.
+        beat_schedule = self.config.get('CELERY_BEAT_SCHEDULE')
+        if beat_schedule:
+            self.celery.conf.beat_schedule = beat_schedule
+
+        self.log.info('Pinging Celery workers')
+        self.log.info('Response: %s', self.celery.control.ping())
+
+    def _config_user_roles(self):
+        """Gathers all user roles from extensions.
+
+        The union of all user roles can be obtained from self.user_roles.
+        """
+
+        for extension in self.pillar_extensions.values():
+            indexed_but_not_defined = extension.user_roles_indexable - extension.user_roles
+            if indexed_but_not_defined:
+                raise ValueError('Extension %s has roles %s indexable but not in user_roles',
+                                 extension.name, indexed_but_not_defined)
+
+            self._user_roles.update(extension.user_roles)
+            self._user_roles_indexable.update(extension.user_roles_indexable)
+
+        self.log.info('Loaded %i user roles from extensions, %i of which are indexable',
+                      len(self._user_roles), len(self._user_roles_indexable))
+
+    def _config_user_caps(self):
+        """Merges all capability settings from app config and extensions."""
+
+        app_caps = collections.defaultdict(frozenset, **self.config['USER_CAPABILITIES'])
+
+        for extension in self.pillar_extensions.values():
+            ext_caps = extension.user_caps
+
+            for role, caps in ext_caps.items():
+                union_caps = frozenset(app_caps[role] | caps)
+                app_caps[role] = union_caps
+
+        self._user_caps = app_caps
+
+        if self.log.isEnabledFor(logging.DEBUG):
+            import pprint
+            self.log.debug('Configured user capabilities: %s', pprint.pformat(self._user_caps))
+
     def register_static_file_endpoint(self, url_prefix, endpoint_name, static_folder):
-        from pillar.web.static import PillarStaticFile
+        from pillar.web.staticfile import PillarStaticFile
 
         view_func = PillarStaticFile.as_view(endpoint_name, static_folder=static_folder)
-        self.add_url_rule('%s/<path:filename>' % url_prefix, view_func=view_func)
+        self.add_url_rule(f'{url_prefix}/<hashed_path:filename>', view_func=view_func)
 
     def process_extensions(self):
+        """This is about Eve extensions, not Pillar extensions."""
+
         # Re-initialise Eve after we allowed Pillar submodules to be loaded.
         # EVIL STARTS HERE. It just copies part of the Eve.__init__() method.
         self.set_defaults()
@@ -360,12 +627,12 @@ class PillarServer(Eve):
                 if node_type:
                     node_type = node_type.replace('_', ' ').title()
                     if doc_name:
-                        description = u'%s "%s" was deleted.' % (node_type, doc_name)
+                        description = '%s "%s" was deleted.' % (node_type, doc_name)
                     else:
-                        description = u'This %s was deleted.' % (node_type, )
+                        description = 'This %s was deleted.' % (node_type,)
                 else:
                     if doc_name:
-                        description = u'"%s" was deleted.' % doc_name
+                        description = '"%s" was deleted.' % doc_name
                     else:
                         description = None
 
@@ -384,7 +651,7 @@ class PillarServer(Eve):
         self.log.info('Forwarding ResourceInvalid exception to client: %s', error, exc_info=True)
 
         # Raising a Werkzeug 422 exception doens't work, as Flask turns it into a 500.
-        return 'The submitted data could not be validated.', 422
+        return _('The submitted data could not be validated.'), 422
 
     def handle_sdk_method_not_allowed(self, error):
         """Forwards 405 Method Not Allowed to the client.
@@ -437,16 +704,21 @@ class PillarServer(Eve):
     def finish_startup(self):
         self.log.info('Using MongoDB database %r', self.config['MONGO_DBNAME'])
 
+        self._config_celery()
+
         api.setup_app(self)
         web.setup_app(self)
+
         authentication.setup_app(self)
 
-        for ext in self.pillar_extensions.itervalues():
+        for ext in self.pillar_extensions.values():
             self.log.info('Setting up extension %s', ext.name)
             ext.setup_app(self)
 
         self._config_jinja_env()
         self._config_static_dirs()
+        self._config_user_roles()
+        self._config_user_caps()
 
         # Only enable this when debugging.
         # self._list_routes()
@@ -468,6 +740,7 @@ class PillarServer(Eve):
         coll = db['tokens']
         coll.create_index([('user', pymongo.ASCENDING)])
         coll.create_index([('token', pymongo.ASCENDING)])
+        coll.create_index([('token_hashed', pymongo.ASCENDING)])
 
         coll = db['notifications']
         coll.create_index([('user', pymongo.ASCENDING)])
@@ -483,6 +756,20 @@ class PillarServer(Eve):
         coll.create_index([('parent', pymongo.ASCENDING)])
         coll.create_index([('short_code', pymongo.ASCENDING)],
                           sparse=True, unique=True)
+        # Used for latest assets & comments
+        coll.create_index([('properties.status', pymongo.ASCENDING),
+                           ('node_type', pymongo.ASCENDING),
+                           ('_created', pymongo.DESCENDING)])
+
+        coll = db['projects']
+        # This index is used for statistics, and for fetching public projects.
+        coll.create_index([('is_private', pymongo.ASCENDING)])
+        coll.create_index([('category', pymongo.ASCENDING)])
+
+        coll = db['organizations']
+        coll.create_index([('ip_ranges.start', pymongo.ASCENDING)])
+        coll.create_index([('ip_ranges.end', pymongo.ASCENDING)])
+        self.log.debug('Created database indices')
 
     def register_api_blueprint(self, blueprint, url_prefix):
         # TODO: use Eve config variable instead of hard-coded '/api'
@@ -494,32 +781,49 @@ class PillarServer(Eve):
 
         return 'basic ' + base64.b64encode('%s:%s' % (username, subclient_id))
 
-    def post_internal(self, resource, payl=None, skip_validation=False):
+    def post_internal(self, resource: str, payl=None, skip_validation=False):
         """Workaround for Eve issue https://github.com/nicolaiarocci/eve/issues/810"""
         from eve.methods.post import post_internal
 
-        with self.test_request_context(method='POST', path='%s/%s' % (self.api_prefix, resource)):
-            return post_internal(resource, payl=payl, skip_validation=skip_validation)
+        url = self.config['URLS'][resource]
+        path = '%s/%s' % (self.api_prefix, url)
+        with self.__fake_request_url_rule('POST', path):
+            return post_internal(resource, payl=payl, skip_validation=skip_validation)[:4]
 
-    def put_internal(self, resource, payload=None, concurrency_check=False,
+    def put_internal(self, resource: str, payload=None, concurrency_check=False,
                      skip_validation=False, **lookup):
         """Workaround for Eve issue https://github.com/nicolaiarocci/eve/issues/810"""
         from eve.methods.put import put_internal
 
-        path = '%s/%s/%s' % (self.api_prefix, resource, lookup['_id'])
-        with self.test_request_context(method='PUT', path=path):
+        url = self.config['URLS'][resource]
+        path = '%s/%s/%s' % (self.api_prefix, url, lookup['_id'])
+        with self.__fake_request_url_rule('PUT', path):
             return put_internal(resource, payload=payload, concurrency_check=concurrency_check,
-                                skip_validation=skip_validation, **lookup)
+                                skip_validation=skip_validation, **lookup)[:4]
 
-    def patch_internal(self, resource, payload=None, concurrency_check=False,
+    def patch_internal(self, resource: str, payload=None, concurrency_check=False,
                        skip_validation=False, **lookup):
         """Workaround for Eve issue https://github.com/nicolaiarocci/eve/issues/810"""
         from eve.methods.patch import patch_internal
 
-        path = '%s/%s/%s' % (self.api_prefix, resource, lookup['_id'])
-        with self.test_request_context(method='PATCH', path=path):
+        url = self.config['URLS'][resource]
+        path = '%s/%s/%s' % (self.api_prefix, url, lookup['_id'])
+        with self.__fake_request_url_rule('PATCH', path):
             return patch_internal(resource, payload=payload, concurrency_check=concurrency_check,
-                                  skip_validation=skip_validation, **lookup)
+                                  skip_validation=skip_validation, **lookup)[:4]
+
+    def delete_internal(self, resource: str, concurrency_check=False,
+                        suppress_callbacks=False, **lookup):
+        """Workaround for Eve issue https://github.com/nicolaiarocci/eve/issues/810"""
+        from eve.methods.delete import deleteitem_internal
+
+        url = self.config['URLS'][resource]
+        path = '%s/%s/%s' % (self.api_prefix, url, lookup['_id'])
+        with self.__fake_request_url_rule('DELETE', path):
+            return deleteitem_internal(resource,
+                                       concurrency_check=concurrency_check,
+                                       suppress_callbacks=suppress_callbacks,
+                                       **lookup)[:4]
 
     def _list_routes(self):
         from pprint import pprint
@@ -537,18 +841,22 @@ class PillarServer(Eve):
                 # and rules that require parameters
                 if "GET" in rule.methods and has_no_empty_params(rule):
                     url = url_for(rule.endpoint, **(rule.defaults or {}))
-                    links.append((url, rule.endpoint))
+                    links.append((url, rule.endpoint, rule.methods))
+                if "PATCH" in rule.methods:
+                    args = {arg: arg for arg in rule.arguments}
+                    url = url_for(rule.endpoint, **args)
+                    links.append((url, rule.endpoint, rule.methods))
 
-        links.sort(key=lambda t: len(t[0]) + 100 * ('/api/' in t[0]))
+        links.sort(key=lambda t: (('/api/' in t[0]), len(t[0])))
 
-        pprint(links)
+        pprint(links, width=300)
 
-    def db(self):
-        """Returns the MongoDB database.
-
-        :rtype: flask_pymongo.PyMongo
-        """
+    def db(self, collection_name: str = None) \
+            -> typing.Union[pymongo.collection.Collection, pymongo.database.Database]:
+        """Returns the MongoDB database, or the collection (if given)"""
 
+        if collection_name:
+            return self.data.driver.db[collection_name]
         return self.data.driver.db
 
     def extension_sidebar_links(self, project):
@@ -562,3 +870,52 @@ class PillarServer(Eve):
 
         return jinja2.Markup(''.join(ext.sidebar_links(project)
                                      for ext in self.pillar_extensions.values()))
+
+    @contextlib.contextmanager
+    def __fake_request_url_rule(self, method: str, url_path: str):
+        """Tries to force-set the request URL rule.
+
+        This is required by Eve (since 0.70) to be able to construct a
+        Location HTTP header that points to the resource item.
+
+        See post_internal, put_internal and patch_internal.
+        """
+
+        import werkzeug.exceptions as wz_exceptions
+
+        with self.test_request_context(method=method, path=url_path) as ctx:
+            try:
+                rule, _ = ctx.url_adapter.match(url_path, method=method, return_rule=True)
+            except (wz_exceptions.MethodNotAllowed, wz_exceptions.NotFound):
+                # We're POSTing things that we haven't told Eve are POSTable. Try again using the
+                # GET method.
+                rule, _ = ctx.url_adapter.match(url_path, method='GET', return_rule=True)
+            current_request = request._get_current_object()
+            current_request.url_rule = rule
+
+            yield ctx
+
+    def validator_for_resource(self, resource_name: str) -> custom_field_validation.ValidateCustomFields:
+        schema = self.config['DOMAIN'][resource_name]['schema']
+        validator = self.validator(schema, resource_name)
+        return validator
+
+    @property
+    def user_roles(self) -> typing.FrozenSet[str]:
+        return frozenset(self._user_roles)
+
+    @property
+    def user_roles_indexable(self) -> typing.FrozenSet[str]:
+        return frozenset(self._user_roles_indexable)
+
+    @property
+    def user_caps(self) -> typing.Mapping[str, typing.FrozenSet[str]]:
+        return self._user_caps
+
+    @property
+    def real_app(self) -> 'PillarServer':
+        """The real application object.
+
+        Can be used to obtain the real app object from a LocalProxy.
+        """
+        return self

pillar/api/__init__.py

@@ -1,9 +1,12 @@
 def setup_app(app):
     from . import encoding, blender_id, projects, local_auth, file_storage
     from . import users, nodes, latest, blender_cloud, service, activities
+    from . import organizations
+    from . import search
 
     encoding.setup_app(app, url_prefix='/encoding')
     blender_id.setup_app(app, url_prefix='/blender_id')
+    search.setup_app(app, url_prefix='/newsearch')
     projects.setup_app(app, api_prefix='/p')
     local_auth.setup_app(app, url_prefix='/auth')
     file_storage.setup_app(app, url_prefix='/storage')
@@ -13,3 +16,4 @@ def setup_app(app):
     service.setup_app(app, api_prefix='/service')
     nodes.setup_app(app, url_prefix='/nodes')
     activities.setup_app(app)
+    organizations.setup_app(app)

pillar/api/activities.py

@@ -1,7 +1,8 @@
 import logging
 
-from flask import g, request, current_app
+from flask import request, current_app
 from pillar.api.utils import gravatar
+from pillar.auth import current_user
 
 log = logging.getLogger(__name__)
 
@@ -30,7 +31,7 @@ def notification_parse(notification):
     object_name = ''
     object_id = activity['object']
 
-    if node['parent']['user'] == g.current_user['user_id']:
+    if node['parent']['user'] == current_user.user_id:
         owner = "your {0}".format(node['parent']['node_type'])
     else:
         parent_comment_user = users_collection.find_one(
@@ -52,7 +53,7 @@ def notification_parse(notification):
         action = activity['verb']
 
     lookup = {
-        'user': g.current_user['user_id'],
+        'user': current_user.user_id,
         'context_object_type': 'node',
         'context_object': context_object_id,
     }

pillar/api/blender_cloud/__init__.py

@@ -24,7 +24,8 @@ def blender_cloud_addon_version():
 
 
 def setup_app(app, url_prefix):
-    from . import texture_libs, home_project
+    from . import texture_libs, home_project, subscription
 
     texture_libs.setup_app(app, url_prefix=url_prefix)
     home_project.setup_app(app, url_prefix=url_prefix)
+    subscription.setup_app(app, url_prefix=url_prefix)

pillar/api/blender_cloud/home_project.py

@@ -1,12 +1,11 @@
 import copy
 import logging
 
-import datetime
-from bson import ObjectId, tz_util
+from bson import ObjectId
 from eve.methods.get import get
-from flask import Blueprint, g, current_app, request
+from flask import Blueprint, current_app, request
 from pillar.api import utils
-from pillar.api.utils import authentication, authorization
+from pillar.api.utils import authentication, authorization, utcnow
 from werkzeug import exceptions as wz_exceptions
 
 from pillar.api.projects import utils as proj_utils
@@ -18,7 +17,7 @@ log = logging.getLogger(__name__)
 HOME_PROJECT_USERS = set()
 
 # Users with any of these roles will get full write access to their home project.
-HOME_PROJECT_WRITABLE_USERS = {u'subscriber', u'demo'}
+HOME_PROJECT_WRITABLE_USERS = {'subscriber', 'demo'}
 
 HOME_PROJECT_DESCRIPTION = ('# Your home project\n\n'
                             'This is your home project. It allows synchronisation '
@@ -30,7 +29,7 @@ HOME_PROJECT_SUMMARY = 'This is your home project. Here you can sync your Blende
 #                             'as a pastebin for text, images and other assets, and '
 #                             'allows synchronisation of your Blender settings.')
 # HOME_PROJECT_SUMMARY = 'This is your home project. Pastebin and Blender settings sync in one!'
-SYNC_GROUP_NODE_NAME = u'Blender Sync'
+SYNC_GROUP_NODE_NAME = 'Blender Sync'
 SYNC_GROUP_NODE_DESC = ('The [Blender Cloud Addon](https://cloud.blender.org/services'
                         '#blender-addon) will synchronize your Blender settings here.')
 
@@ -113,7 +112,7 @@ def create_home_project(user_id, write_access):
 
     # Re-validate the authentication token, so that the put_internal call sees the
     # new group created for the project.
-    authentication.validate_token()
+    authentication.validate_token(force=True)
 
     # There are a few things in the on_insert_projects hook we need to adjust.
 
@@ -135,8 +134,8 @@ def create_home_project(user_id, write_access):
     # This allows people to comment on shared images and see comments.
     node_type_comment = assign_permissions(
         node_type_comment,
-        subscriber_methods=[u'GET', u'POST'],
-        world_methods=[u'GET'])
+        subscriber_methods=['GET', 'POST'],
+        world_methods=['GET'])
 
     project['node_types'] = [
         node_type_group,
@@ -201,8 +200,10 @@ def home_project():
     Eve projections are supported, but at least the following fields must be present:
         'permissions', 'category', 'user'
     """
-    user_id = g.current_user['user_id']
-    roles = g.current_user.get('roles', ())
+    from pillar.auth import current_user
+
+    user_id = current_user.user_id
+    roles = current_user.roles
 
     log.debug('Possibly creating home project for user %s with roles %s', user_id, roles)
     if HOME_PROJECT_USERS and not HOME_PROJECT_USERS.intersection(roles):
@@ -215,7 +216,7 @@ def home_project():
         write_access = write_access_with_roles(roles)
         create_home_project(user_id, write_access)
 
-    resp, _, _, status, _ = get('projects', category=u'home', user=user_id)
+    resp, _, _, status, _ = get('projects', category='home', user=user_id)
     if status != 200:
         return utils.jsonify(resp), status
 
@@ -248,8 +249,8 @@ def home_project_permissions(write_access):
     """
 
     if write_access:
-        return [u'GET', u'PUT', u'POST', u'DELETE']
-    return [u'GET']
+        return ['GET', 'PUT', 'POST', 'DELETE']
+    return ['GET']
 
 
 def has_home_project(user_id):
@@ -280,7 +281,7 @@ def is_home_project(project_id, user_id):
 def mark_node_updated(node_id):
     """Uses pymongo to set the node's _updated to "now"."""
 
-    now = datetime.datetime.now(tz=tz_util.utc)
+    now = utcnow()
     nodes_coll = current_app.data.driver.db['nodes']
 
     return nodes_coll.update_one({'_id': node_id},

pillar/api/blender_cloud/subscription.py

@@ -0,0 +1,180 @@
+import logging
+import typing
+
+import blinker
+from flask import Blueprint, Response
+import requests
+from requests.adapters import HTTPAdapter
+
+from pillar import auth, current_app
+from pillar.api import blender_id
+from pillar.api.utils import authorization, jsonify
+from pillar.auth import current_user
+
+log = logging.getLogger(__name__)
+blueprint = Blueprint('blender_cloud.subscription', __name__)
+
+# Mapping from roles on Blender ID to roles here in Pillar.
+# Roles not mentioned here will not be synced from Blender ID.
+ROLES_BID_TO_PILLAR = {
+    'cloud_subscriber': 'subscriber',
+    'cloud_demo': 'demo',
+    'cloud_has_subscription': 'has_subscription',
+}
+
+user_subscription_updated = blinker.NamedSignal(
+    'user_subscription_updated',
+    'The sender is a UserClass instance, kwargs includes "revoke_roles" and "grant_roles".')
+
+
+@blueprint.route('/update-subscription')
+@authorization.require_login()
+def update_subscription() -> typing.Tuple[str, int]:
+    """Updates the subscription status of the current user.
+
+    Returns an empty HTTP response.
+    """
+
+    my_log: logging.Logger = log.getChild('update_subscription')
+    real_current_user = auth.get_current_user()  # multiple accesses, just get unproxied.
+
+    try:
+        bid_user = blender_id.fetch_blenderid_user()
+    except blender_id.LogoutUser:
+        auth.logout_user()
+        return '', 204
+
+    if not bid_user:
+        my_log.warning('Logged in user %s has no BlenderID account! '
+                       'Unable to update subscription status.', real_current_user.user_id)
+        return '', 204
+
+    do_update_subscription(real_current_user, bid_user)
+    return '', 204
+
+
+@blueprint.route('/update-subscription-for/<user_id>', methods=['POST'])
+@authorization.require_login(require_cap='admin')
+def update_subscription_for(user_id: str):
+    """Updates the user based on their info at Blender ID."""
+
+    from urllib.parse import urljoin
+
+    from pillar.api.utils import str2id
+
+    my_log = log.getChild('update_subscription_for')
+
+    bid_session = requests.Session()
+    bid_session.mount('https://', HTTPAdapter(max_retries=5))
+    bid_session.mount('http://', HTTPAdapter(max_retries=5))
+
+    users_coll = current_app.db('users')
+    db_user = users_coll.find_one({'_id': str2id(user_id)})
+    if not db_user:
+        my_log.warning('User %s not found in database', user_id)
+        return Response(f'User {user_id} not found in our database', status=404)
+
+    log.info('Updating user %s from Blender ID on behalf of %s',
+             db_user['email'], current_user.email)
+
+    bid_user_id = blender_id.get_user_blenderid(db_user)
+    if not bid_user_id:
+        my_log.info('User %s has no Blender ID', user_id)
+        return Response('User has no Blender ID', status=404)
+
+    # Get the user info from Blender ID, and handle errors.
+    api_url = current_app.config['BLENDER_ID_USER_INFO_API']
+    api_token = current_app.config['BLENDER_ID_USER_INFO_TOKEN']
+    url = urljoin(api_url, bid_user_id)
+    resp = bid_session.get(url, headers={'Authorization': f'Bearer {api_token}'})
+    if resp.status_code == 404:
+        my_log.info('User %s has a Blender ID %s but Blender ID itself does not find it',
+                    user_id, bid_user_id)
+        return Response(f'User {bid_user_id} does not exist at Blender ID', status=404)
+    if resp.status_code != 200:
+        my_log.info('Error code %s getting user %s from Blender ID (resp = %s)',
+                    resp.status_code, user_id, resp.text)
+        return Response(f'Error code {resp.status_code} from Blender ID', status=resp.status_code)
+
+    # Update the user in our database.
+    local_user = auth.UserClass.construct('', db_user)
+    bid_user = resp.json()
+    do_update_subscription(local_user, bid_user)
+
+    return '', 204
+
+
+def do_update_subscription(local_user: auth.UserClass, bid_user: dict):
+    """Updates the subscription status of the user given the Blender ID user info.
+
+    Uses the badger service to update the user's roles from Blender ID.
+
+    bid_user should be a dict like:
+    {'id': 1234,
+     'full_name': 'मूंगफली मक्खन प्रेमी',
+     'email': 'here@example.com',
+     'roles': {'cloud_demo': True}}
+
+    The 'roles' key can also be an interable of role names instead of a dict.
+    """
+
+    from pillar.api import service
+
+    my_log: logging.Logger = log.getChild('do_update_subscription')
+
+    try:
+        email = bid_user['email']
+    except KeyError:
+        email = '-missing email-'
+
+    # Transform the BID roles from a dict to a set.
+    bidr = bid_user.get('roles', set())
+    if isinstance(bidr, dict):
+        bid_roles = {role
+                     for role, has_role in bid_user.get('roles', {}).items()
+                     if has_role}
+    else:
+        bid_roles = set(bidr)
+
+    # Handle the role changes via the badger service functionality.
+    plr_roles = set(local_user.roles)
+
+    grant_roles = set()
+    revoke_roles = set()
+    for bid_role, plr_role in ROLES_BID_TO_PILLAR.items():
+        if bid_role in bid_roles and plr_role not in plr_roles:
+            grant_roles.add(plr_role)
+            continue
+        if bid_role not in bid_roles and plr_role in plr_roles:
+            revoke_roles.add(plr_role)
+
+    user_id = local_user.user_id
+
+    if grant_roles:
+        if my_log.isEnabledFor(logging.INFO):
+            my_log.info('granting roles to user %s (Blender ID %s): %s',
+                        user_id, email, ', '.join(sorted(grant_roles)))
+        service.do_badger('grant', roles=grant_roles, user_id=user_id)
+
+    if revoke_roles:
+        if my_log.isEnabledFor(logging.INFO):
+            my_log.info('revoking roles to user %s (Blender ID %s): %s',
+                        user_id, email, ', '.join(sorted(revoke_roles)))
+        service.do_badger('revoke', roles=revoke_roles, user_id=user_id)
+
+    # Let the world know this user's subscription was updated.
+    final_roles = (plr_roles - revoke_roles).union(grant_roles)
+    local_user.roles = list(final_roles)
+    local_user.collect_capabilities()
+    user_subscription_updated.send(local_user,
+                                   grant_roles=grant_roles,
+                                   revoke_roles=revoke_roles)
+
+    # Re-index the user in the search database.
+    from pillar.api.users import hooks
+    hooks.push_updated_user_to_search({'_id': user_id}, {})
+
+
+def setup_app(app, url_prefix):
+    log.info('Registering blueprint at %s', url_prefix)
+    app.register_api_blueprint(blueprint, url_prefix=url_prefix)

pillar/api/blender_cloud/texture_libs.py

@@ -3,12 +3,14 @@ import logging
 
 from eve.methods.get import get
 from eve.utils import config as eve_config
-from flask import Blueprint, request, current_app, g
+from flask import Blueprint, request, current_app
+from werkzeug.datastructures import MultiDict
+from werkzeug.exceptions import InternalServerError
+
 from pillar.api import utils
 from pillar.api.utils.authentication import current_user_id
 from pillar.api.utils.authorization import require_login
-from werkzeug.datastructures import MultiDict
-from werkzeug.exceptions import InternalServerError
+from pillar.auth import current_user
 
 FIRST_ADDON_VERSION_WITH_HDRI = (1, 4, 0)
 TL_PROJECTION = utils.dumps({'name': 1, 'url': 1, 'permissions': 1,})
@@ -25,8 +27,8 @@ log = logging.getLogger(__name__)
 
 
 def keep_fetching_texture_libraries(proj_filter):
-    groups = g.current_user['groups']
-    user_id = g.current_user['user_id']
+    groups = current_user.group_ids
+    user_id = current_user.user_id
 
     page = 1
     max_page = float('inf')
@@ -74,7 +76,7 @@ def texture_libraries():
     # of the Blender Cloud Addon. If the addon version is None, we're dealing
     # with a version of the BCA that's so old it doesn't send its version along.
     addon_version = blender_cloud_addon_version()
-    return_hdri = addon_version >= FIRST_ADDON_VERSION_WITH_HDRI
+    return_hdri = addon_version is not None and addon_version >= FIRST_ADDON_VERSION_WITH_HDRI
     log.debug('User %s has Blender Cloud Addon version %s; return_hdri=%s',
               current_user_id(), addon_version, return_hdri)
 

pillar/api/blender_id.py

@@ -4,20 +4,30 @@ Also contains functionality for other parts of Pillar to perform communication
 with Blender ID.
 """
 
+import datetime
 import logging
 
-import datetime
 import requests
 from bson import tz_util
-from flask import Blueprint, request, current_app, jsonify
-from pillar.api.utils import authentication, remove_private_keys
+from rauth import OAuth2Session
+from flask import Blueprint, request, jsonify, session
 from requests.adapters import HTTPAdapter
-from werkzeug import exceptions as wz_exceptions
+
+from pillar import current_app
+from pillar.api.utils import authentication, utcnow
+from pillar.api.utils.authentication import find_user_in_db, upsert_user
 
 blender_id = Blueprint('blender_id', __name__)
 log = logging.getLogger(__name__)
 
 
+class LogoutUser(Exception):
+    """Raised when Blender ID tells us the current user token is invalid.
+
+    This indicates the user should be immediately logged out.
+    """
+
+
 @blender_id.route('/store_scst', methods=['POST'])
 def store_subclient_token():
     """Verifies & stores a user's subclient-specific token."""
@@ -37,13 +47,6 @@ def store_subclient_token():
                     'subclient_user_id': str(db_user['_id'])}), status
 
 
-def blender_id_endpoint():
-    """Gets the endpoint for the authentication API. If the env variable
-    is defined, it's possible to override the (default) production address.
-    """
-    return current_app.config['BLENDER_ID_ENDPOINT'].rstrip('/')
-
-
 def validate_create_user(blender_id_user_id, token, oauth_subclient_id):
     """Validates a user against Blender ID, creating the user in our database.
 
@@ -64,78 +67,23 @@ def validate_create_user(blender_id_user_id, token, oauth_subclient_id):
     # Blender ID can be queried without user ID, and will always include the
     # correct user ID in its response.
     log.debug('Obtained user info from Blender ID: %s', user_info)
-    blender_id_user_id = user_info['id']
 
     # Store the user info in MongoDB.
-    db_user = find_user_in_db(blender_id_user_id, user_info)
-    db_id, status = upsert_user(db_user, blender_id_user_id)
+    db_user = find_user_in_db(user_info)
+    db_id, status = upsert_user(db_user)
 
     # Store the token in MongoDB.
-    authentication.store_token(db_id, token, token_expiry, oauth_subclient_id)
+    ip_based_roles = current_app.org_manager.roles_for_request()
+    authentication.store_token(db_id, token, token_expiry, oauth_subclient_id,
+                               org_roles=ip_based_roles)
+
+    if current_app.org_manager is not None:
+        roles = current_app.org_manager.refresh_roles(db_id)
+        db_user['roles'] = list(roles)
 
     return db_user, status
 
 
-def upsert_user(db_user, blender_id_user_id):
-    """Inserts/updates the user in MongoDB.
-
-    Retries a few times when there are uniqueness issues in the username.
-
-    :returns: the user's database ID and the status of the PUT/POST.
-        The status is 201 on insert, and 200 on update.
-    :type: (ObjectId, int)
-    """
-
-    if u'subscriber' in db_user.get('groups', []):
-        log.error('Non-ObjectID string found in user.groups: %s', db_user)
-        raise wz_exceptions.InternalServerError('Non-ObjectID string found in user.groups: %s' % db_user)
-
-    r = {}
-    for retry in range(5):
-        if '_id' in db_user:
-            # Update the existing user
-            attempted_eve_method = 'PUT'
-            db_id = db_user['_id']
-            r, _, _, status = current_app.put_internal('users', remove_private_keys(db_user),
-                                                       _id=db_id)
-            if status == 422:
-                log.error('Status %i trying to PUT user %s with values %s, should not happen! %s',
-                          status, db_id, remove_private_keys(db_user), r)
-        else:
-            # Create a new user, retry for non-unique usernames.
-            attempted_eve_method = 'POST'
-            r, _, _, status = current_app.post_internal('users', db_user)
-
-            if status not in {200, 201}:
-                log.error('Status %i trying to create user for BlenderID %s with values %s: %s',
-                          status, blender_id_user_id, db_user, r)
-                raise wz_exceptions.InternalServerError()
-
-            db_id = r['_id']
-            db_user.update(r)  # update with database/eve-generated fields.
-
-        if status == 422:
-            # Probably non-unique username, so retry a few times with different usernames.
-            log.info('Error creating new user: %s', r)
-            username_issue = r.get('_issues', {}).get(u'username', '')
-            if u'not unique' in username_issue:
-                # Retry
-                db_user['username'] = authentication.make_unique_username(db_user['email'])
-                continue
-
-        # Saving was successful, or at least didn't break on a non-unique username.
-        break
-    else:
-        log.error('Unable to create new user %s: %s', db_user, r)
-        raise wz_exceptions.InternalServerError()
-
-    if status not in (200, 201):
-        log.error('internal response from %s to Eve: %r %r', attempted_eve_method, status, r)
-        raise wz_exceptions.InternalServerError()
-
-    return db_id, status
-
-
 def validate_token(user_id, token, oauth_subclient_id):
     """Verifies a subclient token with Blender ID.
 
@@ -159,23 +107,38 @@ def validate_token(user_id, token, oauth_subclient_id):
     payload = {'user_id': user_id,
                'token': token}
     if oauth_subclient_id:
+        # If the subclient ID is set, the token belongs to another OAuth Client,
+        # in which case we do not set the client_id field.
         payload['subclient_id'] = oauth_subclient_id
+    else:
+        # We only want to accept Blender Cloud tokens.
+        payload['client_id'] = current_app.config['OAUTH_CREDENTIALS']['blender-id']['id']
 
-    url = '{0}/u/validate_token'.format(blender_id_endpoint())
+    url = '{0}/u/validate_token'.format(current_app.config['BLENDER_ID_ENDPOINT'])
     log.debug('POSTing to %r', url)
 
     # Retry a few times when POSTing to BlenderID fails.
     # Source: http://stackoverflow.com/a/15431343/875379
     s = requests.Session()
-    s.mount(blender_id_endpoint(), HTTPAdapter(max_retries=5))
+    s.mount(current_app.config['BLENDER_ID_ENDPOINT'], HTTPAdapter(max_retries=5))
 
     # POST to Blender ID, handling errors as negative verification results.
     try:
         r = s.post(url, data=payload, timeout=5,
                    verify=current_app.config['TLS_CERT_FILE'])
-    except requests.exceptions.ConnectionError as e:
+    except requests.exceptions.ConnectionError:
         log.error('Connection error trying to POST to %s, handling as invalid token.', url)
         return None, None
+    except requests.exceptions.ReadTimeout:
+        log.error('Read timeout trying to POST to %s, handling as invalid token.', url)
+        return None, None
+    except requests.exceptions.RequestException as ex:
+        log.error('Requests error "%s" trying to POST to %s, handling as invalid token.', ex, url)
+        return None, None
+    except IOError as ex:
+        log.error('Unknown I/O error "%s" trying to POST to %s, handling as invalid token.',
+                  ex, url)
+        return None, None
 
     if r.status_code != 200:
         log.debug('Token %s invalid, HTTP status %i returned', token, r.status_code)
@@ -199,43 +162,108 @@ def _compute_token_expiry(token_expires_string):
     the token.
     """
 
-    date_format = current_app.config['RFC1123_DATE_FORMAT']
-    blid_expiry = datetime.datetime.strptime(token_expires_string, date_format)
-    blid_expiry = blid_expiry.replace(tzinfo=tz_util.utc)
-    our_expiry = datetime.datetime.now(tz=tz_util.utc) + datetime.timedelta(hours=1)
+    # requirement is called python-dateutil, so PyCharm doesn't find it.
+    # noinspection PyPackageRequirements
+    from dateutil import parser
+
+    blid_expiry = parser.parse(token_expires_string)
+    blid_expiry = blid_expiry.astimezone(tz_util.utc)
+    our_expiry = utcnow() + datetime.timedelta(hours=1)
 
     return min(blid_expiry, our_expiry)
 
 
-def find_user_in_db(blender_id_user_id, user_info):
-    """Find the user in our database, creating/updating the returned document where needed.
+def get_user_blenderid(db_user: dict) -> str:
+    """Returns the Blender ID user ID for this Pillar user.
 
-    Does NOT update the user in the database.
+    Takes the string from 'auth.*.user_id' for the '*' where 'provider'
+    is 'blender-id'.
+
+    :returns the user ID, or the empty string when the user has none.
     """
 
-    users = current_app.data.driver.db['users']
+    bid_user_ids = [auth['user_id']
+                    for auth in db_user['auth']
+                    if auth['provider'] == 'blender-id']
+    try:
+        return bid_user_ids[0]
+    except IndexError:
+        return ''
 
-    query = {'auth': {'$elemMatch': {'user_id': str(blender_id_user_id),
-                                     'provider': 'blender-id'}}}
-    log.debug('Querying: %s', query)
-    db_user = users.find_one(query)
 
-    if db_user:
-        log.debug('User blender_id_user_id=%r already in our database, '
-                  'updating with info from Blender ID.', blender_id_user_id)
-        db_user['email'] = user_info['email']
-    else:
-        log.debug('User %r not yet in our database, create a new one.', blender_id_user_id)
-        db_user = authentication.create_new_user_document(
-            email=user_info['email'],
-            user_id=blender_id_user_id,
-            username=user_info['full_name'])
-        db_user['username'] = authentication.make_unique_username(user_info['email'])
-        if not db_user['full_name']:
-            db_user['full_name'] = db_user['username']
+def fetch_blenderid_user() -> dict:
+    """Returns the user info of the currently logged in user from BlenderID.
 
-    return db_user
+    Returns an empty dict if communication fails.
+
+    Example dict:
+    {
+         "email": "some@email.example.com",
+         "full_name": "dr. Sybren A. St\u00fcvel",
+         "id": 5555,
+         "roles": {
+           "admin": true,
+           "bfct_trainer": false,
+           "cloud_has_subscription": true,
+           "cloud_subscriber": true,
+           "conference_speaker": true,
+           "network_member": true
+         }
+    }
+
+    :raises LogoutUser: when Blender ID tells us the current token is
+        invalid, and the user should be logged out.
+    """
+    import httplib2  # used by the oauth2 package
+
+    my_log = log.getChild('fetch_blenderid_user')
+
+    bid_url = '%s/api/user' % current_app.config['BLENDER_ID_ENDPOINT']
+    my_log.debug('Fetching user info from %s', bid_url)
+
+    credentials = current_app.config['OAUTH_CREDENTIALS']['blender-id']
+    oauth_token = session.get('blender_id_oauth_token')
+    if not oauth_token:
+        my_log.warning('no Blender ID oauth token found in user session')
+        return {}
+
+    assert isinstance(oauth_token, str), f'oauth token must be str, not {type(oauth_token)}'
+
+    oauth_session = OAuth2Session(
+        credentials['id'], credentials['secret'],
+        access_token=oauth_token)
+
+    try:
+        bid_resp = oauth_session.get(bid_url)
+    except httplib2.HttpLib2Error:
+        my_log.exception('Error getting %s from BlenderID', bid_url)
+        return {}
+
+    if bid_resp.status_code == 403:
+        my_log.warning('Error %i from BlenderID %s, logging out user', bid_resp.status_code, bid_url)
+        raise LogoutUser()
+
+    if bid_resp.status_code != 200:
+        my_log.warning('Error %i from BlenderID %s: %s', bid_resp.status_code, bid_url, bid_resp.text)
+        return {}
+
+    payload = bid_resp.json()
+    if not payload:
+        my_log.warning('Empty data returned from BlenderID %s', bid_url)
+        return {}
+
+    my_log.debug('BlenderID returned %s', payload)
+    return payload
 
 
 def setup_app(app, url_prefix):
     app.register_api_blueprint(blender_id, url_prefix=url_prefix)
+
+
+def switch_user_url(next_url: str) -> str:
+    from urllib.parse import quote
+
+    base_url = '%s/switch' % current_app.config['BLENDER_ID_ENDPOINT']
+    if next_url:
+        return '%s?next=%s' % (base_url, quote(next_url))
+    return base_url

pillar/api/custom_field_validation.py

@@ -1,10 +1,13 @@
 import logging
 
 from bson import ObjectId, tz_util
-from datetime import datetime, tzinfo
+from datetime import datetime
+import cerberus.errors
 from eve.io.mongo import Validator
 from flask import current_app
 
+import pillar.markdown
+
 log = logging.getLogger(__name__)
 
 
@@ -61,13 +64,13 @@ class ValidateCustomFields(Validator):
         Only validates the dict values, not the keys. Modifies the given dict in-place.
         """
 
-        assert dict_valueschema[u'type'] == u'dict'
+        assert dict_valueschema['type'] == 'dict'
         assert isinstance(dict_property, dict)
 
         for key, val in dict_property.items():
-            item_schema = {u'item': dict_valueschema}
-            item_prop = {u'item': val}
-            dict_property[key] = self.convert_properties(item_prop, item_schema)[u'item']
+            item_schema = {'item': dict_valueschema}
+            item_prop = {'item': val}
+            dict_property[key] = self.convert_properties(item_prop, item_schema)['item']
 
     def _validate_valid_properties(self, valid_properties, field, value):
         from pillar.api.utils import project_get_node_type
@@ -102,6 +105,9 @@ class ValidateCustomFields(Validator):
         val = v.validate(value)
 
         if val:
+            # This ensures the modifications made by v's coercion rules are
+            # visible to this validator's output.
+            self.current[field] = v.current
             return True
 
         log.warning('Error validating properties for node %s: %s', self.document, v.errors)
@@ -125,3 +131,79 @@ class ValidateCustomFields(Validator):
 
         if not value:
             self._error(field, "Value is required once the document was created")
+
+    def _validate_type_iprange(self, field_name: str, value: str):
+        """Ensure the field contains a valid IP address.
+
+        Supports both IPv6 and IPv4 ranges. Requires the IPy module.
+        """
+
+        from IPy import IP
+
+        try:
+            ip = IP(value, make_net=True)
+        except ValueError as ex:
+            self._error(field_name, str(ex))
+            return
+
+        if ip.prefixlen() == 0:
+            self._error(field_name, 'Zero-length prefix is not allowed')
+
+    def _validate_type_binary(self, field_name: str, value: bytes):
+        """Add support for binary type.
+
+        This type was actually introduced in Cerberus 1.0, so we can drop
+        support for this once Eve starts using that version (or newer).
+        """
+
+        if not isinstance(value, (bytes, bytearray)):
+            self._error(field_name, f'wrong value type {type(value)}, expected bytes or bytearray')
+
+    def _validate_coerce(self, coerce, field: str, value):
+        """Override Cerberus' _validate_coerce method for richer features.
+
+        This now supports named coercion functions (available in Cerberus 1.0+)
+        and passes the field name to coercion functions as well.
+        """
+        if isinstance(coerce, str):
+            coerce = getattr(self, f'_normalize_coerce_{coerce}')
+
+        try:
+            return coerce(field, value)
+        except (TypeError, ValueError):
+            self._error(field, cerberus.errors.ERROR_COERCION_FAILED.format(field))
+
+    def _normalize_coerce_markdown(self, field: str, value):
+        """Render Markdown from this field into {field}_html.
+
+        The field name MUST NOT end in `_html`. The Markdown is read from this
+        field and the rendered HTML is written to the field `{field}_html`.
+        """
+        html = pillar.markdown.markdown(value)
+        field_name = pillar.markdown.cache_field_name(field)
+        self.current[field_name] = html
+        return value
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+
+    v = ValidateCustomFields()
+    v.schema = {
+        'foo': {'type': 'string', 'coerce': 'markdown'},
+        'foo_html': {'type': 'string'},
+        'nested': {
+            'type': 'dict',
+            'schema': {
+                'bar': {'type': 'string', 'coerce': 'markdown'},
+                'bar_html': {'type': 'string'},
+            }
+        }
+    }
+    print('Valid   :', v.validate({
+        'foo': '# Title\n\nHeyyyy',
+        'nested': {'bar': 'bhahaha'},
+    }))
+    print('Document:')
+    pprint(v.document)
+    print('Errors  :', v.errors)

pillar/api/encoding.py

@@ -1,15 +1,16 @@
-import logging
-
 import datetime
+import json
+import logging
 import os
-from bson import ObjectId, tz_util
+
+from bson import ObjectId
 from flask import Blueprint
 from flask import abort
 from flask import current_app
 from flask import request
+
 from pillar.api import utils
-from pillar.api.utils.gcs import GoogleCloudStorageBucket
-from pillar.api.utils import skip_when_testing
+from pillar.api.file_storage_backends import Bucket
 
 encoding = Blueprint('encoding', __name__)
 log = logging.getLogger(__name__)
@@ -32,6 +33,7 @@ def size_descriptor(width, height):
         1280: '720p',
         1920: '1080p',
         2048: '2k',
+        3840: 'UHD',
         4096: '4k',
     }
 
@@ -42,13 +44,6 @@ def size_descriptor(width, height):
     return '%ip' % height
 
 
-@skip_when_testing
-def rename_on_gcs(bucket_name, from_path, to_path):
-    gcs = GoogleCloudStorageBucket(str(bucket_name))
-    blob = gcs.bucket.blob(from_path)
-    gcs.bucket.rename_blob(blob, to_path)
-
-
 @encoding.route('/zencoder/notifications', methods=['POST'])
 def zencoder_notifications():
     """
@@ -102,25 +97,24 @@ def zencoder_notifications():
     file_doc['processing']['status'] = job_state
 
     if job_state == 'failed':
-        log.warning('Zencoder job %i for file %s failed.', zencoder_job_id, file_id)
-        # Log what Zencoder told us went wrong.
-        for output in data['outputs']:
-            if not any('error' in key for key in output):
-                continue
-            log.warning('Errors for output %s:', output['url'])
-            for key in output:
-                if 'error' in key:
-                    log.info('    %s: %s', key, output[key])
+        log.warning('Zencoder job %s for file %s failed: %s', zencoder_job_id, file_id,
+                    json.dumps(data, sort_keys=True, indent=4))
 
         file_doc['status'] = 'failed'
         current_app.put_internal('files', file_doc, _id=file_id)
+
+        # This is 'okay' because we handled the Zencoder notification properly.
         return "You failed, but that's okay.", 200
 
     log.info('Zencoder job %s for file %s completed with status %s.', zencoder_job_id, file_id,
              job_state)
 
     # For every variation encoded, try to update the file object
-    root, _ = os.path.splitext(file_doc['file_path'])
+    storage_name, _ = os.path.splitext(file_doc['file_path'])
+    nice_name, _ = os.path.splitext(file_doc['filename'])
+
+    bucket_class = Bucket.for_backend(file_doc['backend'])
+    bucket = bucket_class(str(file_doc['project']))
 
     for output in data['outputs']:
         video_format = output['format']
@@ -141,16 +135,16 @@ def zencoder_notifications():
 
         # Rename the file to include the now-known size descriptor.
         size = size_descriptor(output['width'], output['height'])
-        new_fname = '{}-{}.{}'.format(root, size, video_format)
+        new_fname = f'{storage_name}-{size}.{video_format}'
 
-        # Rename on Google Cloud Storage
+        # Rename the file on the storage.
+        blob = bucket.blob(variation['file_path'])
         try:
-            rename_on_gcs(file_doc['project'],
-                          '_/' + variation['file_path'],
-                          '_/' + new_fname)
+            new_blob = bucket.rename_blob(blob, new_fname)
+            new_blob.update_filename(f'{nice_name}-{size}.{video_format}')
         except Exception:
-            log.warning('Unable to rename GCS blob %r to %r. Keeping old name.',
-                        variation['file_path'], new_fname, exc_info=True)
+            log.warning('Unable to rename blob %r to %r. Keeping old name.',
+                        blob, new_fname, exc_info=True)
         else:
             variation['file_path'] = new_fname
 
@@ -167,9 +161,12 @@ def zencoder_notifications():
     file_doc['status'] = 'complete'
 
     # Force an update of the links on the next load of the file.
-    file_doc['link_expires'] = datetime.datetime.now(tz=tz_util.utc) - datetime.timedelta(days=1)
+    file_doc['link_expires'] = utils.utcnow() - datetime.timedelta(days=1)
 
-    current_app.put_internal('files', file_doc, _id=file_id)
+    r, _, _, status = current_app.put_internal('files', file_doc, _id=file_id)
+    if status != 200:
+        log.error('unable to save file %s after Zencoder notification: %s', file_id, r)
+        return json.dumps(r), 500
 
     return '', 204
 

pillar/api/eve_settings.py

@@ -88,8 +88,8 @@ users_schema = {
         }
     },
     'auth': {
-        # Storage of authentication credentials (one will be able to auth with
-        # multiple providers on the same account)
+        # Storage of authentication credentials (one will be able to auth with multiple providers on
+        # the same account)
         'type': 'list',
         'required': True,
         'schema': {
@@ -97,13 +97,12 @@ users_schema = {
             'schema': {
                 'provider': {
                     'type': 'string',
-                    'allowed': ["blender-id", "local"],
+                    'allowed': ['local', 'blender-id', 'facebook', 'google'],
                 },
                 'user_id': {
                     'type': 'string'
                 },
-                # A token is considered a "password" in case the provider is
-                # "local".
+                # A token is considered a "password" in case the provider is "local".
                 'token': {
                     'type': 'string'
                 }
@@ -128,7 +127,22 @@ users_schema = {
                 'schema': {'type': 'string'}
             }
         }
-    }
+    },
+
+    # Properties defined by extensions. Extensions should use their name (see the
+    # PillarExtension.name property) as the key, and are free to use whatever they want as value,
+    # but we suggest a dict for future extendability.
+    # Properties can be of two types:
+    # - public: they will be visible to the world (for example as part of the User.find() query)
+    # - private: visible only to their user
+    'extension_props_public': {
+        'type': 'dict',
+        'required': False,
+    },
+    'extension_props_private': {
+        'type': 'dict',
+        'required': False,
+    },
 }
 
 organizations_schema = {
@@ -138,19 +152,12 @@ organizations_schema = {
         'maxlength': 128,
         'required': True
     },
-    'email': {
-        'type': 'string'
-    },
-    'url': {
-        'type': 'string',
-        'minlength': 1,
-        'maxlength': 128,
-        'required': True
-    },
     'description': {
         'type': 'string',
         'maxlength': 256,
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     'website': {
         'type': 'string',
         'maxlength': 256,
@@ -162,7 +169,15 @@ organizations_schema = {
     'picture': dict(
         nullable=True,
         **_file_embedded_schema),
-    'users': {
+    'admin_uid': {
+        'type': 'objectid',
+        'data_relation': {
+            'resource': 'users',
+            'field': '_id',
+        },
+        'required': True,
+    },
+    'members': {
         'type': 'list',
         'default': [],
         'schema': {
@@ -170,51 +185,52 @@ organizations_schema = {
             'data_relation': {
                 'resource': 'users',
                 'field': '_id',
-                'embeddable': True
             }
         }
     },
-    'teams': {
+    'unknown_members': {
+        'type': 'list',  # of email addresses of yet-to-register users.
+        'default': [],
+        'schema': {
+            'type': 'string',
+        },
+    },
+
+    # Maximum size of the organization, i.e. len(members) + len(unknown_members) may
+    # not exceed this.
+    'seat_count': {
+        'type': 'integer',
+        'required': True,
+    },
+
+    # Roles that the members of this organization automatically get.
+    'org_roles': {
         'type': 'list',
         'default': [],
+        'schema': {
+            'type': 'string',
+        },
+    },
+
+    # Identification of the subscription that pays for this organisation
+    # in an external subscription/payment management system.
+    'payment_subscription_id': {
+        'type': 'string',
+    },
+
+    'ip_ranges': {
+        'type': 'list',
         'schema': {
             'type': 'dict',
             'schema': {
-                # Team name
-                'name': {
-                    'type': 'string',
-                    'minlength': 1,
-                    'maxlength': 128,
-                    'required': True
-                },
-                # List of user ids for the team
-                'users': {
-                    'type': 'list',
-                    'default': [],
-                    'schema': {
-                        'type': 'objectid',
-                        'data_relation': {
-                            'resource': 'users',
-                            'field': '_id',
-                        }
+                # see _validate_type_{typename} in ValidateCustomFields:
+                'start': {'type': 'binary', 'required': True},
+                'end': {'type': 'binary', 'required': True},
+                'prefix': {'type': 'integer', 'required': True},
+                'human': {'type': 'iprange', 'required': True},
             }
         },
-                # List of groups assigned to the team (this will automatically
-                # update the groups property of each user in the team)
-                'groups': {
-                    'type': 'list',
-                    'default': [],
-                    'schema': {
-                        'type': 'objectid',
-                        'data_relation': {
-                            'resource': 'groups',
-                            'field': '_id',
-                        }
-                    }
-                }
-            }
-        }
-    }
+    },
 }
 
 permissions_embedded_schema = {
@@ -276,7 +292,9 @@ nodes_schema = {
     },
     'description': {
         'type': 'string',
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     'picture': _file_embedded_schema,
     'order': {
         'type': 'integer',
@@ -326,6 +344,10 @@ tokens_schema = {
         'required': True,
     },
     'token': {
+        'type': 'string',
+        'required': False,
+    },
+    'token_hashed': {
         'type': 'string',
         'required': True,
     },
@@ -336,7 +358,16 @@ tokens_schema = {
     'is_subclient_token': {
         'type': 'boolean',
         'required': False,
-    }
+    },
+
+    # Roles this user gets while this token is valid.
+    'org_roles': {
+        'type': 'list',
+        'default': [],
+        'schema': {
+            'type': 'string',
+        },
+    },
 }
 
 files_schema = {
@@ -394,7 +425,7 @@ files_schema = {
     'backend': {
         'type': 'string',
         'required': True,
-        'allowed': ["attract-web", "pillar", "cdnsun", "gcs", "unittest"]
+        'allowed': ["local", "pillar", "cdnsun", "gcs", "unittest"]
     },
 
     # Where the file is in the backend storage itself. In the case of GCS,
@@ -508,7 +539,9 @@ projects_schema = {
     },
     'description': {
         'type': 'string',
+        'coerce': 'markdown',
     },
+    '_description_html': {'type': 'string'},
     # Short summary for the project
     'summary': {
         'type': 'string',
@@ -534,8 +567,9 @@ projects_schema = {
     'category': {
         'type': 'string',
         'allowed': [
-            'training',
+            'course',
             'film',
+            'workshop',
             'assets',
             'software',
             'game',
@@ -718,13 +752,9 @@ users = {
     'cache_expires': 10,
 
     'resource_methods': ['GET'],
-    'item_methods': ['GET', 'PUT', 'PATCH'],
+    'item_methods': ['GET', 'PUT'],
     'public_item_methods': ['GET'],
 
-    # By default don't include the 'auth' field. It can still be obtained
-    # using projections, though, so we block that in hooks.
-    'datasource': {'projection': {u'auth': 0}},
-
     'schema': users_schema
 }
 
@@ -738,11 +768,12 @@ tokens = {
 }
 
 files = {
+    'schema': files_schema,
     'resource_methods': ['GET', 'POST'],
     'item_methods': ['GET', 'PATCH'],
     'public_methods': ['GET'],
     'public_item_methods': ['GET'],
-    'schema': files_schema
+    'soft_delete': True,
 }
 
 groups = {
@@ -754,8 +785,11 @@ groups = {
 
 organizations = {
     'schema': organizations_schema,
-    'public_item_methods': ['GET'],
-    'public_methods': ['GET']
+    'resource_methods': ['GET', 'POST'],
+    'item_methods': ['GET'],
+    'public_item_methods': [],
+    'public_methods': [],
+    'soft_delete': True,
 }
 
 projects = {

pillar/api/file_storage/__init__.py

@@ -1,32 +1,37 @@
+import datetime
 import io
 import logging
 import mimetypes
+import os
+import pathlib
 import tempfile
+import typing
 import uuid
 from hashlib import md5
-import os
-import requests
-import bson.tz_util
-import datetime
+
 import eve.utils
 import pymongo
 import werkzeug.exceptions as wz_exceptions
+import werkzeug.datastructures
+
 from bson import ObjectId
 from flask import Blueprint
 from flask import current_app
-from flask import g
 from flask import jsonify
 from flask import request
 from flask import send_from_directory
 from flask import url_for, helpers
+
 from pillar.api import utils
-from pillar.api.utils.imaging import generate_local_thumbnails
-from pillar.api.utils import remove_private_keys, authentication
-from pillar.api.utils.authorization import require_login, user_has_role, \
+from pillar.api.file_storage_backends.gcs import GoogleCloudStorageBucket, \
+    GoogleCloudStorageBlob
+from pillar.api.utils import remove_private_keys, imaging
+from pillar.api.utils.authorization import require_login, \
     user_matches_roles
 from pillar.api.utils.cdn import hash_file_path
 from pillar.api.utils.encoding import Encoder
-from pillar.api.utils.gcs import GoogleCloudStorageBucket
+from pillar.api.file_storage_backends import default_storage_backend, Bucket
+from pillar.auth import current_user
 
 log = logging.getLogger(__name__)
 
@@ -45,31 +50,6 @@ mimetypes.add_type('application/x-radiance-hdr', '.hdr')
 mimetypes.add_type('application/x-exr', '.exr')
 
 
-@file_storage.route('/gcs/<bucket_name>/<subdir>/')
-@file_storage.route('/gcs/<bucket_name>/<subdir>/<path:file_path>')
-def browse_gcs(bucket_name, subdir, file_path=None):
-    """Browse the content of a Google Cloud Storage bucket"""
-
-    # Initialize storage client
-    storage = GoogleCloudStorageBucket(bucket_name, subdir=subdir)
-    if file_path:
-        # If we provided a file_path, we try to fetch it
-        file_object = storage.Get(file_path)
-        if file_object:
-            # If it exists, return file properties in a dictionary
-            return jsonify(file_object)
-        else:
-            listing = storage.List(file_path)
-            return jsonify(listing)
-            # We always return an empty listing even if the directory does not
-            # exist. This can be changed later.
-            # return abort(404)
-
-    else:
-        listing = storage.List('')
-        return jsonify(listing)
-
-
 @file_storage.route('/file', methods=['POST'])
 @file_storage.route('/file/<path:file_name>', methods=['GET', 'POST'])
 def index(file_name=None):
@@ -103,7 +83,10 @@ def index(file_name=None):
     return jsonify({'url': url_for('file_storage.index', file_name=file_name)})
 
 
-def _process_image(gcs, file_id, local_file, src_file):
+def _process_image(bucket: Bucket,
+                   file_id: ObjectId,
+                   local_file: tempfile._TemporaryFileWrapper,
+                   src_file: dict):
     from PIL import Image
 
     im = Image.open(local_file)
@@ -113,8 +96,9 @@ def _process_image(gcs, file_id, local_file, src_file):
 
     # Generate previews
     log.info('Generating thumbnails for file %s', file_id)
-    src_file['variations'] = generate_local_thumbnails(src_file['name'],
-                                                       local_file.name)
+    local_path = pathlib.Path(local_file.name)
+    name_base = pathlib.Path(src_file['name']).stem
+    src_file['variations'] = imaging.generate_local_thumbnails(name_base, local_path)
 
     # Send those previews to Google Cloud Storage.
     log.info('Uploading %i thumbnails for file %s to Google Cloud Storage '
@@ -124,11 +108,11 @@ def _process_image(gcs, file_id, local_file, src_file):
     for variation in src_file['variations']:
         fname = variation['file_path']
         if current_app.config['TESTING']:
-            log.warning('  - NOT sending thumbnail %s to GCS', fname)
+            log.warning('  - NOT sending thumbnail %s to %s', fname, bucket)
         else:
-            log.debug('  - Sending thumbnail %s to GCS', fname)
-            blob = gcs.bucket.blob('_/' + fname, chunk_size=256 * 1024 * 2)
-            blob.upload_from_filename(variation['local_path'],
+            blob = bucket.blob(fname)
+            log.debug('  - Sending thumbnail %s to %s', fname, blob)
+            blob.upload_from_path(pathlib.Path(variation['local_path']),
                                   content_type=variation['content_type'])
 
             if variation.get('size') == 't':
@@ -146,11 +130,99 @@ def _process_image(gcs, file_id, local_file, src_file):
     src_file['status'] = 'complete'
 
 
-def _process_video(gcs, file_id, local_file, src_file):
-    """Video is processed by Zencoder; the file isn't even stored locally."""
+def _video_size_pixels(filename: pathlib.Path) -> typing.Tuple[int, int]:
+    """Figures out the size (in pixels) of the video file.
+
+    Returns (0, 0) if there was any error detecting the size.
+    """
+
+    import json
+    import subprocess
+
+    cli_args = [
+        current_app.config['BIN_FFPROBE'],
+        '-loglevel', 'error',
+        '-hide_banner',
+        '-print_format', 'json',
+        '-select_streams', 'v:0',  # we only care about the first video stream
+        '-show_streams',
+        str(filename),
+    ]
+
+    if log.isEnabledFor(logging.INFO):
+        import shlex
+        cmd = ' '.join(shlex.quote(s) for s in cli_args)
+        log.info('Calling %s', cmd)
+
+    ffprobe = subprocess.run(
+        cli_args,
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        timeout=10,  # seconds
+    )
+
+    if ffprobe.returncode:
+        import shlex
+        cmd = ' '.join(shlex.quote(s) for s in cli_args)
+        log.error('Error running %s: stopped with return code %i',
+                  cmd, ffprobe.returncode)
+        log.error('Output was: %s', ffprobe.stdout)
+        return 0, 0
+
+    try:
+        ffprobe_info = json.loads(ffprobe.stdout)
+    except json.JSONDecodeError:
+        log.exception('ffprobe produced invalid JSON: %s', ffprobe.stdout)
+        return 0, 0
+
+    try:
+        stream_info = ffprobe_info['streams'][0]
+        return stream_info['width'], stream_info['height']
+    except (KeyError, IndexError):
+        log.exception('ffprobe produced unexpected JSON: %s', ffprobe.stdout)
+        return 0, 0
+
+
+def _video_cap_at_1080(width: int, height: int) -> typing.Tuple[int, int]:
+    """Returns an appropriate width/height for a video capped at 1920x1080.
+
+    Takes into account that h264 has limitations:
+        - the width must be a multiple of 16
+        - the height must be a multiple of 8
+    """
+
+    if width > 1920:
+        # The height must be a multiple of 8
+        new_height = height / width * 1920
+        height = new_height - (new_height % 8)
+        width = 1920
+
+    if height > 1080:
+        # The width must be a multiple of 16
+        new_width = width / height * 1080
+        width = new_width - (new_width % 16)
+        height = 1080
+
+    return int(width), int(height)
+
+
+def _process_video(gcs,
+                   file_id: ObjectId,
+                   local_file: tempfile._TemporaryFileWrapper,
+                   src_file: dict):
+    """Video is processed by Zencoder."""
 
     log.info('Processing video for file %s', file_id)
 
+    # Use ffprobe to find the size (in pixels) of the video.
+    # Even though Zencoder can do resizing to a maximum resolution without upscaling,
+    # by determining the video size here we already have this information in the file
+    # document before Zencoder calls our notification URL. It also opens up possibilities
+    # for other encoding backends that don't support this functionality.
+    video_width, video_height = _video_size_pixels(pathlib.Path(local_file.name))
+    capped_video_width, capped_video_height = _video_cap_at_1080(video_width, video_height)
+
     # Create variations
     root, _ = os.path.splitext(src_file['file_path'])
     src_file['variations'] = []
@@ -163,8 +235,8 @@ def _process_video(gcs, file_id, local_file, src_file):
         file_path='{}-{}.{}'.format(root, v, v),
         size='',
         duration=0,
-        width=0,
-        height=0,
+        width=capped_video_width,
+        height=capped_video_height,
         length=0,
         md5='',
     )
@@ -175,8 +247,8 @@ def _process_video(gcs, file_id, local_file, src_file):
     if current_app.config['TESTING']:
         log.warning('_process_video: NOT sending out encoding job due to '
                     'TESTING=%r', current_app.config['TESTING'])
-        j = type('EncoderJob', (), {'process_id': 'fake-process-id',
-                                    'backend': 'fake'})
+        j = {'process_id': 'fake-process-id',
+             'backend': 'fake'}
     else:
         j = Encoder.job_create(src_file)
         if j is None:
@@ -194,14 +266,14 @@ def _process_video(gcs, file_id, local_file, src_file):
         'backend': j['backend']}
 
 
-def process_file(gcs, file_id, local_file):
+def process_file(bucket: Bucket,
+                 file_id: typing.Union[str, ObjectId],
+                 local_file: tempfile._TemporaryFileWrapper):
     """Process the file by creating thumbnails, sending to Zencoder, etc.
 
     :param file_id: '_id' key of the file
-    :type file_id: ObjectId or str
     :param local_file: locally stored file, or None if no local processing is
     needed.
-    :type local_file: file
     """
 
     file_id = ObjectId(file_id)
@@ -218,8 +290,8 @@ def process_file(gcs, file_id, local_file):
     # TODO: overrule the content type based on file extention & magic numbers.
     mime_category, src_file['format'] = src_file['content_type'].split('/', 1)
 
-    # Prevent video handling for non-admins.
-    if not user_has_role(u'admin') and mime_category == 'video':
+    # Only allow video encoding when the user has the correct capability.
+    if not current_user.has_cap('encode-video') and mime_category == 'video':
         if src_file['format'].startswith('x-'):
             xified = src_file['format']
         else:
@@ -227,10 +299,10 @@ def process_file(gcs, file_id, local_file):
 
         src_file['content_type'] = 'application/%s' % xified
         mime_category = 'application'
-        log.info('Not processing video file %s for non-admin user', file_id)
+        log.info('Not processing video file %s for non-video-encoding user', file_id)
 
     # Run the required processor, based on the MIME category.
-    processors = {
+    processors: typing.Mapping[str, typing.Callable] = {
         'image': _process_image,
         'video': _process_video,
     }
@@ -249,7 +321,7 @@ def process_file(gcs, file_id, local_file):
         update_file_doc(file_id, status='processing')
 
         try:
-            processor(gcs, file_id, local_file, src_file)
+            processor(bucket, file_id, local_file, src_file)
         except Exception:
             log.warning('process_file(%s): error when processing file, '
                         'resetting status to '
@@ -265,65 +337,41 @@ def process_file(gcs, file_id, local_file):
                     file_id, status, r)
 
 
-def delete_file(file_item):
-    def process_file_delete(file_item):
-        """Given a file item, delete the actual file from the storage backend.
-        This function can be probably made self-calling."""
-        if file_item['backend'] == 'gcs':
-            storage = GoogleCloudStorageBucket(str(file_item['project']))
-            storage.Delete(file_item['file_path'])
-            # Delete any file variation found in the file_item document
-            if 'variations' in file_item:
-                for v in file_item['variations']:
-                    storage.Delete(v['file_path'])
-            return True
-        elif file_item['backend'] == 'pillar':
-            pass
-        elif file_item['backend'] == 'cdnsun':
-            pass
-        else:
-            pass
-
-    files_collection = current_app.data.driver.db['files']
-    # Collect children (variations) of the original file
-    children = files_collection.find({'parent': file_item['_id']})
-    for child in children:
-        process_file_delete(child)
-    # Finally remove the original file
-    process_file_delete(file_item)
-
-
-def generate_link(backend, file_path, project_id=None, is_public=False):
+def generate_link(backend, file_path: str, project_id: str=None, is_public=False) -> str:
     """Hook to check the backend of a file resource, to build an appropriate link
     that can be used by the client to retrieve the actual file.
     """
 
-    if backend == 'gcs':
-        if current_app.config['TESTING']:
+    # TODO: replace config['TESTING'] with mocking GCS.
+    if backend == 'gcs' and current_app.config['TESTING']:
         log.info('Skipping GCS link generation, and returning a fake link '
                  'instead.')
         return '/path/to/testing/gcs/%s' % file_path
 
-        storage = GoogleCloudStorageBucket(project_id)
-        blob = storage.Get(file_path)
+    if backend in {'gcs', 'local'}:
+        from ..file_storage_backends import Bucket
+
+        bucket_cls = Bucket.for_backend(backend)
+        storage = bucket_cls(project_id)
+        blob = storage.get_blob(file_path)
+
         if blob is None:
-            log.warning('generate_link(%r, %r): unable to find blob for file path,'
-                        ' returning empty link.', backend, file_path)
+            log.warning('generate_link(%r, %r): unable to find blob for file'
+                        ' path, returning empty link.', backend, file_path)
             return ''
 
-        if is_public:
-            return blob['public_url']
-        return blob['signed_url']
+        return blob.get_url(is_public=is_public)
 
-    if backend == 'pillar':
+    if backend == 'pillar':  # obsolete, replace with local.
         return url_for('file_storage.index', file_name=file_path,
                        _external=True, _scheme=current_app.config['SCHEME'])
     if backend == 'cdnsun':
         return hash_file_path(file_path, None)
     if backend == 'unittest':
-        return 'https://unit.test/%s' % md5(file_path).hexdigest()
+        return 'https://unit.test/%s' % md5(file_path.encode()).hexdigest()
 
-    log.warning('generate_link(): Unknown backend %r, returning empty string as new link.',
+    log.warning('generate_link(): Unknown backend %r, returning empty string '
+                'as new link.',
                 backend)
     return ''
 
@@ -338,12 +386,8 @@ def before_returning_file(response):
 
 def strip_link_and_variations(response):
     # Check the access level of the user.
-    if g.current_user is None:
-        has_full_access = False
-    else:
-        user_roles = g.current_user['roles']
-        access_roles = current_app.config['FULL_FILE_ACCESS_ROLES']
-        has_full_access = bool(user_roles.intersection(access_roles))
+    capability = current_app.config['FULL_FILE_ACCESS_CAP']
+    has_full_access = current_user.has_cap(capability)
 
     # Strip all file variations (unless image) and link to the actual file.
     if not has_full_access:
@@ -369,7 +413,7 @@ def ensure_valid_link(response):
     # log.debug('Inspecting link for file %s', response['_id'])
 
     # Check link expiry.
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
+    now = utils.utcnow()
     if 'link_expires' in response:
         link_expires = response['link_expires']
         if now < link_expires:
@@ -397,7 +441,14 @@ def generate_all_links(response, now):
         response['project']) if 'project' in response else None
     # TODO: add project id to all files
     backend = response['backend']
+
+    if 'file_path' in response:
         response['link'] = generate_link(backend, response['file_path'], project_id)
+    else:
+        import pprint
+        log.error('File without file_path properly, unable to generate links: %s',
+                  pprint.pformat(response))
+        return
 
     variations = response.get('variations')
     if variations:
@@ -410,6 +461,12 @@ def generate_all_links(response, now):
     response['link_expires'] = now + datetime.timedelta(seconds=validity_secs)
 
     patch_info = remove_private_keys(response)
+
+    # The project could have been soft-deleted, in which case it's fine to
+    # update the links to the file. However, Eve/Cerberus doesn't allow this;
+    # removing the 'project' key from the PATCH works around this.
+    patch_info.pop('project', None)
+
     file_id = ObjectId(response['_id'])
     (patch_resp, _, _, _) = current_app.patch_internal('files', patch_info,
                                                        _id=file_id)
@@ -427,22 +484,28 @@ def generate_all_links(response, now):
     response['_etag'] = etag_doc['_etag']
 
 
-def before_deleting_file(item):
-    delete_file(item)
-
-
 def on_pre_get_files(_, lookup):
     # Override the HTTP header, we always want to fetch the document from
     # MongoDB.
     parsed_req = eve.utils.parse_request('files')
     parsed_req.if_modified_since = None
 
+    # If there is no lookup, we would refresh *all* file documents,
+    # which is far too heavy to do in one client HTTP request.
+    if not lookup:
+        return
+
     # Only fetch it if the date got expired.
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
+    now = utils.utcnow()
     lookup_expired = lookup.copy()
     lookup_expired['link_expires'] = {'$lte': now}
 
     cursor = current_app.data.find('files', parsed_req, lookup_expired)
+    if cursor.count() == 0:
+        return
+
+    log.debug('Updating expired links for %d files that matched lookup %s',
+              cursor.count(), lookup_expired)
     for file_doc in cursor:
         # log.debug('Updating expired links for file %r.', file_doc['_id'])
         generate_all_links(file_doc, now)
@@ -458,7 +521,7 @@ def refresh_links_for_project(project_uuid, chunk_size, expiry_seconds):
     # Retrieve expired links.
     files_collection = current_app.data.driver.db['files']
 
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
+    now = utils.utcnow()
     expire_before = now + datetime.timedelta(seconds=expiry_seconds)
     log.info('Limiting to links that expire before %s', expire_before)
 
@@ -481,33 +544,43 @@ def refresh_links_for_project(project_uuid, chunk_size, expiry_seconds):
 def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds):
     import gcloud.exceptions
 
+    my_log = log.getChild(f'refresh_links_for_backend.{backend_name}')
+
     # Retrieve expired links.
     files_collection = current_app.data.driver.db['files']
     proj_coll = current_app.data.driver.db['projects']
 
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
+    now = utils.utcnow()
     expire_before = now + datetime.timedelta(seconds=expiry_seconds)
-    log.info('Limiting to links that expire before %s', expire_before)
+    my_log.info('Limiting to links that expire before %s', expire_before)
 
+    base_query = {'backend': backend_name, '_deleted': {'$ne': True}}
     to_refresh = files_collection.find(
-        {'$or': [{'backend': backend_name, 'link_expires': None},
-                 {'backend': backend_name, 'link_expires': {
-                     '$lt': expire_before}},
-                 {'backend': backend_name, 'link': None}]
+        {'$or': [{'link_expires': None, **base_query},
+                 {'link_expires': {'$lt': expire_before}, **base_query},
+                 {'link': None, **base_query}]
          }).sort([('link_expires', pymongo.ASCENDING)]).limit(
         chunk_size).batch_size(5)
 
-    if to_refresh.count() == 0:
-        log.info('No links to refresh.')
+    document_count = to_refresh.count()
+    if document_count == 0:
+        my_log.info('No links to refresh.')
         return
 
+    if 0 < chunk_size == document_count:
+        my_log.info('Found %d documents to refresh, probably limited by the chunk size.',
+                    document_count)
+    else:
+        my_log.info('Found %d documents to refresh.', document_count)
+
     refreshed = 0
+    report_chunks = min(max(5, document_count // 25), 100)
     for file_doc in to_refresh:
         try:
             file_id = file_doc['_id']
             project_id = file_doc.get('project')
             if project_id is None:
-                log.debug('Skipping file %s, it has no project.', file_id)
+                my_log.debug('Skipping file %s, it has no project.', file_id)
                 continue
 
             count = proj_coll.count({'_id': project_id, '$or': [
@@ -516,46 +589,50 @@ def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds):
             ]})
 
             if count == 0:
-                log.debug('Skipping file %s, project %s does not exist.',
+                my_log.debug('Skipping file %s, project %s does not exist.',
                              file_id, project_id)
                 continue
 
             if 'file_path' not in file_doc:
-                log.warning("Skipping file %s, missing 'file_path' property.",
+                my_log.warning("Skipping file %s, missing 'file_path' property.",
                                file_id)
                 continue
 
-            log.debug('Refreshing links for file %s', file_id)
+            my_log.debug('Refreshing links for file %s', file_id)
 
             try:
                 generate_all_links(file_doc, now)
             except gcloud.exceptions.Forbidden:
-                log.warning('Skipping file %s, GCS forbids us access to '
+                my_log.warning('Skipping file %s, GCS forbids us access to '
                                'project %s bucket.', file_id, project_id)
                 continue
             refreshed += 1
+
+            if refreshed % report_chunks == 0:
+                my_log.info('Refreshed %i links', refreshed)
         except KeyboardInterrupt:
-            log.warning('Aborting due to KeyboardInterrupt after refreshing %i '
+            my_log.warning('Aborting due to KeyboardInterrupt after refreshing %i '
                            'links', refreshed)
             return
 
-    log.info('Refreshed %i links', refreshed)
+    my_log.info('Refreshed %i links', refreshed)
 
 
 @require_login()
 def create_file_doc(name, filename, content_type, length, project,
-                    backend='gcs', **extra_fields):
+                    backend=None, **extra_fields):
     """Creates a minimal File document for storage in MongoDB.
 
     Doesn't save it to MongoDB yet.
     """
 
-    current_user = g.get('current_user')
+    if backend is None:
+        backend = current_app.config['STORAGE_BACKEND']
 
     file_doc = {'name': name,
                 'filename': filename,
                 'file_path': '',
-                'user': current_user['user_id'],
+                'user': current_user.user_id,
                 'backend': backend,
                 'md5': '',
                 'content_type': content_type,
@@ -601,10 +678,10 @@ def override_content_type(uploaded_file):
         del uploaded_file._parsed_content_type
 
 
-def assert_file_size_allowed(file_size):
+def assert_file_size_allowed(file_size: int):
     """Asserts that the current user is allowed to upload a file of the given size.
 
-    :raises
+    :raises wz_exceptions.RequestEntityTooLarge:
     """
 
     roles = current_app.config['ROLES_FOR_UNLIMITED_UPLOADS']
@@ -618,7 +695,7 @@ def assert_file_size_allowed(file_size):
     filesize_limit_mb = filesize_limit / 2.0 ** 20
     log.info('User %s tried to upload a %.3f MiB file, but is only allowed '
              '%.3f MiB.',
-             authentication.current_user_id(), file_size / 2.0 ** 20,
+             current_user.user_id, file_size / 2.0 ** 20,
              filesize_limit_mb)
     raise wz_exceptions.RequestEntityTooLarge(
         'To upload files larger than %i MiB, subscribe to Blender Cloud' %
@@ -627,7 +704,7 @@ def assert_file_size_allowed(file_size):
 
 @file_storage.route('/stream/<string:project_id>', methods=['POST', 'OPTIONS'])
 @require_login()
-def stream_to_storage(project_id):
+def stream_to_storage(project_id: str):
     project_oid = utils.str2id(project_id)
 
     projects = current_app.data.driver.db['projects']
@@ -637,14 +714,14 @@ def stream_to_storage(project_id):
         raise wz_exceptions.NotFound('Project %s does not exist' % project_id)
 
     log.info('Streaming file to bucket for project=%s user_id=%s', project_id,
-             authentication.current_user_id())
+             current_user.user_id)
     log.info('request.headers[Origin] = %r', request.headers.get('Origin'))
     log.info('request.content_length = %r', request.content_length)
 
-    # Try a check for the content length before we access request.files[]. This allows us
-    # to abort the upload early. The entire body content length is always a bit larger than
-    # the actual file size, so if we accept here, we're sure it'll be accepted in subsequent
-    # checks as well.
+    # Try a check for the content length before we access request.files[].
+    # This allows us to abort the upload early. The entire body content length
+    # is always a bit larger than the actual file size, so if we accept here,
+    # we're sure it'll be accepted in subsequent checks as well.
     if request.content_length:
         assert_file_size_allowed(request.content_length)
 
@@ -659,50 +736,56 @@ def stream_to_storage(project_id):
 
     override_content_type(uploaded_file)
     if not uploaded_file.content_type:
-        log.warning('File uploaded to project %s without content type.', project_oid)
+        log.warning('File uploaded to project %s without content type.',
+                    project_oid)
         raise wz_exceptions.BadRequest('Missing content type.')
 
-    if uploaded_file.content_type.startswith('image/'):
-        # We need to do local thumbnailing, so we have to write the stream
+    if uploaded_file.content_type.startswith('image/') or uploaded_file.content_type.startswith(
+            'video/'):
+        # We need to do local thumbnailing and ffprobe, so we have to write the stream
         # both to Google Cloud Storage and to local storage.
-        local_file = tempfile.NamedTemporaryFile(dir=current_app.config['STORAGE_DIR'])
+        local_file = tempfile.NamedTemporaryFile(
+            dir=current_app.config['STORAGE_DIR'])
         uploaded_file.save(local_file)
-        local_file.seek(0)  # Make sure that a re-read starts from the beginning.
-        stream_for_gcs = local_file
+        local_file.seek(0)  # Make sure that re-read starts from the beginning.
     else:
-        local_file = None
-        stream_for_gcs = uploaded_file.stream
+        local_file = uploaded_file.stream
 
+    result = upload_and_process(local_file, uploaded_file, project_id)
+    resp = jsonify(result)
+    resp.status_code = result['status_code']
+    add_access_control_headers(resp)
+    return resp
+
+
+def upload_and_process(local_file: typing.Union[io.BytesIO, typing.BinaryIO],
+                       uploaded_file: werkzeug.datastructures.FileStorage,
+                       project_id: str):
     # Figure out the file size, as we need to pass this in explicitly to GCloud.
     # Otherwise it always uses os.fstat(file_obj.fileno()).st_size, which isn't
     # supported by a BytesIO object (even though it does have a fileno
     # attribute).
-    if isinstance(stream_for_gcs, io.BytesIO):
-        file_size = len(stream_for_gcs.getvalue())
+    if isinstance(local_file, io.BytesIO):
+        file_size = len(local_file.getvalue())
     else:
-        file_size = os.fstat(stream_for_gcs.fileno()).st_size
+        file_size = os.fstat(local_file.fileno()).st_size
 
     # Check the file size again, now that we know its size for sure.
     assert_file_size_allowed(file_size)
 
     # Create file document in MongoDB.
-    file_id, internal_fname, status = create_file_doc_for_upload(project_oid,
-                                                                 uploaded_file)
+    file_id, internal_fname, status = create_file_doc_for_upload(project_id, uploaded_file)
 
-    if current_app.config['TESTING']:
-        log.warning('NOT streaming to GCS because TESTING=%r',
-                    current_app.config['TESTING'])
-        # Fake a Blob object.
-        gcs = None
-        blob = type('Blob', (), {'size': file_size})
-    else:
-        blob, gcs = stream_to_gcs(file_id, file_size, internal_fname,
-                                  project_id, stream_for_gcs,
-                                  uploaded_file.mimetype)
+    # Copy the file into storage.
+    bucket = default_storage_backend(project_id)
+    blob = bucket.blob(internal_fname)
+    blob.create_from_file(local_file,
+                          file_size=file_size,
+                          content_type=uploaded_file.mimetype)
 
     log.debug('Marking uploaded file id=%s, fname=%s, '
               'size=%i as "queued_for_processing"',
-              file_id, internal_fname, blob.size)
+              file_id, internal_fname, file_size)
     update_file_doc(file_id,
                     status='queued_for_processing',
                     file_path=internal_fname,
@@ -711,7 +794,7 @@ def stream_to_storage(project_id):
 
     log.debug('Processing uploaded file id=%s, fname=%s, size=%i', file_id,
               internal_fname, blob.size)
-    process_file(gcs, file_id, local_file)
+    process_file(bucket, file_id, local_file)
 
     # Local processing is done, we can close the local file so it is removed.
     if local_file is not None:
@@ -723,26 +806,20 @@ def stream_to_storage(project_id):
     # Status is 200 if the file already existed, and 201 if it was newly
     # created.
     # TODO: add a link to a thumbnail in the response.
-    resp = jsonify(status='ok', file_id=str(file_id))
-    resp.status_code = status
-    add_access_control_headers(resp)
-    return resp
+    return dict(status='ok', file_id=str(file_id), status_code=status)
 
 
-def stream_to_gcs(file_id, file_size, internal_fname, project_id,
-                  stream_for_gcs, content_type):
+from ..file_storage_backends.abstract import FileType
+
+
+def stream_to_gcs(file_id: ObjectId, file_size: int, internal_fname: str, project_id: ObjectId,
+                  stream_for_gcs: FileType, content_type: str) \
+        -> typing.Tuple[GoogleCloudStorageBlob, GoogleCloudStorageBucket]:
     # Upload the file to GCS.
-    from gcloud.streaming import transfer
-    log.debug('Streaming file to GCS bucket; id=%s, fname=%s, size=%i',
-              file_id, internal_fname, file_size)
-    # Files larger than this many bytes will be streamed directly from disk,
-    # smaller ones will be read into memory and then uploaded.
-    transfer.RESUMABLE_UPLOAD_THRESHOLD = 102400
     try:
-        gcs = GoogleCloudStorageBucket(project_id)
-        blob = gcs.bucket.blob('_/' + internal_fname, chunk_size=256 * 1024 * 2)
-        blob.upload_from_file(stream_for_gcs, size=file_size,
-                              content_type=content_type)
+        bucket = GoogleCloudStorageBucket(str(project_id))
+        blob = bucket.blob(internal_fname)
+        blob.create_from_file(stream_for_gcs, file_size=file_size, content_type=content_type)
     except Exception:
         log.exception('Error uploading file to Google Cloud Storage (GCS),'
                       ' aborting handling of uploaded file (id=%s).', file_id)
@@ -750,9 +827,7 @@ def stream_to_gcs(file_id, file_size, internal_fname, project_id,
         raise wz_exceptions.InternalServerError(
             'Unable to stream file to Google Cloud Storage')
 
-    # Reload the blob to get the file size according to Google.
-    blob.reload()
-    return blob, gcs
+    return blob, bucket
 
 
 def add_access_control_headers(resp):
@@ -766,15 +841,6 @@ def add_access_control_headers(resp):
     return resp
 
 
-def update_file_doc(file_id, **updates):
-    files = current_app.data.driver.db['files']
-    res = files.update_one({'_id': ObjectId(file_id)},
-                           {'$set': updates})
-    log.debug('update_file_doc(%s, %s): %i matched, %i updated.',
-              file_id, updates, res.matched_count, res.modified_count)
-    return res
-
-
 def create_file_doc_for_upload(project_id, uploaded_file):
     """Creates a secure filename and a document in MongoDB for the file.
 
@@ -852,10 +918,17 @@ def setup_app(app, url_prefix):
     app.on_fetched_item_files += before_returning_file
     app.on_fetched_resource_files += before_returning_files
 
-    app.on_delete_item_files += before_deleting_file
-
     app.on_update_files += compute_aggregate_length
     app.on_replace_files += compute_aggregate_length
     app.on_insert_files += compute_aggregate_length_items
 
     app.register_api_blueprint(file_storage, url_prefix=url_prefix)
+
+
+def update_file_doc(file_id, **updates):
+    files = current_app.data.driver.db['files']
+    res = files.update_one({'_id': ObjectId(file_id)},
+                           {'$set': updates})
+    log.debug('update_file_doc(%s, %s): %i matched, %i updated.',
+              file_id, updates, res.matched_count, res.modified_count)
+    return res

pillar/api/file_storage/moving.py

@@ -1,20 +1,18 @@
 """Code for moving files between backends."""
 
-import datetime
 import logging
 import os
 import tempfile
 
-from bson import ObjectId
-import bson.tz_util
-from flask import current_app
 import requests
 import requests.exceptions
+from bson import ObjectId
+from flask import current_app
 
+from pillar.api import utils
 from . import stream_to_gcs, generate_all_links, ensure_valid_link
-import pillar.api.utils.gcs
 
-__all__ = ['PrerequisiteNotMetError', 'change_file_storage_backend']
+__all__ = ['PrerequisiteNotMetError', 'change_file_storage_backend', 'move_to_bucket']
 
 log = logging.getLogger(__name__)
 
@@ -29,7 +27,7 @@ def change_file_storage_backend(file_id, dest_backend):
     Files on the original backend are not deleted automatically.
     """
 
-    dest_backend = unicode(dest_backend)
+    dest_backend = str(dest_backend)
     file_id = ObjectId(file_id)
 
     # Fetch file document
@@ -75,8 +73,7 @@ def change_file_storage_backend(file_id, dest_backend):
     # Generate new links for the file & all variations. This also saves
     # the new backend we set here.
     f['backend'] = dest_backend
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
-    generate_all_links(f, now)
+    generate_all_links(f, utils.utcnow())
 
 
 def copy_file_to_backend(file_id, project_id, file_or_var, src_backend, dest_backend):
@@ -90,6 +87,7 @@ def copy_file_to_backend(file_id, project_id, file_or_var, src_backend, dest_bac
     else:
         local_finfo = fetch_file_from_link(file_or_var['link'])
 
+    try:
         # Upload to GCS
         if dest_backend != 'gcs':
             raise ValueError('Only dest_backend="gcs" is supported now.')
@@ -100,10 +98,10 @@ def copy_file_to_backend(file_id, project_id, file_or_var, src_backend, dest_bac
             # TODO check for name collisions
             stream_to_gcs(file_id, local_finfo['file_size'],
                           internal_fname=internal_fname,
-                      project_id=str(project_id),
+                          project_id=project_id,
                           stream_for_gcs=local_finfo['local_file'],
                           content_type=local_finfo['content_type'])
-
+    finally:
         # No longer needed, so it can be closed & dispersed of.
         local_finfo['local_file'].close()
 
@@ -152,29 +150,39 @@ def fetch_file_from_local(file_doc):
     return local_finfo
 
 
-def gcs_move_to_bucket(file_id, dest_project_id, skip_gcs=False):
-    """Moves a file from its own bucket to the new project_id bucket."""
+def move_to_bucket(file_id: ObjectId, dest_project_id: ObjectId, *, skip_storage=False):
+    """Move a file + variations from its own bucket to the new project_id bucket.
 
-    files_coll = current_app.db()['files']
+    :param file_id: ID of the file to move.
+    :param dest_project_id: Project to move to.
+    :param skip_storage: If True, the storage bucket will not be touched.
+        Only use this when you know what you're doing.
+    """
 
+    files_coll = current_app.db('files')
     f = files_coll.find_one(file_id)
     if f is None:
-        raise ValueError('File with _id: {} not found'.format(file_id))
-
-    # Check that new backend differs from current one
-    if f['backend'] != 'gcs':
-        raise ValueError('Only Google Cloud Storage is supported for now.')
+        raise ValueError(f'File with _id: {file_id} not found')
 
     # Move file and variations to the new bucket.
-    if skip_gcs:
-        log.warning('NOT ACTUALLY MOVING file %s on GCS, just updating MongoDB', file_id)
+    if skip_storage:
+        log.warning('NOT ACTUALLY MOVING file %s on storage, just updating MongoDB', file_id)
     else:
-        src_project = f['project']
-        pillar.api.utils.gcs.copy_to_bucket(f['file_path'], src_project, dest_project_id)
+        from pillar.api.file_storage_backends import Bucket
+        bucket_class = Bucket.for_backend(f['backend'])
+        src_bucket = bucket_class(str(f['project']))
+        dst_bucket = bucket_class(str(dest_project_id))
+
+        src_blob = src_bucket.get_blob(f['file_path'])
+        src_bucket.copy_blob(src_blob, dst_bucket)
+
         for var in f.get('variations', []):
-            pillar.api.utils.gcs.copy_to_bucket(var['file_path'], src_project, dest_project_id)
+            src_blob = src_bucket.get_blob(var['file_path'])
+            src_bucket.copy_blob(src_blob, dst_bucket)
 
     # Update the file document after moving was successful.
+    # No need to update _etag or _updated, since that'll be done when
+    # the links are regenerated at the end of this function.
     log.info('Switching file %s to project %s', file_id, dest_project_id)
     update_result = files_coll.update_one({'_id': file_id},
                                           {'$set': {'project': dest_project_id}})
@@ -188,4 +196,4 @@ def gcs_move_to_bucket(file_id, dest_project_id, skip_gcs=False):
 
     # Regenerate the links for this file
     f['project'] = dest_project_id
-    generate_all_links(f, now=datetime.datetime.now(tz=bson.tz_util.utc))
+    generate_all_links(f, now=utils.utcnow())

pillar/api/file_storage_backends/__init__.py

@@ -0,0 +1,29 @@
+"""Storage backends.
+
+To obtain a storage backend, use either of the two forms:
+
+>>> bucket = default_storage_backend('bucket_name')
+
+>>> BucketClass = Bucket.for_backend('backend_name')
+>>> bucket = BucketClass('bucket_name')
+
+"""
+
+from .abstract import Bucket
+
+# Import the other backends so that they register.
+from . import local
+from . import gcs
+
+
+def default_storage_backend(name: str) -> Bucket:
+    """Returns an instance of a Bucket, based on the default backend.
+
+    Depending on the backend this may actually create the bucket.
+    """
+    from flask import current_app
+
+    backend_name = current_app.config['STORAGE_BACKEND']
+    backend_cls = Bucket.for_backend(backend_name)
+
+    return backend_cls(name)

pillar/api/file_storage_backends/abstract.py

@@ -0,0 +1,161 @@
+import abc
+import io
+import logging
+import typing
+
+import pathlib
+from bson import ObjectId
+
+__all__ = ['Bucket', 'Blob', 'Path', 'FileType']
+
+# Shorthand for the type of path we use.
+Path = pathlib.PurePosixPath
+
+# This is a mess: typing.IO keeps mypy-0.501 happy, but not in all cases,
+# and io.FileIO + io.BytesIO keeps PyCharm-2017.1 happy.
+FileType = typing.Union[typing.IO, io.FileIO, io.BytesIO]
+
+
+class Bucket(metaclass=abc.ABCMeta):
+    """Can be a GCS bucket or simply a project folder in Pillar
+
+    :type name: string
+    :param name: Name of the bucket. As a convention, we use the ID of
+    the project to name the bucket.
+
+    """
+
+    # Mapping from backend name to Bucket class
+    backends: typing.Dict[str, typing.Type['Bucket']] = {}
+
+    backend_name: str = None  # define in subclass.
+
+    def __init__(self, name: str) -> None:
+        self.name = str(name)
+
+    def __init_subclass__(cls):
+        assert cls.backend_name, '%s.backend_name must be non-empty string' % cls
+        cls.backends[cls.backend_name] = cls
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__} name={self.name!r}>'
+
+    @classmethod
+    def for_backend(cls, backend_name: str) -> typing.Type['Bucket']:
+        """Returns the Bucket subclass for the given backend."""
+        return cls.backends[backend_name]
+
+    @abc.abstractmethod
+    def blob(self, blob_name: str) -> 'Blob':
+        """Factory constructor for blob object.
+
+        :param blob_name: The path of the blob to be instantiated.
+        """
+
+    @abc.abstractmethod
+    def get_blob(self, blob_name: str) -> typing.Optional['Blob']:
+        """Get a blob object by name.
+
+        If the blob exists return the object, otherwise None.
+        """
+
+    @abc.abstractmethod
+    def copy_blob(self, blob: 'Blob', to_bucket: 'Bucket'):
+        """Copies a blob from the current bucket to the other bucket.
+        
+        Implementations only need to support copying between buckets of the
+        same storage backend.
+        """
+
+    @abc.abstractmethod
+    def rename_blob(self, blob: 'Blob', new_name: str) -> 'Blob':
+        """Rename the blob, returning the new Blob."""
+
+    @classmethod
+    def copy_to_bucket(cls, blob_name, src_project_id: ObjectId, dest_project_id: ObjectId):
+        """Copies a file from one bucket to the other."""
+
+        src_storage = cls(str(src_project_id))
+        dest_storage = cls(str(dest_project_id))
+
+        blob = src_storage.get_blob(blob_name)
+        src_storage.copy_blob(blob, dest_storage)
+
+
+Bu = typing.TypeVar('Bu', bound=Bucket)
+
+
+class Blob(metaclass=abc.ABCMeta):
+    """A wrapper for file or blob objects."""
+
+    def __init__(self, name: str, bucket: Bucket) -> None:
+        self.name = name
+        self.bucket = bucket
+        self._size_in_bytes: typing.Optional[int] = None
+
+        self.filename: str = None
+        """Name of the file for the Content-Disposition header when downloading it."""
+
+        self._log = logging.getLogger(f'{__name__}.Blob')
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__} bucket={self.bucket.name!r} name={self.name!r}>'
+
+    @property
+    def size(self) -> typing.Optional[int]:
+        """Size of the object, in bytes.
+
+        :returns: The size of the blob or ``None`` if the property
+                  is not set locally.
+        """
+
+        size = self._size_in_bytes
+        if size is None:
+            return None
+        return int(size)
+
+    @abc.abstractmethod
+    def create_from_file(self, file_obj: FileType, *,
+                         content_type: str,
+                         file_size: int = -1):
+        """Copies the file object to the storage.
+        
+        :param file_obj: The file object to send to storage.
+        :param content_type: The content type of the file.
+        :param file_size: The size of the file in bytes, or -1 if unknown
+        """
+
+    def upload_from_path(self, path: pathlib.Path, content_type: str):
+        file_size = path.stat().st_size
+
+        with path.open('rb') as infile:
+            self.create_from_file(infile, content_type=content_type,
+                                  file_size=file_size)
+
+    @abc.abstractmethod
+    def update_filename(self, filename: str):
+        """Sets the filename which is used when downloading the file.
+        
+        Not all storage backends support this, and will use the on-disk filename instead.
+        """
+
+    @abc.abstractmethod
+    def get_url(self, *, is_public: bool) -> str:
+        """Returns the URL to access this blob.
+        
+        Note that this may involve API calls to generate a signed URL.
+        """
+
+    @abc.abstractmethod
+    def make_public(self):
+        """Makes the blob publicly available.
+        
+        Only performs an actual action on backends that support temporary links.
+        """
+
+    @abc.abstractmethod
+    def exists(self) -> bool:
+        """Returns True iff the file exists on the storage backend."""
+
+
+Bl = typing.TypeVar('Bl', bound=Blob)

pillar/api/file_storage_backends/gcs.py

@@ -0,0 +1,263 @@
+import os
+import datetime
+import logging
+import typing
+
+from bson import ObjectId
+from gcloud.storage.client import Client
+import gcloud.storage.blob
+import gcloud.exceptions as gcloud_exc
+from flask import current_app, g
+from werkzeug.local import LocalProxy
+
+from pillar.api import utils
+from .abstract import Bucket, Blob, FileType
+
+log = logging.getLogger(__name__)
+
+
+def get_client() -> Client:
+    """Stores the GCS client on the global Flask object.
+
+    The GCS client is not user-specific anyway.
+    """
+
+    _gcs = getattr(g, '_gcs_client', None)
+    if _gcs is None:
+        _gcs = g._gcs_client = Client()
+    return _gcs
+
+
+# This hides the specifics of how/where we store the GCS client,
+# and allows the rest of the code to use 'gcs' as a simple variable
+# that does the right thing.
+gcs: Client = LocalProxy(get_client)
+
+
+class GoogleCloudStorageBucket(Bucket):
+    """Cloud Storage bucket interface. We create a bucket for every project. In
+    the bucket we create first level subdirs as follows:
+    - '_' (will contain hashed assets, and stays on top of default listing)
+    - 'svn' (svn checkout mirror)
+    - 'shared' (any additional folder of static folder that is accessed via a
+      node of 'storage' node_type)
+
+    :type bucket_name: string
+    :param bucket_name: Name of the bucket.
+
+    :type subdir: string
+    :param subdir: The local entry point to browse the bucket.
+
+    """
+
+    backend_name = 'gcs'
+
+    def __init__(self, name: str, subdir='_') -> None:
+        super().__init__(name=name)
+
+        self._log = logging.getLogger(f'{__name__}.GoogleCloudStorageBucket')
+
+        try:
+            self._gcs_bucket = gcs.get_bucket(name)
+        except gcloud_exc.NotFound:
+            self._gcs_bucket = gcs.bucket(name)
+            # Hardcode the bucket location to EU
+            self._gcs_bucket.location = 'EU'
+            # Optionally enable CORS from * (currently only used for vrview)
+            # self.gcs_bucket.cors = [
+            #     {
+            #       "origin": ["*"],
+            #       "responseHeader": ["Content-Type"],
+            #       "method": ["GET", "HEAD", "DELETE"],
+            #       "maxAgeSeconds": 3600
+            #     }
+            # ]
+            self._gcs_bucket.create()
+            log.info('Created GCS instance for project %s', name)
+
+        self.subdir = subdir
+
+    def blob(self, blob_name: str) -> 'GoogleCloudStorageBlob':
+        return GoogleCloudStorageBlob(name=blob_name, bucket=self)
+
+    def get_blob(self, internal_fname: str) -> typing.Optional['GoogleCloudStorageBlob']:
+        blob = self.blob(internal_fname)
+        if not blob.gblob.exists():
+            return None
+        return blob
+
+    def _gcs_get(self, path: str, *, chunk_size=None) -> gcloud.storage.Blob:
+        """Get selected file info if the path matches.
+
+        :param path: The path to the file, relative to the bucket's subdir.
+        """
+        path = os.path.join(self.subdir, path)
+        blob = self._gcs_bucket.blob(path, chunk_size=chunk_size)
+        return blob
+
+    def _gcs_post(self, full_path, *, path=None) -> typing.Optional[gcloud.storage.Blob]:
+        """Create new blob and upload data to it.
+        """
+        path = path if path else os.path.join(self.subdir, os.path.basename(full_path))
+        gblob = self._gcs_bucket.blob(path)
+        if gblob.exists():
+            self._log.error(f'Trying to upload to {path}, but that blob already exists. '
+                            f'Not uploading.')
+            return None
+
+        gblob.upload_from_filename(full_path)
+        return gblob
+        # return self.blob_to_dict(blob) # Has issues with threading
+
+    def delete_blob(self, path: str) -> bool:
+        """Deletes the blob (when removing an asset or replacing a preview)"""
+
+        # We want to get the actual blob to delete
+        gblob = self._gcs_get(path)
+        try:
+            gblob.delete()
+            return True
+        except gcloud_exc.NotFound:
+            return False
+
+    def copy_blob(self, blob: Blob, to_bucket: Bucket):
+        """Copies the given blob from this bucket to the other bucket.
+
+        Returns the new blob.
+        """
+
+        assert isinstance(blob, GoogleCloudStorageBlob)
+        assert isinstance(to_bucket, GoogleCloudStorageBucket)
+
+        self._log.info('Copying %s to bucket %s', blob, to_bucket)
+
+        return self._gcs_bucket.copy_blob(blob.gblob, to_bucket._gcs_bucket)
+
+    def rename_blob(self, blob: 'GoogleCloudStorageBlob', new_name: str) \
+            -> 'GoogleCloudStorageBlob':
+        """Rename the blob, returning the new Blob."""
+
+        assert isinstance(blob, GoogleCloudStorageBlob)
+
+        new_name = os.path.join(self.subdir, new_name)
+
+        self._log.info('Renaming %s to %r', blob, new_name)
+        new_gblob = self._gcs_bucket.rename_blob(blob.gblob, new_name)
+        return GoogleCloudStorageBlob(new_gblob.name, self, gblob=new_gblob)
+
+
+class GoogleCloudStorageBlob(Blob):
+    """GCS blob interface."""
+
+    def __init__(self, name: str, bucket: GoogleCloudStorageBucket,
+                 *, gblob: gcloud.storage.blob.Blob=None) -> None:
+        super().__init__(name, bucket)
+
+        self._log = logging.getLogger(f'{__name__}.GoogleCloudStorageBlob')
+        self.gblob = gblob or bucket._gcs_get(name, chunk_size=256 * 1024 * 2)
+
+    def create_from_file(self, file_obj: FileType, *,
+                         content_type: str,
+                         file_size: int = -1) -> None:
+        from gcloud.streaming import transfer
+
+        self._log.debug('Streaming file to GCS bucket %r, size=%i', self, file_size)
+
+        # Files larger than this many bytes will be streamed directly from disk,
+        # smaller ones will be read into memory and then uploaded.
+        transfer.RESUMABLE_UPLOAD_THRESHOLD = 102400
+        self.gblob.upload_from_file(file_obj,
+                                    size=file_size,
+                                    content_type=content_type)
+
+        # Reload the blob to get the file size according to Google.
+        self.gblob.reload()
+        self._size_in_bytes = self.gblob.size
+
+    def update_filename(self, filename: str):
+        """Set the ContentDisposition metadata so that when a file is downloaded
+        it has a human-readable name.
+        """
+
+        if '"' in filename:
+            raise ValueError(f'Filename is not allowed to have double quote in it: {filename!r}')
+
+        self.gblob.content_disposition = f'attachment; filename="{filename}"'
+        self.gblob.patch()
+
+    def get_url(self, *, is_public: bool) -> str:
+        if is_public:
+            return self.gblob.public_url
+
+        expiration = utils.utcnow() + datetime.timedelta(days=1)
+        return self.gblob.generate_signed_url(expiration)
+
+    def make_public(self):
+        self.gblob.make_public()
+
+    def exists(self) -> bool:
+        # Reload to get the actual file properties from Google.
+        try:
+            self.gblob.reload()
+        except gcloud_exc.NotFound:
+            return False
+        return self.gblob.exists()
+
+
+def update_file_name(node):
+    """Assign to the CGS blob the same name of the asset node. This way when
+    downloading an asset we get a human-readable name.
+    """
+
+    # Process only files that are not processing
+    if node['properties'].get('status', '') == 'processing':
+        return
+
+    def _format_name(name, override_ext, size=None, map_type=''):
+        root, _ = os.path.splitext(name)
+        size = '-{}'.format(size) if size else ''
+        map_type = '-{}'.format(map_type) if map_type else ''
+        return '{}{}{}{}'.format(root, size, map_type, override_ext)
+
+    def _update_name(file_id, file_props):
+        files_collection = current_app.data.driver.db['files']
+        file_doc = files_collection.find_one({'_id': ObjectId(file_id)})
+
+        if file_doc is None or file_doc.get('backend') != 'gcs':
+            return
+
+        # For textures -- the map type should be part of the name.
+        map_type = file_props.get('map_type', '')
+
+        storage = GoogleCloudStorageBucket(str(node['project']))
+        blob = storage.get_blob(file_doc['file_path'])
+        if blob is None:
+            log.warning('Unable to find blob for file %s in project %s',
+                        file_doc['file_path'], file_doc['project'])
+            return
+
+        # Pick file extension from original filename
+        _, ext = os.path.splitext(file_doc['filename'])
+        name = _format_name(node['name'], ext, map_type=map_type)
+        blob.update_filename(name)
+
+        # Assign the same name to variations
+        for v in file_doc.get('variations', []):
+            _, override_ext = os.path.splitext(v['file_path'])
+            name = _format_name(node['name'], override_ext, v['size'], map_type=map_type)
+            blob = storage.get_blob(v['file_path'])
+            if blob is None:
+                log.info('Unable to find blob for file %s in project %s. This can happen if the '
+                         'video encoding is still processing.', v['file_path'], node['project'])
+                continue
+            blob.update_filename(name)
+
+    # Currently we search for 'file' and 'files' keys in the object properties.
+    # This could become a bit more flexible and realy on a true reference of the
+    # file object type from the schema.
+    if 'file' in node['properties']:
+        _update_name(node['properties']['file'], {})
+
+    if 'files' in node['properties']:
+        for file_props in node['properties']['files']:
+            _update_name(file_props['file'], file_props)

pillar/api/file_storage_backends/local.py

@@ -0,0 +1,131 @@
+import logging
+import pathlib
+import typing
+
+from flask import current_app
+
+__all__ = ['LocalBucket', 'LocalBlob']
+
+from .abstract import Bucket, Blob, FileType, Path
+
+
+class LocalBucket(Bucket):
+    backend_name = 'local'
+
+    def __init__(self, name: str) -> None:
+        super().__init__(name)
+
+        self._log = logging.getLogger(f'{__name__}.LocalBucket')
+
+        # For local storage, the name is actually a partial path, relative
+        # to the local storage root.
+        self.root = pathlib.Path(current_app.config['STORAGE_DIR'])
+        self.bucket_path = pathlib.PurePosixPath(self.name[:2]) / self.name
+        self.abspath = self.root / self.bucket_path
+
+    def blob(self, blob_name: str) -> 'LocalBlob':
+        return LocalBlob(name=blob_name, bucket=self)
+
+    def get_blob(self, blob_name: str) -> typing.Optional['LocalBlob']:
+        # TODO: Check if file exists, otherwise None
+        return self.blob(blob_name)
+
+    def copy_blob(self, blob: Blob, to_bucket: Bucket):
+        """Copies a blob from the current bucket to the other bucket.
+        
+        Implementations only need to support copying between buckets of the
+        same storage backend.
+        """
+
+        assert isinstance(blob, LocalBlob)
+        assert isinstance(to_bucket, LocalBucket)
+
+        self._log.info('Copying %s to bucket %s', blob, to_bucket)
+
+        dest_blob = to_bucket.blob(blob.name)
+
+        # TODO: implement content type handling for local storage.
+        self._log.warning('Unable to set correct file content type for %s', dest_blob)
+
+        fpath = blob.abspath()
+        if not fpath.exists():
+            if not fpath.parent.exists():
+                raise FileNotFoundError(f'File {fpath} does not exist, and neither does its parent,'
+                                        f' unable to copy to {to_bucket}')
+            raise FileNotFoundError(f'File {fpath} does not exist, unable to copy to {to_bucket}')
+
+        with open(fpath, 'rb') as src_file:
+            dest_blob.create_from_file(src_file, content_type='application/x-octet-stream')
+
+    def rename_blob(self, blob: 'LocalBlob', new_name: str) -> 'LocalBlob':
+        """Rename the blob, returning the new Blob."""
+
+        assert isinstance(blob, LocalBlob)
+
+        self._log.info('Renaming %s to %r', blob, new_name)
+        new_blob = LocalBlob(new_name, self)
+
+        old_path = blob.abspath()
+        new_path = new_blob.abspath()
+        new_path.parent.mkdir(parents=True, exist_ok=True)
+        old_path.rename(new_path)
+
+        return new_blob
+
+
+class LocalBlob(Blob):
+    """Blob representing a local file on the filesystem."""
+
+    bucket: LocalBucket
+
+    def __init__(self, name: str, bucket: LocalBucket) -> None:
+        super().__init__(name, bucket)
+
+        self._log = logging.getLogger(f'{__name__}.LocalBlob')
+        self.partial_path = Path(name[:2]) / name
+
+    def abspath(self) -> pathlib.Path:
+        """Returns a concrete, absolute path to the local file."""
+
+        return pathlib.Path(self.bucket.abspath / self.partial_path)
+
+    def get_url(self, *, is_public: bool) -> str:
+        from flask import url_for
+
+        path = self.bucket.bucket_path / self.partial_path
+        url = url_for('file_storage.index', file_name=str(path), _external=True,
+                      _scheme=current_app.config['SCHEME'])
+        return url
+
+    def create_from_file(self, file_obj: FileType, *,
+                         content_type: str,
+                         file_size: int = -1):
+        assert hasattr(file_obj, 'read')
+
+        import shutil
+
+        # Ensure path exists before saving
+        my_path = self.abspath()
+        my_path.parent.mkdir(exist_ok=True, parents=True)
+
+        with my_path.open('wb') as outfile:
+            shutil.copyfileobj(typing.cast(typing.IO, file_obj), outfile)
+
+        self._size_in_bytes = file_size
+
+    def update_filename(self, filename: str):
+        # TODO: implement this for local storage.
+        self._log.info('update_filename(%r) not supported', filename)
+
+    def make_public(self):
+        # No-op on this storage backend.
+        pass
+
+    def exists(self) -> bool:
+        return self.abspath().exists()
+
+    def touch(self):
+        """Touch the file, creating parent directories if needed."""
+        path = self.abspath()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.touch(exist_ok=True)

pillar/api/latest.py

@@ -1,5 +1,6 @@
-import itertools
+import typing
 
+import bson
 import pymongo
 from flask import Blueprint, current_app
 
@@ -8,103 +9,85 @@ from pillar.api.utils import jsonify
 blueprint = Blueprint('latest', __name__)
 
 
-def keep_fetching(collection, db_filter, projection, sort, py_filter,
-                  batch_size=12):
-    """Yields results for which py_filter returns True"""
+def _public_project_ids() -> typing.List[bson.ObjectId]:
+    """Returns a list of ObjectIDs of public projects.
 
-    projection['_deleted'] = 1
-    curs = collection.find(db_filter, projection).sort(sort)
-    curs.batch_size(batch_size)
+    Memoized in setup_app().
+    """
 
-    for doc in curs:
-        if doc.get('_deleted'):
-            continue
-        doc.pop('_deleted', None)
-        if py_filter(doc):
-            yield doc
+    proj_coll = current_app.db('projects')
+    result = proj_coll.find({'is_private': False}, {'_id': 1})
+    return [p['_id'] for p in result]
 
 
-def latest_nodes(db_filter, projection, py_filter, limit):
-    nodes = current_app.data.driver.db['nodes']
+def latest_nodes(db_filter, projection, limit):
+    """Returns the latest nodes, of a certain type, of public projects.
+
+    Also includes information about the project and the user of each node.
+    """
 
     proj = {
         '_created': 1,
         '_updated': 1,
+        'user.full_name': 1,
+        'project._id': 1,
+        'project.url': 1,
+        'project.name': 1,
+        'name': 1,
+        'node_type': 1,
+        'parent': 1,
+        **projection,
     }
-    proj.update(projection)
 
-    latest = keep_fetching(nodes, db_filter, proj,
-                           [('_created', pymongo.DESCENDING)],
-                           py_filter, limit)
+    nodes_coll = current_app.db('nodes')
+    pipeline = [
+        {'$match': {'_deleted': {'$ne': True}}},
+        {'$match': db_filter},
+        {'$match': {'project': {'$in': _public_project_ids()}}},
+        {'$sort': {'_created': pymongo.DESCENDING}},
+        {'$limit': limit},
+        {'$lookup': {"from": "users",
+                     "localField": "user",
+                     "foreignField": "_id",
+                     "as": "user"}},
+        {'$unwind': {'path': "$user"}},
+        {'$lookup': {"from": "projects",
+                     "localField": "project",
+                     "foreignField": "_id",
+                     "as": "project"}},
+        {'$unwind': {'path': "$project"}},
+        {'$project': proj},
+    ]
 
-    result = list(itertools.islice(latest, limit))
-    return result
-
-
-def has_public_project(node_doc):
-    """Returns True iff the project the node belongs to is public."""
-
-    project_id = node_doc.get('project')
-    return is_project_public(project_id)
-
-
-# TODO: cache result, for a limited amt. of time, or for this HTTP request.
-def is_project_public(project_id):
-    """Returns True iff the project is public."""
-
-    project = current_app.data.driver.db['projects'].find_one(project_id)
-    if not project:
-        return False
-
-    return not project.get('is_private')
+    latest = nodes_coll.aggregate(pipeline)
+    return list(latest)
 
 
 @blueprint.route('/assets')
 def latest_assets():
     latest = latest_nodes({'node_type': 'asset',
                            'properties.status': 'published'},
-                          {'name': 1, 'project': 1, 'user': 1, 'node_type': 1,
+                          {'name': 1, 'node_type': 1,
                            'parent': 1, 'picture': 1, 'properties.status': 1,
                            'properties.content_type': 1,
                            'permissions.world': 1},
-                          has_public_project, 12)
-
-    embed_user(latest)
-    embed_project(latest)
+                          12)
 
     return jsonify({'_items': latest})
 
 
-def embed_user(latest):
-    users = current_app.data.driver.db['users']
-
-    for comment in latest:
-        user_id = comment['user']
-        comment['user'] = users.find_one(user_id, {
-            'auth': 0, 'groups': 0, 'roles': 0, 'settings': 0, 'email': 0,
-            '_created': 0, '_updated': 0, '_etag': 0})
-
-
-def embed_project(latest):
-    projects = current_app.data.driver.db['projects']
-
-    for comment in latest:
-        project_id = comment['project']
-        comment['project'] = projects.find_one(project_id, {'_id': 1, 'name': 1,
-                                                            'url': 1})
-
-
 @blueprint.route('/comments')
 def latest_comments():
     latest = latest_nodes({'node_type': 'comment',
                            'properties.status': 'published'},
-                          {'project': 1, 'parent': 1, 'user': 1,
+                          {'parent': 1,
                            'properties.content': 1, 'node_type': 1,
                            'properties.status': 1,
                            'properties.is_reply': 1},
-                          has_public_project, 10)
+                          10)
 
     # Embed the comments' parents.
+    # TODO: move to aggregation pipeline.
     nodes = current_app.data.driver.db['nodes']
     parents = {}
     for comment in latest:
@@ -118,11 +101,12 @@ def latest_comments():
         parents[parent_id] = parent
         comment['parent'] = parent
 
-    embed_project(latest)
-    embed_user(latest)
-
     return jsonify({'_items': latest})
 
 
 def setup_app(app, url_prefix):
+    global _public_project_ids
+
     app.register_api_blueprint(blueprint, url_prefix=url_prefix)
+    cached = app.cache.cached(timeout=3600)
+    _public_project_ids = cached(_public_project_ids)

pillar/api/local_auth.py

@@ -1,15 +1,16 @@
 import base64
+import datetime
 import hashlib
 import logging
+import typing
 
 import bcrypt
-import datetime
-import rsa.randnum
-from bson import tz_util
+
 from flask import abort, Blueprint, current_app, jsonify, request
 from pillar.api.utils.authentication import create_new_user_document
 from pillar.api.utils.authentication import make_unique_username
 from pillar.api.utils.authentication import store_token
+from pillar.api.utils import utcnow
 
 blueprint = Blueprint('authentication', __name__)
 log = logging.getLogger(__name__)
@@ -37,17 +38,7 @@ def create_local_user(email, password):
     return r['_id']
 
 
-@blueprint.route('/make-token', methods=['POST'])
-def make_token():
-    """Direct login for a user, without OAuth, using local database. Generates
-    a token that is passed back to Pillar Web and used in subsequent
-    transactions.
-
-    :return: a token string
-    """
-    username = request.form['username']
-    password = request.form['password']
-
+def get_local_user(username, password):
     # Look up user in db
     users_collection = current_app.data.driver.db['users']
     user = users_collection.find_one({'username': username})
@@ -62,35 +53,70 @@ def make_token():
     hashed_password = hash_password(password, salt)
     if hashed_password != credentials['token']:
         return abort(403)
+    return user
+
+
+@blueprint.route('/make-token', methods=['POST'])
+def make_token():
+    """Direct login for a user, without OAuth, using local database. Generates
+    a token that is passed back to Pillar Web and used in subsequent
+    transactions.
+
+    :return: a token string
+    """
+    username = request.form['username']
+    password = request.form['password']
+
+    user = get_local_user(username, password)
 
     token = generate_and_store_token(user['_id'])
     return jsonify(token=token['token'])
 
 
-def generate_and_store_token(user_id, days=15, prefix=''):
+def generate_and_store_token(user_id, days=15, prefix=b'') -> dict:
     """Generates token based on random bits.
 
+    NOTE: the returned document includes the plain-text token.
+    DO NOT STORE OR LOG THIS unless there is a good reason to.
+
     :param user_id: ObjectId of the owning user.
     :param days: token will expire in this many days.
-    :param prefix: the token will be prefixed by this string, for easy identification.
-    :return: the token document.
+    :param prefix: the token will be prefixed by these bytes, for easy identification.
+    :return: the token document with the token in plain text as well as hashed.
     """
 
-    random_bits = rsa.randnum.read_random_bits(256)
+    if not isinstance(prefix, bytes):
+        raise TypeError('prefix must be bytes, not %s' % type(prefix))
+
+    import secrets
+
+    random_bits = secrets.token_bytes(32)
 
     # Use 'xy' as altargs to prevent + and / characters from appearing.
     # We never have to b64decode the string anyway.
-    token = prefix + base64.b64encode(random_bits, altchars='xy').strip('=')
+    token_bytes = prefix + base64.b64encode(random_bits, altchars=b'xy').strip(b'=')
+    token = token_bytes.decode('ascii')
 
-    token_expiry = datetime.datetime.now(tz=tz_util.utc) + datetime.timedelta(days=days)
-    return store_token(user_id, token, token_expiry)
+    token_expiry = utcnow() + datetime.timedelta(days=days)
+    token_data = store_token(user_id, token, token_expiry)
+
+    # Include the token in the returned document so that it can be stored client-side,
+    # in configuration, etc.
+    token_data['token'] = token
+
+    return token_data
 
 
-def hash_password(password, salt):
-    if isinstance(salt, unicode):
+def hash_password(password: str, salt: typing.Union[str, bytes]) -> str:
+    password = password.encode()
+
+    if isinstance(salt, str):
         salt = salt.encode('utf-8')
-    encoded_password = base64.b64encode(hashlib.sha256(password).digest())
-    return bcrypt.hashpw(encoded_password, salt)
+
+    hash = hashlib.sha256(password).digest()
+    encoded_password = base64.b64encode(hash)
+    hashed_password = bcrypt.hashpw(encoded_password, salt)
+    return hashed_password.decode('ascii')
 
 
 def setup_app(app, url_prefix):

pillar/api/node_types/__init__.py

@@ -7,9 +7,9 @@ _file_embedded_schema = {
     }
 }
 
-ATTACHMENT_SLUG_REGEX = '[a-zA-Z0-9_ ]+'
+ATTACHMENT_SLUG_REGEX = r'[a-zA-Z0-9_\-]+'
 
-_attachments_embedded_schema = {
+attachments_embedded_schema = {
     'type': 'dict',
     # TODO: will be renamed to 'keyschema' in Cerberus 1.0
     'propertyschema': {
@@ -40,6 +40,51 @@ _attachments_embedded_schema = {
     },
 }
 
+# TODO (fsiddi) reference this schema in all node_types that allow ratings
+ratings_embedded_schema = {
+    'type': 'dict',
+    # Total count of positive ratings (updated at every rating action)
+    'schema': {
+        'positive': {
+            'type': 'integer',
+        },
+        # Total count of negative ratings (updated at every rating action)
+        'negative': {
+            'type': 'integer',
+        },
+        # Collection of ratings, keyed by user
+        'ratings': {
+            'type': 'list',
+            'schema': {
+                'type': 'dict',
+                'schema': {
+                    'user': {
+                        'type': 'objectid',
+                        'data_relation': {
+                            'resource': 'users',
+                            'field': '_id',
+                            'embeddable': False
+                        }
+                    },
+                    'is_positive': {
+                        'type': 'boolean'
+                    },
+                    # Weight of the rating based on user rep and the context.
+                    # Currently we have the following weights:
+                    # - 1 auto null
+                    # - 2 manual null
+                    # - 3 auto valid
+                    # - 4 manual valid
+                    'weight': {
+                        'type': 'integer'
+                    }
+                }
+            }
+        },
+        'hot': {'type': 'float'},
+    },
+}
+
 # Import after defining the common embedded schemas, to prevent dependency cycles.
 from pillar.api.node_types.asset import node_type_asset
 from pillar.api.node_types.blog import node_type_blog

pillar/api/node_types/asset.py

@@ -1,4 +1,4 @@
-from pillar.api.node_types import _file_embedded_schema, _attachments_embedded_schema
+from pillar.api.node_types import _file_embedded_schema, attachments_embedded_schema
 
 node_type_asset = {
     'name': 'asset',
@@ -27,7 +27,7 @@ node_type_asset = {
         # We point to the original file (and use it to extract any relevant
         # variation useful for our scope).
         'file': _file_embedded_schema,
-        'attachments': _attachments_embedded_schema,
+        'attachments': attachments_embedded_schema,
         # Tags for search
         'tags': {
             'type': 'list',

pillar/api/node_types/blog.py

@@ -2,10 +2,6 @@ node_type_blog = {
     'name': 'blog',
     'description': 'Container for node_type post.',
     'dyn_schema': {
-        # Path for a custom template to be used for rendering the posts
-        'template': {
-            'type': 'string',
-        },
         'categories': {
             'type': 'list',
             'schema': {
@@ -17,5 +13,5 @@ node_type_blog = {
         'categories': {},
         'template': {},
     },
-    'parent': ['project',],
+    'parent': ['project', ],
 }

pillar/api/node_types/comment.py

@@ -2,16 +2,14 @@ node_type_comment = {
     'name': 'comment',
     'description': 'Comments for asset nodes, pages, etc.',
     'dyn_schema': {
-        # The actual comment content (initially Markdown format)
+        # The actual comment content
         'content': {
             'type': 'string',
             'minlength': 5,
             'required': True,
+            'coerce': 'markdown',
         },
-        # The converted-to-HTML content.
-        'content_html': {
-            'type': 'string',
-        },
+        '_content_html': {'type': 'string'},
         'status': {
             'type': 'string',
             'allowed': [

pillar/api/node_types/hdri.py

@@ -7,6 +7,11 @@ node_type_hdri = {
     'description': 'HDR Image',
     'parent': ['group_hdri'],
     'dyn_schema': {
+        # Default yaw angle in degrees.
+        'default_yaw': {
+            'type': 'float',
+            'default': 0.0
+        },
         'status': {
             'type': 'string',
             'allowed': [

pillar/api/node_types/page.py

@@ -1,16 +1,9 @@
-from pillar.api.node_types import _attachments_embedded_schema
+from pillar.api.node_types import attachments_embedded_schema
 
 node_type_page = {
     'name': 'page',
     'description': 'A single page',
     'dyn_schema': {
-        # The page content (Markdown format)
-        'content': {
-            'type': 'string',
-            'minlength': 5,
-            'maxlength': 90000,
-            'required': True
-        },
         'status': {
             'type': 'string',
             'allowed': [
@@ -22,7 +15,7 @@ node_type_page = {
         'url': {
             'type': 'string'
         },
-        'attachments': _attachments_embedded_schema,
+        'attachments': attachments_embedded_schema,
     },
     'form_schema': {
         'attachments': {'visible': False},

pillar/api/node_types/post.py

@@ -1,16 +1,17 @@
-from pillar.api.node_types import _attachments_embedded_schema
+from pillar.api.node_types import attachments_embedded_schema
 
 node_type_post = {
     'name': 'post',
     'description': 'A blog post, for any project',
     'dyn_schema': {
-        # The blogpost content (Markdown format)
         'content': {
             'type': 'string',
             'minlength': 5,
             'maxlength': 90000,
-            'required': True
+            'required': True,
+            'coerce': 'markdown',
         },
+        '_content_html': {'type': 'string'},
         'status': {
             'type': 'string',
             'allowed': [
@@ -26,7 +27,7 @@ node_type_post = {
         'url': {
             'type': 'string'
         },
-        'attachments': _attachments_embedded_schema,
+        'attachments': attachments_embedded_schema,
     },
     'form_schema': {
         'attachments': {'visible': False},

pillar/api/node_types/storage.py

@@ -16,7 +16,7 @@ node_type_storage = {
         'subdir': {
             'type': 'string',
         },
-        # Which backend is used to store the files (gcs, pillar, bam, cdnsun)
+        # Which backend is used to store the files (gcs, local)
         'backend': {
             'type': 'string',
         },

pillar/api/node_types/texture.py

@@ -27,13 +27,19 @@ node_type_texture = {
                     'map_type': {
                         'type': 'string',
                         'allowed': [
-                            'color',
-                            'specular',
-                            'bump',
-                            'normal',
-                            'translucency',
-                            'emission',
-                            'alpha'
+                            "alpha",
+                            "ambient occlusion",
+                            "bump",
+                            "color",
+                            "displacement",
+                            "emission",
+                            "glossiness",
+                            "id",
+                            "mask",
+                            "normal",
+                            "roughness",
+                            "specular",
+                            "translucency",
                     ]}
                 }
             }

pillar/api/nodes/__init__.py

@@ -1,26 +1,22 @@
 import base64
 import functools
 import logging
-import urlparse
+import urllib.parse
 
 import pymongo.errors
-import rsa.randnum
 import werkzeug.exceptions as wz_exceptions
 from bson import ObjectId
-from flask import current_app, g, Blueprint, request
+from flask import current_app, Blueprint, request
 
-import pillar.markdown
-from pillar.api.node_types import PILLAR_NAMED_NODE_TYPES
 from pillar.api.activities import activity_subscribe, activity_object_add
-from pillar.api.utils.algolia import algolia_index_node_delete
-from pillar.api.utils.algolia import algolia_index_node_save
+from pillar.api.node_types import PILLAR_NAMED_NODE_TYPES
+from pillar.api.file_storage_backends.gcs import update_file_name
 from pillar.api.utils import str2id, jsonify
 from pillar.api.utils.authorization import check_permissions, require_login
-from pillar.api.utils.gcs import update_file_name
 
 log = logging.getLogger(__name__)
 blueprint = Blueprint('nodes_api', __name__)
-ROLES_FOR_SHARING = {u'subscriber', u'demo'}
+ROLES_FOR_SHARING = {'subscriber', 'demo'}
 
 
 def only_for_node_type_decorator(*required_node_type_names):
@@ -138,7 +134,7 @@ def make_world_gettable(node):
     log.debug('Ensuring the world can read node %s', node_id)
 
     world_perms = set(node.get('permissions', {}).get('world', []))
-    world_perms.add(u'GET')
+    world_perms.add('GET')
     world_perms = list(world_perms)
 
     result = nodes_coll.update_one({'_id': node_id},
@@ -150,13 +146,19 @@ def make_world_gettable(node):
                                                 node_id)
 
 
-def create_short_code(node):
+def create_short_code(node) -> str:
     """Generates a new 'short code' for the node."""
 
+    import secrets
+
     length = current_app.config['SHORT_CODE_LENGTH']
-    bits = rsa.randnum.read_random_bits(32)
-    short_code = base64.b64encode(bits, altchars='xy').rstrip('=')
-    short_code = short_code[:length]
+
+    # Base64 encoding will expand it a bit, so we'll cut that off later.
+    # It's a good idea to start with enough bytes, though.
+    bits = secrets.token_bytes(length)
+
+    short_code = base64.b64encode(bits, altchars=b'xy').rstrip(b'=')
+    short_code = short_code[:length].decode('ascii')
 
     return short_code
 
@@ -164,7 +166,8 @@ def create_short_code(node):
 def short_link_info(short_code):
     """Returns the short link info in a dict."""
 
-    short_link = urlparse.urljoin(current_app.config['SHORT_LINK_BASE_URL'], short_code)
+    short_link = urllib.parse.urljoin(
+        current_app.config['SHORT_LINK_BASE_URL'], short_code)
 
     return {
         'short_code': short_code,
@@ -182,33 +185,29 @@ def after_replacing_node(item, original):
     project is private, prevent public indexing.
     """
 
+    from pillar.celery import search_index_tasks as index
+
     projects_collection = current_app.data.driver.db['projects']
     project = projects_collection.find_one({'_id': item['project']})
     if project.get('is_private', False):
         # Skip index updating and return
         return
 
-    from algoliasearch.client import AlgoliaException
     status = item['properties'].get('status', 'unpublished')
+    node_id = str(item['_id'])
 
     if status == 'published':
-        try:
-            algolia_index_node_save(item)
-        except AlgoliaException as ex:
-            log.warning('Unable to push node info to Algolia for node %s; %s',
-                        item.get('_id'), ex)
+        index.node_save.delay(node_id)
     else:
-        try:
-            algolia_index_node_delete(item)
-        except AlgoliaException as ex:
-            log.warning('Unable to delete node info to Algolia for node %s; %s',
-                        item.get('_id'), ex)
+        index.node_delete.delay(node_id)
 
 
 def before_inserting_nodes(items):
     """Before inserting a node in the collection we check if the user is allowed
     and we append the project id to it.
     """
+    from pillar.auth import current_user
+
     nodes_collection = current_app.data.driver.db['nodes']
 
     def find_parent_project(node):
@@ -230,7 +229,7 @@ def before_inserting_nodes(items):
                 item['project'] = project['_id']
 
         # Default the 'user' property to the current user.
-        item.setdefault('user', g.current_user['user_id'])
+        item.setdefault('user', current_user.user_id)
 
 
 def after_inserting_nodes(items):
@@ -349,7 +348,7 @@ def node_set_default_picture(node, original=None):
         # Find the colour map, defaulting to the first image map available.
         image_file_id = None
         for image in props.get('files', []):
-            if image_file_id is None or image.get('map_type') == u'color':
+            if image_file_id is None or image.get('map_type') == 'color':
                 image_file_id = image.get('file')
     else:
         log.debug('Not setting default picture on node type %s content type %s',
@@ -369,37 +368,36 @@ def nodes_set_default_picture(nodes):
         node_set_default_picture(node)
 
 
+def before_deleting_node(node: dict):
+    check_permissions('nodes', node, 'DELETE')
+
+
 def after_deleting_node(item):
-    from algoliasearch.client import AlgoliaException
-    try:
-        algolia_index_node_delete(item)
-    except AlgoliaException as ex:
-        log.warning('Unable to delete node info to Algolia for node %s; %s',
-                    item.get('_id'), ex)
+    from pillar.celery import search_index_tasks as index
+    index.node_delete.delay(str(item['_id']))
 
 
-only_for_comments = only_for_node_type_decorator('comment')
+only_for_textures = only_for_node_type_decorator('texture')
 
 
-@only_for_comments
-def convert_markdown(node, original=None):
-    """Converts comments from Markdown to HTML.
-
-    Always does this on save, even when the original Markdown hasn't changed,
-    because our Markdown -> HTML conversion rules might have.
-    """
+@only_for_textures
+def texture_sort_files(node, original=None):
+    """Sort files alphabetically by map type, with colour map first."""
 
     try:
-        content = node['properties']['content']
+        files = node['properties']['files']
     except KeyError:
-        node['properties']['content_html'] = ''
-    else:
-        node['properties']['content_html'] = pillar.markdown.markdown(content)
+        return
+
+    # Sort the map types alphabetically, ensuring 'color' comes first.
+    as_dict = {f['map_type']: f for f in files}
+    types = sorted(as_dict.keys(), key=lambda k: '\0' if k == 'color' else k)
+    node['properties']['files'] = [as_dict[map_type] for map_type in types]
 
 
-def nodes_convert_markdown(nodes):
+def textures_sort_files(nodes):
     for node in nodes:
-        convert_markdown(node)
+        texture_sort_files(node)
 
 
 def setup_app(app, url_prefix):
@@ -410,7 +408,7 @@ def setup_app(app, url_prefix):
     app.on_fetched_resource_nodes += before_returning_nodes
 
     app.on_replace_nodes += before_replacing_node
-    app.on_replace_nodes += convert_markdown
+    app.on_replace_nodes += texture_sort_files
     app.on_replace_nodes += deduct_content_type
     app.on_replace_nodes += node_set_default_picture
     app.on_replaced_nodes += after_replacing_node
@@ -418,11 +416,12 @@ def setup_app(app, url_prefix):
     app.on_insert_nodes += before_inserting_nodes
     app.on_insert_nodes += nodes_deduct_content_type
     app.on_insert_nodes += nodes_set_default_picture
-    app.on_insert_nodes += nodes_convert_markdown
+    app.on_insert_nodes += textures_sort_files
     app.on_inserted_nodes += after_inserting_nodes
 
-    app.on_update_nodes += convert_markdown
+    app.on_update_nodes += texture_sort_files
 
+    app.on_delete_item_nodes += before_deleting_node
     app.on_deleted_item_nodes += after_deleting_node
 
     app.register_api_blueprint(blueprint, url_prefix=url_prefix)

pillar/api/nodes/custom/comment.py

@@ -2,7 +2,6 @@
 
 import logging
 
-from eve.methods.patch import patch_internal
 from flask import current_app
 import werkzeug.exceptions as wz_exceptions
 
@@ -11,20 +10,19 @@ from pillar.api.utils import authorization, authentication, jsonify
 from . import register_patch_handler
 
 log = logging.getLogger(__name__)
-ROLES_FOR_COMMENT_VOTING = {u'subscriber', u'demo'}
-COMMENT_VOTING_OPS = {u'upvote', u'downvote', u'revoke'}
-VALID_COMMENT_OPERATIONS = COMMENT_VOTING_OPS.union({u'edit'})
+COMMENT_VOTING_OPS = {'upvote', 'downvote', 'revoke'}
+VALID_COMMENT_OPERATIONS = COMMENT_VOTING_OPS.union({'edit'})
 
 
-@register_patch_handler(u'comment')
+@register_patch_handler('comment')
 def patch_comment(node_id, patch):
     assert_is_valid_patch(node_id, patch)
     user_id = authentication.current_user_id()
 
-    if patch[u'op'] in COMMENT_VOTING_OPS:
+    if patch['op'] in COMMENT_VOTING_OPS:
         result, node = vote_comment(user_id, node_id, patch)
     else:
-        assert patch[u'op'] == u'edit', 'Invalid patch operation %s' % patch[u'op']
+        assert patch['op'] == 'edit', 'Invalid patch operation %s' % patch['op']
         result, node = edit_comment(user_id, node_id, patch)
 
     return jsonify({'_status': 'OK',
@@ -43,11 +41,15 @@ def vote_comment(user_id, node_id, patch):
                   '$or': [{'properties.ratings.$.user': {'$exists': False}},
                           {'properties.ratings.$.user': user_id}]}
     node = nodes_coll.find_one(node_query,
-                               projection={'properties': 1})
+                               projection={'properties': 1, 'user': 1})
     if node is None:
         log.warning('User %s wanted to patch non-existing node %s' % (user_id, node_id))
         raise wz_exceptions.NotFound('Node %s not found' % node_id)
 
+    # We don't allow the user to down/upvote their own nodes.
+    if user_id == node['user']:
+        raise wz_exceptions.Forbidden('You cannot vote on your own node')
+
     props = node['properties']
 
     # Find the current rating (if any)
@@ -95,9 +97,9 @@ def vote_comment(user_id, node_id, patch):
         return update
 
     actions = {
-        u'upvote': upvote,
-        u'downvote': downvote,
-        u'revoke': revoke,
+        'upvote': upvote,
+        'downvote': downvote,
+        'revoke': revoke,
     }
     action = actions[patch['op']]
     mongo_update = action()
@@ -141,11 +143,11 @@ def edit_comment(user_id, node_id, patch):
         log.warning('User %s wanted to patch non-existing node %s' % (user_id, node_id))
         raise wz_exceptions.NotFound('Node %s not found' % node_id)
 
-    if node['user'] != user_id and not authorization.user_has_role(u'admin'):
+    if node['user'] != user_id and not authorization.user_has_role('admin'):
         raise wz_exceptions.Forbidden('You can only edit your own comments.')
 
     # Use Eve to PATCH this node, as that also updates the etag.
-    r, _, _, status = patch_internal('nodes',
+    r, _, _, status = current_app.patch_internal('nodes',
                                                  {'properties.content': patch['content'],
                                                   'project': node['project'],
                                                   'user': node['user'],
@@ -160,7 +162,10 @@ def edit_comment(user_id, node_id, patch):
         log.info('User %s edited comment %s', user_id, node_id)
 
     # Fetch the new content, so the client can show these without querying again.
-    node = nodes_coll.find_one(node_id, projection={'properties.content_html': 1})
+    node = nodes_coll.find_one(node_id, projection={
+        'properties.content': 1,
+        'properties._content_html': 1,
+    })
     return status, node
 
 
@@ -173,15 +178,15 @@ def assert_is_valid_patch(node_id, patch):
         raise wz_exceptions.BadRequest("PATCH should have a key 'op' indicating the operation.")
 
     if op not in VALID_COMMENT_OPERATIONS:
-        raise wz_exceptions.BadRequest(u'Operation should be one of %s',
-                                       u', '.join(VALID_COMMENT_OPERATIONS))
+        raise wz_exceptions.BadRequest('Operation should be one of %s',
+                                       ', '.join(VALID_COMMENT_OPERATIONS))
 
     if op not in COMMENT_VOTING_OPS:
         # We can't check here, we need the node owner for that.
         return
 
     # See whether the user is allowed to patch
-    if authorization.user_matches_roles(ROLES_FOR_COMMENT_VOTING):
+    if authorization.user_matches_roles(current_app.config['ROLES_FOR_COMMENT_VOTING']):
         log.debug('User is allowed to upvote/downvote comment')
         return
 

pillar/api/nodes/moving.py

@@ -61,8 +61,8 @@ class NodeMover(object):
         """Moves a single file to another project"""
 
         self._log.info('Moving file %s to project %s', file_id, dest_proj['_id'])
-        pillar.api.file_storage.moving.gcs_move_to_bucket(file_id, dest_proj['_id'],
-                                                          skip_gcs=self.skip_gcs)
+        pillar.api.file_storage.moving.move_to_bucket(file_id, dest_proj['_id'],
+                                                      skip_storage=self.skip_gcs)
 
     def _files(self, file_ref, *properties):
         """Yields file ObjectIDs."""

pillar/api/organizations/__init__.py

@@ -0,0 +1,444 @@
+"""Organization management.
+
+Assumes role names that are given to users by organization membership
+start with the string "org-".
+"""
+
+import logging
+import typing
+
+import attr
+import bson
+import flask
+import werkzeug.exceptions as wz_exceptions
+
+from pillar import attrs_extra, current_app
+from pillar.api.utils import remove_private_keys, utcnow
+
+
+class OrganizationError(Exception):
+    """Superclass for all Organization-related errors."""
+
+
+@attr.s
+class NotEnoughSeats(OrganizationError):
+    """Thrown when trying to add too many members to the organization."""
+
+    org_id = attr.ib(validator=attr.validators.instance_of(bson.ObjectId))
+    seat_count = attr.ib(validator=attr.validators.instance_of(int))
+    attempted_seat_count = attr.ib(validator=attr.validators.instance_of(int))
+
+
+@attr.s
+class OrgManager:
+    """Organization manager.
+
+    Performs actions on an Organization. Does *NOT* test user permissions -- the caller
+    is responsible for that.
+    """
+
+    _log = attrs_extra.log('%s.OrgManager' % __name__)
+
+    def create_new_org(self,
+                       name: str,
+                       admin_uid: bson.ObjectId,
+                       seat_count: int,
+                       *,
+                       org_roles: typing.Iterable[str] = None) -> dict:
+        """Creates a new Organization.
+
+        Returns the new organization document.
+        """
+
+        assert isinstance(admin_uid, bson.ObjectId)
+
+        org_doc = {
+            'name': name,
+            'admin_uid': admin_uid,
+            'seat_count': seat_count,
+        }
+
+        if org_roles:
+            org_doc['org_roles'] = list(org_roles)
+
+        r, _, _, status = current_app.post_internal('organizations', org_doc)
+        if status != 201:
+            self._log.error('Error creating organization; status should be 201, not %i: %s',
+                            status, r)
+            raise ValueError(f'Unable to create organization, status code {status}')
+
+        org_doc.update(r)
+        return org_doc
+
+    def assign_users(self,
+                     org_id: bson.ObjectId,
+                     emails: typing.List[str]) -> dict:
+        """Assigns users to the organization.
+
+        Checks the seat count and throws a NotEnoughSeats exception when the
+        seat count is not sufficient to assign the requested users.
+
+        Users are looked up by email address, and known users are
+        automatically mapped.
+
+        :returns: the new organization document.
+        """
+
+        self._log.info('Adding %i new members to organization %s', len(emails), org_id)
+
+        users_coll = current_app.db('users')
+        existing_user_docs = list(users_coll.find({'email': {'$in': emails}},
+                                                  projection={'_id': 1, 'email': 1}))
+        unknown_users = set(emails) - {user['email'] for user in existing_user_docs}
+        existing_users = {user['_id'] for user in existing_user_docs}
+
+        return self._assign_users(org_id, unknown_users, existing_users)
+
+    def assign_single_user(self, org_id: bson.ObjectId, *, user_id: bson.ObjectId) -> dict:
+        """Assigns a single, known user to the organization.
+
+        :returns: the new organization document.
+        """
+
+        self._log.info('Adding new member %s to organization %s', user_id, org_id)
+        return self._assign_users(org_id, set(), {user_id})
+
+    def _assign_users(self, org_id: bson.ObjectId,
+                      unknown_users: typing.Set[str],
+                      existing_users: typing.Set[bson.ObjectId]) -> dict:
+
+        if self._log.isEnabledFor(logging.INFO):
+            self._log.info('  - found users: %s', ', '.join(str(uid) for uid in existing_users))
+            self._log.info('  - unknown users: %s', ', '.join(unknown_users))
+
+        org_doc = self._get_org(org_id)
+
+        # Compute the new members.
+        members = set(org_doc.get('members') or []) | existing_users
+        unknown_members = set(org_doc.get('unknown_members') or []) | unknown_users
+
+        # Make sure we don't exceed the current seat count.
+        new_seat_count = len(members) + len(unknown_members)
+        if new_seat_count > org_doc['seat_count']:
+            self._log.warning('assign_users(%s, ...): Trying to increase seats to %i, '
+                              'but org only has %i seats.',
+                              org_id, new_seat_count, org_doc['seat_count'])
+            raise NotEnoughSeats(org_id, org_doc['seat_count'], new_seat_count)
+
+        # Update the organization.
+        org_doc['members'] = list(members)
+        org_doc['unknown_members'] = list(unknown_members)
+
+        r, _, _, status = current_app.put_internal('organizations',
+                                                   remove_private_keys(org_doc),
+                                                   _id=org_id)
+        if status != 200:
+            self._log.error('Error updating organization; status should be 200, not %i: %s',
+                            status, r)
+            raise ValueError(f'Unable to update organization, status code {status}')
+        org_doc.update(r)
+
+        # Update the roles for the affected members
+        for uid in existing_users:
+            self.refresh_roles(uid)
+
+        return org_doc
+
+    def assign_admin(self, org_id: bson.ObjectId, *, user_id: bson.ObjectId):
+        """Assigns a user as admin user for this organization."""
+
+        assert isinstance(org_id, bson.ObjectId)
+        assert isinstance(user_id, bson.ObjectId)
+
+        org_coll = current_app.db('organizations')
+        users_coll = current_app.db('users')
+
+        if users_coll.count({'_id': user_id}) == 0:
+            raise ValueError('User not found')
+
+        self._log.info('Updating organization %s, setting admin user to %s', org_id, user_id)
+        org_coll.update_one({'_id': org_id},
+                            {'$set': {'admin_uid': user_id}})
+
+    def remove_user(self,
+                    org_id: bson.ObjectId,
+                    *,
+                    user_id: bson.ObjectId = None,
+                    email: str = None) -> dict:
+        """Removes a user from the organization.
+
+        The user can be identified by either user ID or email.
+
+        Returns the new organization document.
+        """
+
+        users_coll = current_app.db('users')
+
+        assert user_id or email
+
+        # Collect the email address if not given. This ensures the removal
+        # if the email was accidentally in the unknown_members list.
+        if email is None:
+            user_doc = users_coll.find_one(user_id, projection={'email': 1})
+            if user_doc is not None:
+                email = user_doc['email']
+
+        # See if we know this user.
+        if user_id is None:
+            user_doc = users_coll.find_one({'email': email}, projection={'_id': 1})
+            if user_doc is not None:
+                user_id = user_doc['_id']
+
+        if user_id and not users_coll.count({'_id': user_id}):
+            raise wz_exceptions.UnprocessableEntity('User does not exist')
+
+        self._log.info('Removing user %s / %s from organization %s', user_id, email, org_id)
+
+        org_doc = self._get_org(org_id)
+
+        # Compute the new members.
+        if user_id:
+            members = set(org_doc.get('members') or []) - {user_id}
+            org_doc['members'] = list(members)
+
+        if email:
+            unknown_members = set(org_doc.get('unknown_members')) - {email}
+            org_doc['unknown_members'] = list(unknown_members)
+
+        r, _, _, status = current_app.put_internal('organizations',
+                                                   remove_private_keys(org_doc),
+                                                   _id=org_id)
+        if status != 200:
+            self._log.error('Error updating organization; status should be 200, not %i: %s',
+                            status, r)
+            raise ValueError(f'Unable to update organization, status code {status}')
+        org_doc.update(r)
+
+        # Update the roles for the affected member.
+        if user_id:
+            self.refresh_roles(user_id)
+
+        return org_doc
+
+    def _get_org(self, org_id: bson.ObjectId, *, projection=None):
+        """Returns the organization, or raises a ValueError."""
+
+        assert isinstance(org_id, bson.ObjectId)
+
+        org_coll = current_app.db('organizations')
+        org = org_coll.find_one(org_id, projection=projection)
+        if org is None:
+            raise ValueError(f'Organization {org_id} not found')
+        return org
+
+    def refresh_all_user_roles(self, org_id: bson.ObjectId):
+        """Refreshes the roles of all members."""
+
+        assert isinstance(org_id, bson.ObjectId)
+
+        org = self._get_org(org_id, projection={'members': 1})
+        members = org.get('members')
+        if not members:
+            self._log.info('Organization %s has no members, nothing to refresh.', org_id)
+            return
+
+        for uid in members:
+            self.refresh_roles(uid)
+
+    def refresh_roles(self, user_id: bson.ObjectId) -> typing.Set[str]:
+        """Refreshes the user's roles to own roles + organizations' roles.
+
+        :returns: the applied set of roles.
+        """
+
+        assert isinstance(user_id, bson.ObjectId)
+
+        from pillar.api.service import do_badger
+
+        self._log.info('Refreshing roles for user %s', user_id)
+
+        org_coll = current_app.db('organizations')
+        tokens_coll = current_app.db('tokens')
+
+        def aggr_roles(coll, match: dict) -> typing.Set[str]:
+            query = coll.aggregate([
+                {'$match': match},
+                {'$project': {'org_roles': 1}},
+                {'$unwind': {'path': '$org_roles'}},
+                {'$group': {
+                    '_id': None,
+                    'org_roles': {'$addToSet': '$org_roles'},
+                }}])
+
+            # If the user has no organizations/tokens at all, the query will have no results.
+            try:
+                org_roles_doc = query.next()
+            except StopIteration:
+                return set()
+            return set(org_roles_doc['org_roles'])
+
+        # Join all organization-given roles and roles from the tokens collection.
+        org_roles = aggr_roles(org_coll, {'members': user_id})
+        self._log.debug('Organization-given roles for user %s: %s', user_id, org_roles)
+        token_roles = aggr_roles(tokens_coll, {
+            'user': user_id,
+            'expire_time': {"$gt": utcnow()},
+        })
+        self._log.debug('Token-given roles for user %s: %s', user_id, token_roles)
+        org_roles.update(token_roles)
+
+        users_coll = current_app.db('users')
+        user_doc = users_coll.find_one(user_id, projection={'roles': 1})
+        if not user_doc:
+            self._log.warning('Trying refresh roles of non-existing user %s, ignoring', user_id)
+            return set()
+
+        all_user_roles = set(user_doc.get('roles') or [])
+        existing_org_roles = {role for role in all_user_roles
+                              if role.startswith('org-')}
+
+        grant_roles = org_roles - all_user_roles
+        revoke_roles = existing_org_roles - org_roles
+
+        if grant_roles:
+            do_badger('grant', roles=grant_roles, user_id=user_id)
+        if revoke_roles:
+            do_badger('revoke', roles=revoke_roles, user_id=user_id)
+
+        return all_user_roles.union(grant_roles) - revoke_roles
+
+    def user_is_admin(self, org_id: bson.ObjectId) -> bool:
+        """Returns whether the currently logged in user is the admin of the organization."""
+
+        from pillar.api.utils.authentication import current_user_id
+
+        uid = current_user_id()
+        if uid is None:
+            return False
+
+        org = self._get_org(org_id, projection={'admin_uid': 1})
+        return org.get('admin_uid') == uid
+
+    def unknown_member_roles(self, member_email: str) -> typing.Set[str]:
+        """Returns the set of organization roles for this user.
+
+        Assumes the user is not yet known, i.e. part of the unknown_members lists.
+        """
+
+        org_coll = current_app.db('organizations')
+
+        # Aggregate all org-given roles for this user.
+        query = org_coll.aggregate([
+            {'$match': {'unknown_members': member_email}},
+            {'$project': {'org_roles': 1}},
+            {'$unwind': {'path': '$org_roles'}},
+            {'$group': {
+                '_id': None,
+                'org_roles': {'$addToSet': '$org_roles'},
+            }}])
+
+        # If the user has no organizations at all, the query will have no results.
+        try:
+            org_roles_doc = query.next()
+        except StopIteration:
+            return set()
+
+        return set(org_roles_doc['org_roles'])
+
+    def make_member_known(self, member_uid: bson.ObjectId, member_email: str):
+        """Moves the given member from the unknown_members to the members lists."""
+
+        # This uses a direct PyMongo query rather than using Eve's put_internal,
+        # to prevent simultaneous updates from dropping users.
+
+        org_coll = current_app.db('organizations')
+        for org in org_coll.find({'unknown_members': member_email}):
+            self._log.info('Updating organization %s, marking member %s/%s as known',
+                           org['_id'], member_uid, member_email)
+            org_coll.update_one({'_id': org['_id']},
+                                {'$addToSet': {'members': member_uid},
+                                 '$pull': {'unknown_members': member_email}
+                                 })
+
+    def org_members(self, member_sting_ids: typing.Iterable[str]) -> typing.List[dict]:
+        """Returns the user documents of the organization members.
+
+        This is a workaround to provide membership information for
+        organizations without giving 'mortal' users access to /api/users.
+        """
+        from pillar.api.utils import str2id
+
+        if not member_sting_ids:
+            return []
+
+        member_ids = [str2id(uid) for uid in member_sting_ids]
+        users_coll = current_app.db('users')
+        users = users_coll.find({'_id': {'$in': member_ids}},
+                                projection={'_id': 1, 'full_name': 1, 'email': 1})
+        return list(users)
+
+    def user_has_organizations(self, user_id: bson.ObjectId) -> bool:
+        """Returns True iff the user has anything to do with organizations.
+
+        That is, if the user is admin for and/or member of any organization.
+        """
+
+        org_coll = current_app.db('organizations')
+
+        org_count = org_coll.count({'$or': [
+            {'admin_uid': user_id},
+            {'members': user_id}
+        ]})
+
+        return bool(org_count)
+
+    def user_is_unknown_member(self, member_email: str) -> bool:
+        """Return True iff the email is an unknown member of some org."""
+
+        org_coll = current_app.db('organizations')
+        org_count = org_coll.count({'unknown_members': member_email})
+        return bool(org_count)
+
+    def roles_for_ip_address(self, remote_addr: str) -> typing.Set[str]:
+        """Find the roles given to the user via org IP range definitions."""
+
+        from . import ip_ranges
+
+        org_coll = current_app.db('organizations')
+        try:
+            q = ip_ranges.query(remote_addr)
+        except ValueError as ex:
+            self._log.warning('Invalid remote address %s, ignoring IP-based roles: %s',
+                              remote_addr, ex)
+            return set()
+
+        orgs = org_coll.find(
+            {'ip_ranges': q},
+            projection={'org_roles': True},
+        )
+        return set(role
+                   for org in orgs
+                   for role in org.get('org_roles', []))
+
+    def roles_for_request(self) -> typing.Set[str]:
+        """Find roles for user via the request's remote IP address."""
+
+        try:
+            remote_addr = flask.request.access_route[0]
+        except IndexError:
+            return set()
+
+        if not remote_addr:
+            return set()
+
+        roles = self.roles_for_ip_address(remote_addr)
+        self._log.debug('Roles for IP address %s: %s', remote_addr, roles)
+
+        return roles
+
+
+def setup_app(app):
+    from . import patch, hooks
+
+    hooks.setup_app(app)
+    patch.setup_app(app)

pillar/api/organizations/hooks.py

@@ -0,0 +1,48 @@
+import werkzeug.exceptions as wz_exceptions
+
+from pillar.api.utils.authentication import current_user
+
+
+def pre_get_organizations(request, lookup):
+    user = current_user()
+    if user.is_anonymous:
+        raise wz_exceptions.Forbidden()
+
+    if user.has_cap('admin'):
+        # Allow all lookups to admins.
+        return
+
+    # Only allow users to see their own organizations.
+    lookup['$or'] = [{'admin_uid': user.user_id}, {'members': user.user_id}]
+
+
+def on_fetched_item_organizations(org_doc: dict):
+    """Filter out binary data.
+
+    Eve cannot return binary data, at least not until we upgrade to a version
+    that depends on Cerberus >= 1.0.
+    """
+
+    for ipr in org_doc.get('ip_ranges') or []:
+        ipr.pop('start', None)
+        ipr.pop('end', None)
+        ipr.pop('prefix', None)  # not binary, but useless without the other fields.
+
+
+def on_fetched_resource_organizations(response: dict):
+    for org_doc in response.get('_items', []):
+        on_fetched_item_organizations(org_doc)
+
+
+def pre_post_organizations(request):
+    user = current_user()
+    if not user.has_cap('create-organization'):
+        raise wz_exceptions.Forbidden()
+
+
+def setup_app(app):
+    app.on_pre_GET_organizations += pre_get_organizations
+    app.on_pre_POST_organizations += pre_post_organizations
+
+    app.on_fetched_item_organizations += on_fetched_item_organizations
+    app.on_fetched_resource_organizations += on_fetched_resource_organizations

pillar/api/organizations/ip_ranges.py

@@ -0,0 +1,75 @@
+"""IP range support for Organizations."""
+
+from IPy import IP
+
+# 128 bits all set to 1
+ONES_128 = 2 ** 128 - 1
+
+
+def doc(iprange: str, min_prefixlen6: int=0, min_prefixlen4: int=0) -> dict:
+    """Convert a human-readable string like '1.2.3.4/24' to a Mongo document.
+
+    This converts the address to IPv6 and computes the start/end addresses
+    of the range. The address, its prefix size, and start and end address,
+    are returned as a dict.
+
+    Addresses are stored as big-endian binary data because MongoDB doesn't
+    support 128 bits integers.
+
+    :param iprange: the IP address and mask size, can be IPv6 or IPv4.
+    :param min_prefixlen6: if given, causes a ValuError when the mask size
+                           is too low. Note that the mask size is always
+                           evaluated only for IPv6 addresses.
+    :param min_prefixlen4: if given, causes a ValuError when the mask size
+                           is too low. Note that the mask size is always
+                           evaluated only for IPv4 addresses.
+    :returns: a dict like: {
+        'start': b'xxxxx' with the lowest IP address in the range.
+        'end': b'yyyyy' with the highest IP address in the range.
+        'human': 'aaaa:bbbb::cc00/120' with the human-readable representation.
+        'prefix': 120, the prefix length of the netmask in bits.
+    }
+    """
+
+    ip = IP(iprange, make_net=True)
+    prefixlen = ip.prefixlen()
+    if ip.version() == 4:
+        if prefixlen < min_prefixlen4:
+            raise ValueError(f'Prefix length {prefixlen} smaller than allowed {min_prefixlen4}')
+        ip = ip.v46map()
+    else:
+        if prefixlen < min_prefixlen6:
+            raise ValueError(f'Prefix length {prefixlen} smaller than allowed {min_prefixlen6}')
+
+    addr = ip.int()
+
+    # Set all address bits to 1 where the mask is 0 to obtain the largest address.
+    end = addr | (ONES_128 % ip.netmask().int())
+
+    # This ensures that even a single host is represented as /128 in the human-readable form.
+    ip.NoPrefixForSingleIp = False
+
+    return {
+        'start': addr.to_bytes(16, 'big'),
+        'end': end.to_bytes(16, 'big'),
+        'human': ip.strCompressed(),
+        'prefix': ip.prefixlen(),
+    }
+
+
+def query(address: str) -> dict:
+    """Return a dict usable for querying all organizations whose IP range matches the given one.
+
+    :returns: a dict like:
+        {$elemMatch: {'start': {$lte: b'xxxxx'}, 'end': {$gte: b'xxxxx'}}}
+    """
+
+    ip = IP(address)
+    if ip.version() == 4:
+        ip = ip.v46map()
+    for_mongo = ip.ip.to_bytes(16, 'big')
+
+    return {'$elemMatch': {
+        'start': {'$lte': for_mongo},
+        'end': {'$gte': for_mongo},
+    }}

pillar/api/organizations/patch.py

@@ -0,0 +1,228 @@
+"""Organization patching support."""
+
+import logging
+
+import bson
+from flask import Blueprint, jsonify
+import werkzeug.exceptions as wz_exceptions
+
+from pillar.api.utils.authentication import current_user
+from pillar.api.utils import authorization, str2id, jsonify
+from pillar.api import patch_handler
+from pillar import current_app
+
+log = logging.getLogger(__name__)
+patch_api_blueprint = Blueprint('pillar.api.organizations.patch', __name__)
+
+
+class OrganizationPatchHandler(patch_handler.AbstractPatchHandler):
+    item_name = 'organization'
+
+    @authorization.require_login()
+    def patch_assign_users(self, org_id: bson.ObjectId, patch: dict):
+        """Assigns users to an organization.
+
+        The calling user must be admin of the organization.
+        """
+        from . import NotEnoughSeats
+
+        self._assert_is_admin(org_id)
+
+        # Do some basic validation.
+        try:
+            emails = patch['emails']
+        except KeyError:
+            raise wz_exceptions.BadRequest('No key "email" in patch.')
+
+        # Skip empty emails.
+        emails = [stripped
+                  for stripped in (email.strip() for email in emails)
+                  if stripped]
+
+        log.info('User %s uses PATCH to add users to organization %s',
+                 current_user().user_id, org_id)
+        try:
+            org_doc = current_app.org_manager.assign_users(org_id, emails)
+        except NotEnoughSeats:
+            resp = jsonify({'_message': f'Not enough seats to assign {len(emails)} users'})
+            resp.status_code = 422
+            return resp
+
+        return jsonify(org_doc)
+
+    @authorization.require_login()
+    def patch_assign_user(self, org_id: bson.ObjectId, patch: dict):
+        """Assigns a single user by User ID to an organization.
+
+        The calling user must be admin of the organization.
+        """
+        from . import NotEnoughSeats
+        self._assert_is_admin(org_id)
+
+        # Do some basic validation.
+        try:
+            user_id = patch['user_id']
+        except KeyError:
+            raise wz_exceptions.BadRequest('No key "user_id" in patch.')
+
+        user_oid = str2id(user_id)
+        log.info('User %s uses PATCH to add user %s to organization %s',
+                 current_user().user_id, user_oid, org_id)
+        try:
+            org_doc = current_app.org_manager.assign_single_user(org_id, user_id=user_oid)
+        except NotEnoughSeats:
+            resp = jsonify({'_message': f'Not enough seats to assign this user'})
+            resp.status_code = 422
+            return resp
+
+        return jsonify(org_doc)
+
+    @authorization.require_login()
+    def patch_assign_admin(self, org_id: bson.ObjectId, patch: dict):
+        """Assigns a single user by User ID as admin of the organization.
+
+        The calling user must be admin of the organization.
+        """
+
+        self._assert_is_admin(org_id)
+
+        # Do some basic validation.
+        try:
+            user_id = patch['user_id']
+        except KeyError:
+            raise wz_exceptions.BadRequest('No key "user_id" in patch.')
+
+        user_oid = str2id(user_id)
+        log.info('User %s uses PATCH to set user %s as admin for organization %s',
+                 current_user().user_id, user_oid, org_id)
+        current_app.org_manager.assign_admin(org_id, user_id=user_oid)
+
+    @authorization.require_login()
+    def patch_remove_user(self, org_id: bson.ObjectId, patch: dict):
+        """Removes a user from an organization.
+
+        The calling user must be admin of the organization.
+        """
+
+        # Do some basic validation.
+        email = patch.get('email') or None
+        user_id = patch.get('user_id')
+        user_oid = str2id(user_id) if user_id else None
+
+        # Users require admin rights on the org, except when removing themselves.
+        current_user_id = current_user().user_id
+        if user_oid is None or user_oid != current_user_id:
+            self._assert_is_admin(org_id)
+
+        log.info('User %s uses PATCH to remove user %s from organization %s',
+                 current_user_id, user_oid, org_id)
+
+        org_doc = current_app.org_manager.remove_user(org_id, user_id=user_oid, email=email)
+        return jsonify(org_doc)
+
+    def _assert_is_admin(self, org_id):
+        om = current_app.org_manager
+
+        if current_user().has_cap('admin'):
+            # Always allow admins to edit every organization.
+            return
+
+        if not om.user_is_admin(org_id):
+            log.warning('User %s uses PATCH to edit organization %s, '
+                        'but is not admin of that Organization. Request denied.',
+                        current_user().user_id, org_id)
+            raise wz_exceptions.Forbidden()
+
+    @authorization.require_login()
+    def patch_edit_from_web(self, org_id: bson.ObjectId, patch: dict):
+        """Updates Organization fields from the web.
+
+        The PATCH command supports the following payload. The 'name' field must
+        be set, all other fields are optional. When an optional field is
+        ommitted it will be handled as an instruction to clear that field.
+            {'name': str,
+             'description': str,
+             'website': str,
+             'location': str,
+             'ip_ranges': list of human-readable IP ranges}
+        """
+
+        from pymongo.results import UpdateResult
+        from . import ip_ranges
+
+        self._assert_is_admin(org_id)
+        user = current_user()
+        current_user_id = user.user_id
+
+        # Only take known fields from the patch, don't just copy everything.
+        update = {
+            'name': patch['name'].strip(),
+            'description': patch.get('description', '').strip(),
+            'website': patch.get('website', '').strip(),
+            'location': patch.get('location', '').strip(),
+        }
+        unset = {}
+
+        # Special transformation for IP ranges
+        iprs = patch.get('ip_ranges')
+        if iprs:
+            ipr_docs = []
+            for r in iprs:
+                try:
+                    doc = ip_ranges.doc(r, min_prefixlen6=48, min_prefixlen4=8)
+                except ValueError as ex:
+                    raise wz_exceptions.UnprocessableEntity(f'Invalid IP range {r!r}: {ex}')
+                ipr_docs.append(doc)
+            update['ip_ranges'] = ipr_docs
+        else:
+            unset['ip_ranges'] = True
+
+        refresh_user_roles = False
+        if user.has_cap('admin'):
+            if 'seat_count' in patch:
+                update['seat_count'] = int(patch['seat_count'])
+            if 'org_roles' in patch:
+                org_roles = [stripped for stripped in (role.strip() for role in patch['org_roles'])
+                             if stripped]
+                if not all(role.startswith('org-') for role in org_roles):
+                    raise wz_exceptions.UnprocessableEntity(
+                        'Invalid role given, all roles must start with "org-"')
+
+                update['org_roles'] = org_roles
+                refresh_user_roles = True
+
+        self.log.info('User %s edits Organization %s: %s', current_user_id, org_id, update)
+
+        validator = current_app.validator_for_resource('organizations')
+        if not validator.validate_update(update, org_id):
+            resp = jsonify({
+                '_errors': validator.errors,
+                '_message': ', '.join(f'{field}: {error}'
+                                      for field, error in validator.errors.items()),
+            })
+            resp.status_code = 422
+            return resp
+
+        # Figure out what to set and what to unset
+        for_mongo = {'$set': update}
+        if unset:
+            for_mongo['$unset'] = unset
+
+        organizations_coll = current_app.db('organizations')
+        result: UpdateResult = organizations_coll.update_one({'_id': org_id}, for_mongo)
+
+        if result.matched_count != 1:
+            self.log.warning('User %s edits Organization %s but update matched %i items',
+                             current_user_id, org_id, result.matched_count)
+            raise wz_exceptions.BadRequest()
+
+        if refresh_user_roles:
+            self.log.info('Organization roles set for org %s, refreshing users', org_id)
+            current_app.org_manager.refresh_all_user_roles(org_id)
+
+        return '', 204
+
+
+def setup_app(app):
+    OrganizationPatchHandler(patch_api_blueprint)
+    app.register_api_blueprint(patch_api_blueprint, url_prefix='/organizations')

pillar/api/patch_handler.py

@@ -0,0 +1,92 @@
+"""Handler for PATCH requests.
+
+This supports PATCH request in the sense described by William Durand:
+http://williamdurand.fr/2014/02/14/please-do-not-patch-like-an-idiot/
+
+Each PATCH should be a JSON dict with at least a key 'op' with the
+name of the operation to perform.
+"""
+
+import logging
+
+import flask
+
+from pillar.api.utils import authorization
+
+log = logging.getLogger(__name__)
+
+
+class AbstractPatchHandler:
+    """Abstract PATCH handler supporting multiple operations.
+
+    Each operation, i.e. possible value of the 'op' key in the PATCH body,
+    should be matched to a similarly named "patch_xxx" function in a subclass.
+    For example, the operation "set-owner" is mapped to "patch_set_owner".
+
+    :cvar route: the Flask/Werkzeug route to attach this handler to.
+        For most handlers, the default will be fine.
+    :cvar item_name: the name of the things to patch, like "job", "task" etc.
+        Only used for logging.
+    """
+
+    route: str = '/<object_id>'
+    item_name: str = None
+
+    def __init_subclass__(cls, **kwargs):
+        if not cls.route:
+            raise ValueError('Subclass must set route')
+        if not cls.item_name:
+            raise ValueError('Subclass must set item_name')
+
+    def __init__(self, blueprint: flask.Blueprint):
+        self.log: logging.Logger = log.getChild(self.__class__.__name__)
+        self.patch_handlers = {
+            name[6:].replace('_', '-'): getattr(self, name)
+            for name in dir(self)
+            if name.startswith('patch_') and callable(getattr(self, name))
+        }
+
+        if self.log.isEnabledFor(logging.INFO):
+            self.log.info('Creating PATCH handler %s.%s%s for operations: %s',
+                          blueprint.name, self.patch.__name__, self.route,
+                          sorted(self.patch_handlers.keys()))
+
+        blueprint.add_url_rule(self.route,
+                               self.patch.__name__,
+                               self.patch,
+                               methods=['PATCH'])
+
+    @authorization.require_login()
+    def patch(self, object_id: str):
+        from flask import request
+        import werkzeug.exceptions as wz_exceptions
+        from pillar.api.utils import str2id, authentication
+
+        # Parse the request
+        real_object_id = str2id(object_id)
+        patch = request.get_json()
+        if not patch:
+            self.log.info('Bad PATCH request, did not contain JSON')
+            raise wz_exceptions.BadRequest('Patch must contain JSON')
+
+        try:
+            patch_op = patch['op']
+        except KeyError:
+            self.log.info("Bad PATCH request, did not contain 'op' key")
+            raise wz_exceptions.BadRequest("PATCH should contain 'op' key to denote operation.")
+
+        log.debug('User %s wants to PATCH "%s" %s %s',
+                  authentication.current_user_id(), patch_op, self.item_name, real_object_id)
+
+        # Find the PATCH handler for the operation.
+        try:
+            handler = self.patch_handlers[patch_op]
+        except KeyError:
+            log.warning('No %s PATCH handler for operation %r', self.item_name, patch_op)
+            raise wz_exceptions.BadRequest('Operation %r not supported' % patch_op)
+
+        # Let the PATCH handler do its thing.
+        response = handler(real_object_id, patch)
+        if response is None:
+            return '', 204
+        return response

pillar/api/projects/__init__.py

@@ -3,13 +3,20 @@ from .routes import blueprint_api
 
 
 def setup_app(app, api_prefix):
+    from . import patch
+    patch.setup_app(app)
+
     app.on_replace_projects += hooks.override_is_private_field
     app.on_replace_projects += hooks.before_edit_check_permissions
     app.on_replace_projects += hooks.protect_sensitive_fields
+
     app.on_update_projects += hooks.override_is_private_field
     app.on_update_projects += hooks.before_edit_check_permissions
     app.on_update_projects += hooks.protect_sensitive_fields
+
     app.on_delete_item_projects += hooks.before_delete_project
+    app.on_deleted_item_projects += hooks.after_delete_project
+
     app.on_insert_projects += hooks.before_inserting_override_is_private_field
     app.on_insert_projects += hooks.before_inserting_projects
     app.on_inserted_projects += hooks.after_inserting_projects

pillar/api/projects/hooks.py

@@ -1,17 +1,19 @@
 import copy
 import logging
 
-from flask import request, abort, current_app
-from gcloud import exceptions as gcs_exceptions
+from flask import request, abort
+
+from pillar import current_app
 from pillar.api.node_types.asset import node_type_asset
 from pillar.api.node_types.comment import node_type_comment
 from pillar.api.node_types.group import node_type_group
 from pillar.api.node_types.group_texture import node_type_group_texture
 from pillar.api.node_types.texture import node_type_texture
-from pillar.api.utils.gcs import GoogleCloudStorageBucket
+from pillar.api.file_storage_backends import default_storage_backend
 from pillar.api.utils import authorization, authentication
 from pillar.api.utils import remove_private_keys
 from pillar.api.utils.authorization import user_has_role, check_permissions
+from pillar.auth import current_user
 from .utils import abort_with_error
 
 log = logging.getLogger(__name__)
@@ -28,7 +30,7 @@ def before_inserting_projects(items):
     """
 
     # Allow admin users to do whatever they want.
-    if user_has_role(u'admin'):
+    if user_has_role('admin'):
         return
 
     for item in items:
@@ -64,13 +66,27 @@ def before_delete_project(document):
     """Checks permissions before we allow deletion"""
 
     check_permissions('projects', document, request.method)
+    log.info('Deleting project %s on behalf of user %s', document['_id'], current_user)
+
+
+def after_delete_project(project: dict):
+    """Perform delete on the project's files too."""
+
+    from eve.methods.delete import delete
+
+    pid = project['_id']
+    log.info('Project %s was deleted, also deleting its files.', pid)
+
+    r, _, _, status = delete('files', {'project': pid})
+    if status != 204:
+        log.warning('Unable to delete files of project %s: %s', pid, r)
 
 
 def protect_sensitive_fields(document, original):
     """When not logged in as admin, prevents update to certain fields."""
 
     # Allow admin users to do whatever they want.
-    if user_has_role(u'admin'):
+    if user_has_role('admin'):
         return
 
     def revert(name):
@@ -108,6 +124,8 @@ def after_inserting_projects(projects):
 
 
 def after_inserting_project(project, db_user):
+    from pillar.auth import UserClass
+
     project_id = project['_id']
     user_id = db_user['_id']
 
@@ -133,7 +151,8 @@ def after_inserting_project(project, db_user):
     log.debug('Made user %s member of group %s', user_id, admin_group_id)
 
     # Assign the group to the project with admin rights
-    is_admin = authorization.is_admin(db_user)
+    owner_user = UserClass.construct('', db_user)
+    is_admin = authorization.is_admin(owner_user)
     world_permissions = ['GET'] if is_admin else []
     permissions = {
         'world': world_permissions,
@@ -166,18 +185,8 @@ def after_inserting_project(project, db_user):
         else:
             project['url'] = "p-{!s}".format(project_id)
 
-    # Initialize storage page (defaults to GCS)
-    if current_app.config.get('TESTING'):
-        log.warning('Not creating Google Cloud Storage bucket while running unit tests!')
-    else:
-        try:
-            gcs_storage = GoogleCloudStorageBucket(str(project_id))
-            if gcs_storage.bucket.exists():
-                log.info('Created GCS instance for project %s', project_id)
-            else:
-                log.warning('Unable to create GCS instance for project %s', project_id)
-        except gcs_exceptions.Forbidden as ex:
-            log.warning('GCS forbids me to create CGS instance for project %s: %s', project_id, ex)
+    # Initialize storage using the default specified in STORAGE_BACKEND
+    default_storage_backend(str(project_id))
 
     # Commit the changes directly to the MongoDB; a PUT is not allowed yet,
     # as the project doesn't have a valid permission structure.
@@ -232,5 +241,3 @@ def project_node_type_has_method(response):
 def projects_node_type_has_method(response):
     for project in response['_items']:
         project_node_type_has_method(project)
-
-

pillar/api/projects/merging.py

@@ -0,0 +1,44 @@
+"""Code for merging projects."""
+import logging
+
+from bson import ObjectId
+
+from pillar import current_app
+from pillar.api.file_storage.moving import move_to_bucket
+from pillar.api.utils import random_etag, utcnow
+
+log = logging.getLogger(__name__)
+
+
+def merge_project(pid_from: ObjectId, pid_to: ObjectId):
+    """Move nodes and files from one project to another.
+
+    Note that this may invalidate the nodes, as their node type definition
+    may differ between projects.
+    """
+    log.info('Moving project contents from %s to %s', pid_from, pid_to)
+    assert isinstance(pid_from, ObjectId)
+    assert isinstance(pid_to, ObjectId)
+
+    files_coll = current_app.db('files')
+    nodes_coll = current_app.db('nodes')
+
+    # Move the files first. Since this requires API calls to an external
+    # service, this is more likely to go wrong than moving the nodes.
+    to_move = files_coll.find({'project': pid_from}, projection={'_id': 1})
+    log.info('Moving %d files to project %s', to_move.count(), pid_to)
+    for file_doc in to_move:
+        fid = file_doc['_id']
+        log.debug('moving file %s to project %s', fid, pid_to)
+        move_to_bucket(fid, pid_to)
+
+    # Mass-move the nodes.
+    etag = random_etag()
+    result = nodes_coll.update_many(
+        {'project': pid_from},
+        {'$set': {'project': pid_to,
+                  '_etag': etag,
+                  '_updated': utcnow(),
+                  }}
+    )
+    log.info('Moved %d nodes to project %s', result.modified_count, pid_to)

pillar/api/projects/patch.py

@@ -0,0 +1,85 @@
+"""Project patching support."""
+
+import logging
+
+import flask
+from flask import Blueprint, request
+import werkzeug.exceptions as wz_exceptions
+
+from pillar import current_app
+from pillar.auth import current_user
+from pillar.api.utils import random_etag, str2id, utcnow
+from pillar.api.utils import authorization
+
+log = logging.getLogger(__name__)
+blueprint = Blueprint('projects.patch', __name__)
+
+
+@blueprint.route('/<project_id>', methods=['PATCH'])
+@authorization.require_login()
+def patch_project(project_id: str):
+    """Undelete a project.
+
+    This is done via a custom PATCH due to the lack of transactions of MongoDB;
+    we cannot undelete both project-referenced files and file-referenced
+    projects in one atomic operation.
+    """
+
+    # Parse the request
+    pid = str2id(project_id)
+    patch = request.get_json()
+    if not patch:
+        raise wz_exceptions.BadRequest('Expected JSON body')
+
+    log.debug('User %s wants to PATCH project %s: %s', current_user, pid, patch)
+
+    # 'undelete' is the only operation we support now, so no fancy handler registration.
+    op = patch.get('op', '')
+    if op != 'undelete':
+        log.warning('User %s sent unsupported PATCH op %r to project %s: %s',
+                    current_user, op, pid, patch)
+        raise wz_exceptions.BadRequest(f'unsupported operation {op!r}')
+
+    # Get the project to find the user's permissions.
+    proj_coll = current_app.db('projects')
+    proj = proj_coll.find_one({'_id': pid})
+    if not proj:
+        raise wz_exceptions.NotFound(f'project {pid} not found')
+    allowed = authorization.compute_allowed_methods('projects', proj)
+    if 'PUT' not in allowed:
+        log.warning('User %s tried to undelete project %s but only has permissions %r',
+                    current_user, pid, allowed)
+        raise wz_exceptions.Forbidden(f'no PUT access to project {pid}')
+
+    if not proj.get('_deleted', False):
+        raise wz_exceptions.BadRequest(f'project {pid} was not deleted, unable to undelete')
+
+    # Undelete the files. We cannot do this via Eve, as it doesn't support
+    # PATCHing collections, so direct MongoDB modification is used to set
+    # _deleted=False and provide new _etag and _updated values.
+    new_etag = random_etag()
+
+    log.debug('undeleting files before undeleting project %s', pid)
+    files_coll = current_app.db('files')
+    update_result = files_coll.update_many(
+        {'project': pid},
+        {'$set': {'_deleted': False,
+                  '_etag': new_etag,
+                  '_updated': utcnow()}})
+    log.info('undeleted %d of %d file documents of project %s',
+             update_result.modified_count, update_result.matched_count, pid)
+
+    log.info('undeleting project %s on behalf of user %s', pid, current_user)
+    update_result = proj_coll.update_one({'_id': pid},
+                                         {'$set': {'_deleted': False}})
+    log.info('undeleted %d project document %s', update_result.modified_count, pid)
+
+    resp = flask.Response('', status=204)
+    resp.location = flask.url_for('projects.view', project_url=proj['url'])
+    return resp
+
+
+def setup_app(app):
+    # This needs to be on the same URL prefix as Eve uses for the collection,
+    # and not /p as used for the other Projects API calls.
+    app.register_api_blueprint(blueprint, url_prefix='/projects')

pillar/api/projects/routes.py

@@ -2,11 +2,13 @@ import json
 import logging
 
 from bson import ObjectId
-from flask import Blueprint, g, request, current_app, make_response, url_for
+from flask import Blueprint, request, current_app, make_response, url_for
+from werkzeug import exceptions as wz_exceptions
+
 from pillar.api.utils import authorization, jsonify, str2id
 from pillar.api.utils import mongo
 from pillar.api.utils.authorization import require_login, check_permissions
-from werkzeug import exceptions as wz_exceptions
+from pillar.auth import current_user
 
 from . import utils
 
@@ -16,7 +18,7 @@ blueprint_api = Blueprint('projects_api', __name__)
 
 
 @blueprint_api.route('/create', methods=['POST'])
-@authorization.require_login(require_roles={u'admin', u'subscriber', u'demo'})
+@authorization.require_login(require_cap='subscriber')
 def create_project(overrides=None):
     """Creates a new project."""
 
@@ -24,7 +26,7 @@ def create_project(overrides=None):
         project_name = request.json['name']
     else:
         project_name = request.form['project_name']
-    user_id = g.current_user['user_id']
+    user_id = current_user.user_id
 
     project = utils.create_new_project(project_name, user_id, overrides)
 
@@ -41,6 +43,8 @@ def project_manage_users():
     No changes are done on the project itself.
     """
 
+    from pillar.api.utils import str2id
+
     projects_collection = current_app.data.driver.db['projects']
     users_collection = current_app.data.driver.db['users']
 
@@ -57,17 +61,19 @@ def project_manage_users():
 
     # The request is not a form, since it comes from the API sdk
     data = json.loads(request.data)
-    project_id = ObjectId(data['project_id'])
-    target_user_id = ObjectId(data['user_id'])
+    project_id = str2id(data['project_id'])
+    target_user_id = str2id(data['user_id'])
     action = data['action']
-    current_user_id = g.current_user['user_id']
+    current_user_id = current_user.user_id
 
     project = projects_collection.find_one({'_id': project_id})
 
     # Check if the current_user is owner of the project, or removing themselves.
-    if not authorization.user_has_role(u'admin'):
+    if not authorization.user_has_role('admin'):
         remove_self = target_user_id == current_user_id and action == 'remove'
         if project['user'] != current_user_id and not remove_self:
+            log.warning('User %s tries to %s %s to/from project %s, but is not allowed',
+                        current_user_id, action, target_user_id, project_id)
             utils.abort_with_error(403)
 
     admin_group = utils.get_admin_group(project)

pillar/api/projects/utils.py

@@ -1,10 +1,13 @@
 import logging
+import typing
 
 from bson import ObjectId
-from flask import current_app
 from werkzeug import exceptions as wz_exceptions
 from werkzeug.exceptions import abort
 
+from pillar import current_app
+from pillar.auth import current_user
+
 log = logging.getLogger(__name__)
 
 
@@ -27,12 +30,30 @@ def project_total_file_size(project_id):
         return 0
 
 
-def get_admin_group(project):
+def get_admin_group_id(project_id: ObjectId) -> ObjectId:
+    assert isinstance(project_id, ObjectId)
+
+    project = current_app.db('projects').find_one({'_id': project_id},
+                                                  {'permissions': 1})
+    if not project:
+        raise ValueError(f'Project {project_id} does not exist.')
+
+    # TODO: search through all groups to find the one with the project ID as its name,
+    # or identify "the admin group" in a different way (for example the group with DELETE rights).
+    try:
+        admin_group_id = ObjectId(project['permissions']['groups'][0]['group'])
+    except KeyError:
+        raise ValueError(f'Project {project_id} does not seem to have an admin group')
+
+    return admin_group_id
+
+
+def get_admin_group(project: dict) -> dict:
     """Returns the admin group for the project."""
 
     groups_collection = current_app.data.driver.db['groups']
 
-    # TODO: search through all groups to find the one with the project ID as its name.
+    # TODO: see get_admin_group_id
     admin_group_id = ObjectId(project['permissions']['groups'][0]['group'])
     group = groups_collection.find_one({'_id': admin_group_id})
 
@@ -40,11 +61,27 @@ def get_admin_group(project):
         raise ValueError('Unable to handle project without admin group.')
 
     if group['name'] != str(project['_id']):
+        log.error('User %s tries to get admin group for project %s, '
+                  'but that does not have the project ID as group name: %s',
+                  current_user.user_id, project.get('_id', '-unknown-'), group)
         return abort_with_error(403)
 
     return group
 
 
+def user_rights_in_project(project_id: ObjectId) -> frozenset:
+    """Returns the set of HTTP methods allowed on the given project for the current user."""
+
+    from pillar.api.utils import authorization
+
+    assert isinstance(project_id, ObjectId)
+
+    proj_coll = current_app.db().projects
+    proj = proj_coll.find_one({'_id': project_id})
+
+    return frozenset(authorization.compute_allowed_methods('projects', proj))
+
+
 def abort_with_error(status):
     """Aborts with the given status, or 500 if the status doesn't indicate an error.
 
@@ -97,3 +134,56 @@ def get_node_type(project, node_type_name):
 
     return next((nt for nt in project['node_types']
                  if nt['name'] == node_type_name), None)
+
+
+def node_type_dict(project: dict) -> typing.Dict[str, dict]:
+    """Return the node types of the project as dictionary.
+
+    The returned dictionary will be keyed by the node type name.
+    """
+    return {nt['name']: nt for nt in project['node_types']}
+
+
+def project_id(project_url: str) -> ObjectId:
+    """Returns the object ID, or raises a ValueError when not found."""
+
+    proj_coll = current_app.db('projects')
+    proj = proj_coll.find_one({'url': project_url}, projection={'_id': True})
+
+    if not proj:
+        raise ValueError(f'project with url={project_url!r} not found')
+    return proj['_id']
+
+
+def get_project(project_url: str) -> dict:
+    """Find a project in the database, raises ValueError if not found.
+
+    :param project_url: URL of the project
+    """
+
+    proj_coll = current_app.db('projects')
+    project = proj_coll.find_one({'url': project_url, '_deleted': {'$ne': True}})
+    if not project:
+        raise ValueError(f'project url={project_url!r} does not exist')
+
+    return project
+
+
+def put_project(project: dict):
+    """Puts a project into the database via Eve.
+
+    :param project: the project data, should be the entire project document
+    :raises ValueError: if the project cannot be saved.
+    """
+
+    from pillar.api.utils import remove_private_keys
+    from pillarsdk.utils import remove_none_attributes
+
+    pid = ObjectId(project['_id'])
+    proj_no_priv = remove_private_keys(project)
+    proj_no_none = remove_none_attributes(proj_no_priv)
+    result, _, _, status_code = current_app.put_internal('projects', proj_no_none, _id=pid)
+
+    if status_code != 200:
+        raise ValueError(f"Can't update project {pid}, "
+                         f"status {status_code} with issues: {result}")

pillar/api/search/__init__.py

@@ -0,0 +1,9 @@
+from .routes import blueprint_search
+from . import queries
+
+
+def setup_app(app, url_prefix: str = None):
+    app.register_api_blueprint(
+        blueprint_search, url_prefix=url_prefix)
+
+    queries.setup_app(app)

pillar/api/search/algolia_indexing.py

@@ -0,0 +1,40 @@
+import logging
+
+from algoliasearch.helpers import AlgoliaException
+
+log = logging.getLogger(__name__)
+
+
+def push_updated_user(user_to_index: dict):
+    """Push an update to the index when a user document is updated."""
+
+    from pillar.api.utils.algolia import index_user_save
+
+    try:
+        index_user_save(user_to_index)
+    except AlgoliaException as ex:
+        log.warning(
+            'Unable to push user info to Algolia for user "%s", id=%s; %s',  # noqa
+            user_to_index.get('username'),
+            user_to_index.get('objectID'), ex)
+
+
+def index_node_save(node_to_index: dict):
+    """Save parsed node document to the index."""
+    from pillar.api.utils import algolia
+
+    try:
+        algolia.index_node_save(node_to_index)
+    except AlgoliaException as ex:
+        log.warning(
+            'Unable to push node info to Algolia for node %s; %s', node_to_index, ex)  # noqa
+
+
+def index_node_delete(delete_id: str):
+    """Delete node using id."""
+    from pillar.api.utils import algolia
+
+    try:
+        algolia.index_node_delete(delete_id)
+    except AlgoliaException as ex:
+        log.warning('Unable to delete node info to Algolia for node %s; %s', delete_id, ex)  # noqa

pillar/api/search/documents.py

@@ -0,0 +1,189 @@
+"""
+Define elasticsearch document mapping.
+
+Elasticsearch consist of two parts:
+
+- Part 1: Define the documents in which you define who fields will be indexed.
+- Part 2: Building elasticsearch json queries.
+
+BOTH of these parts are equally importand to havea search API that returns
+relevant results.
+"""
+import logging
+import typing
+
+import elasticsearch_dsl as es
+from elasticsearch_dsl import analysis
+
+log = logging.getLogger(__name__)
+
+edge_ngram_filter = analysis.token_filter(
+    'edge_ngram_filter',
+    type='edge_ngram',
+    min_gram=1,
+    max_gram=15
+)
+
+autocomplete = es.analyzer(
+    'autocomplete',
+    tokenizer='standard',
+    filter=['standard', 'asciifolding', 'lowercase', edge_ngram_filter]
+)
+
+
+class User(es.DocType):
+    """Elastic document describing user."""
+
+    objectID = es.Keyword()
+
+    username = es.Text(fielddata=True, analyzer=autocomplete)
+    username_exact = es.Keyword()
+    full_name = es.Text(fielddata=True, analyzer=autocomplete)
+
+    roles = es.Keyword(multi=True)
+    groups = es.Keyword(multi=True)
+
+    email = es.Text(fielddata=True, analyzer=autocomplete)
+    email_exact = es.Keyword()
+
+    class Meta:
+        index = 'users'
+
+
+class Node(es.DocType):
+    """
+    Elastic document describing user
+    """
+
+    node_type = es.Keyword()
+
+    objectID = es.Keyword()
+
+    name = es.Text(
+        fielddata=True,
+        analyzer=autocomplete
+    )
+
+    user = es.Object(
+        fields={
+            'id': es.Keyword(),
+            'name': es.Text(
+                fielddata=True,
+                analyzer=autocomplete)
+        }
+    )
+
+    description = es.Text()
+
+    is_free = es.Boolean()
+
+    project = es.Object(
+        fields={
+            'id': es.Keyword(),
+            'name': es.Keyword(),
+        }
+    )
+
+    media = es.Keyword()
+
+    picture = es.Keyword()
+
+    tags = es.Keyword(multi=True)
+    license_notes = es.Text()
+
+    created_at = es.Date()
+    updated_at = es.Date()
+
+    class Meta:
+        index = 'nodes'
+
+
+def create_doc_from_user_data(user_to_index: dict) -> typing.Optional[User]:
+    """
+    Create the document to store in a search engine for this user.
+
+    See pillar.celery.search_index_task
+
+    :returns: an ElasticSearch document or None if user_to_index has no data.
+    """
+
+    if not user_to_index:
+        return None
+
+    doc_id = str(user_to_index.get('objectID', ''))
+
+    if not doc_id:
+        log.error('USER ID is missing %s', user_to_index)
+        raise KeyError('Trying to create document without id')
+
+    doc = User(_id=doc_id)
+    doc.objectID = str(user_to_index['objectID'])
+    doc.username = user_to_index['username']
+    doc.username_exact = user_to_index['username']
+    doc.full_name = user_to_index['full_name']
+    doc.roles = list(map(str, user_to_index['roles']))
+    doc.groups = list(map(str, user_to_index['groups']))
+    doc.email = user_to_index['email']
+    doc.email_exact = user_to_index['email']
+
+    return doc
+
+
+def create_doc_from_node_data(node_to_index: dict) -> typing.Optional[Node]:
+    """
+    Create the document to store in a search engine for this node.
+
+    See pillar.celery.search_index_task
+
+    :returns: an ElasticSearch document or None if node_to_index has no data.
+    """
+
+    if not node_to_index:
+        return None
+
+    # node stuff
+    doc_id = str(node_to_index.get('objectID', ''))
+
+    if not doc_id:
+        log.error('ID missing %s', node_to_index)
+        return None
+
+    doc = Node(_id=doc_id)
+
+    doc.objectID = str(node_to_index['objectID'])
+    doc.node_type = node_to_index['node_type']
+    doc.name = node_to_index['name']
+    doc.user.id = str(node_to_index['user']['_id'])
+    doc.user.name = node_to_index['user']['full_name']
+    doc.project.id = str(node_to_index['project']['_id'])
+    doc.project.name = node_to_index['project']['name']
+
+    if node_to_index['node_type'] == 'asset':
+        doc.media = node_to_index['media']
+
+    doc.picture = node_to_index.get('picture')
+
+    doc.tags = node_to_index.get('tags')
+    doc.license_notes = node_to_index.get('license_notes')
+
+    doc.created_at = node_to_index['created']
+    doc.updated_at = node_to_index['updated']
+
+    return doc
+
+
+def create_doc_from_user(user_to_index: dict) -> User:
+    """
+    Create a user document from user
+    """
+
+    doc_id = str(user_to_index['objectID'])
+    doc = User(_id=doc_id)
+    doc.objectID = str(user_to_index['objectID'])
+    doc.full_name = user_to_index['full_name']
+    doc.username = user_to_index['username']
+    doc.roles = user_to_index['roles']
+    doc.groups = user_to_index['groups']
+    doc.email = user_to_index['email']
+
+    return doc

pillar/api/search/elastic_indexing.py

@@ -0,0 +1,65 @@
+import logging
+
+from elasticsearch_dsl.connections import connections
+from elasticsearch.exceptions import NotFoundError
+
+from pillar import current_app
+from . import documents
+
+log = logging.getLogger(__name__)
+
+elk_hosts = current_app.config['ELASTIC_SEARCH_HOSTS']
+
+connections.create_connection(
+    hosts=elk_hosts,
+    sniff_on_start=False,
+    timeout=20)
+
+
+def push_updated_user(user_to_index: dict):
+    """
+    Push an update to the Elastic index when a user item is updated.
+    """
+    if not user_to_index:
+        return
+
+    doc = documents.create_doc_from_user_data(user_to_index)
+
+    if not doc:
+        return
+
+    index = current_app.config['ELASTIC_INDICES']['USER']
+    log.debug('Index %r update user doc %s in ElasticSearch.', index, doc._id)
+    doc.save(index=index)
+
+
+def index_node_save(node_to_index: dict):
+    """
+    Push an update to the Elastic index when a node item is saved.
+    """
+    if not node_to_index:
+        return
+
+    doc = documents.create_doc_from_node_data(node_to_index)
+
+    if not doc:
+        return
+
+    index = current_app.config['ELASTIC_INDICES']['NODE']
+    log.debug('Index %r update node doc %s in ElasticSearch.', index, doc._id)
+    doc.save(index=index)
+
+
+def index_node_delete(delete_id: str):
+    """
+    Delete node document from Elastic index useing a node id
+    """
+    index = current_app.config['ELASTIC_INDICES']['NODE']
+    log.debug('Index %r node doc delete %s', index, delete_id)
+
+    try:
+        doc: documents.Node = documents.Node.get(id=delete_id)
+        doc.delete(index=index)
+    except NotFoundError:
+        # seems to be gone already..
+        pass

pillar/api/search/index.py

@@ -0,0 +1,64 @@
+import logging
+from typing import List
+
+from elasticsearch.exceptions import NotFoundError
+from elasticsearch_dsl.connections import connections
+import elasticsearch_dsl as es
+
+from pillar import current_app
+
+from . import documents
+
+log = logging.getLogger(__name__)
+
+
+class ResetIndexTask(object):
+    """ Clear and build index / mapping """
+
+    # Key into the ELASTIC_INDICES dict in the app config.
+    index_key: str = ''
+
+    # List of elastic document  types
+    doc_types: List = []
+    name = 'remove index'
+
+    def __init__(self):
+        if not self.index_key:
+            raise ValueError("No index specified")
+
+        if not self.doc_types:
+            raise ValueError("No doc_types specified")
+
+        connections.create_connection(
+            hosts=current_app.config['ELASTIC_SEARCH_HOSTS'],
+            # sniff_on_start=True,
+            retry_on_timeout=True,
+        )
+
+    def execute(self):
+        index = current_app.config['ELASTIC_INDICES'][self.index_key]
+        idx = es.Index(index)
+
+        try:
+            idx.delete(ignore=404)
+        except NotFoundError:
+            log.warning("Could not delete index '%s', ignoring", index)
+        else:
+            log.info("Deleted index %s", index)
+
+        # create doc types
+        for dt in self.doc_types:
+            idx.doc_type(dt)
+
+        # create index
+        idx.create()
+
+
+class ResetNodeIndex(ResetIndexTask):
+    index_key = 'NODE'
+    doc_types = [documents.Node]
+
+
+class ResetUserIndex(ResetIndexTask):
+    index_key = 'USER'
+    doc_types = [documents.User]

pillar/api/search/queries.py

@@ -0,0 +1,183 @@
+import json
+import logging
+import typing
+
+from elasticsearch import Elasticsearch
+from elasticsearch_dsl import Search, Q
+from elasticsearch_dsl.query import Query
+
+from pillar import current_app
+
+log = logging.getLogger(__name__)
+
+NODE_AGG_TERMS = ['node_type', 'media', 'tags', 'is_free']
+USER_AGG_TERMS = ['roles', ]
+ITEMS_PER_PAGE = 10
+
+# Will be set in setup_app()
+client: Elasticsearch = None
+
+
+def add_aggs_to_search(search, agg_terms):
+    """
+    Add facets / aggregations to the search result
+    """
+
+    for term in agg_terms:
+        search.aggs.bucket(term, 'terms', field=term)
+
+
+def make_must(must: list, terms: dict) -> list:
+    """ Given term parameters append must queries to the must list """
+
+    for field, value in terms.items():
+        if value:
+            must.append({'match': {field: value}})
+
+    return must
+
+
+def nested_bool(must: list, should: list, terms: dict, *, index_alias: str) -> Search:
+    """
+    Create a nested bool, where the aggregation selection is a must.
+
+    :param index_alias: 'USER' or 'NODE', see ELASTIC_INDICES config.
+    """
+    must = make_must(must, terms)
+    bool_query = Q('bool', should=should)
+    must.append(bool_query)
+    bool_query = Q('bool', must=must)
+
+    index = current_app.config['ELASTIC_INDICES'][index_alias]
+    search = Search(using=client, index=index)
+    search.query = bool_query
+
+    return search
+
+
+def do_node_search(query: str, terms: dict, page: int, project_id: str='') -> dict:
+    """
+    Given user query input and term refinements
+    search for public published nodes
+    """
+
+    should = [
+        Q('match', name=query),
+
+        {"match": {"project.name": query}},
+        {"match": {"user.name": query}},
+
+        Q('match', description=query),
+        Q('term', media=query),
+        Q('term', tags=query),
+    ]
+
+    must = []
+    if project_id:
+        must.append({'term': {'project.id': project_id}})
+
+    if not query:
+        should = []
+
+    search = nested_bool(must, should, terms, index_alias='NODE')
+    if not query:
+        search = search.sort('-created_at')
+    add_aggs_to_search(search, NODE_AGG_TERMS)
+    search = paginate(search, page)
+
+    if log.isEnabledFor(logging.DEBUG):
+        log.debug(json.dumps(search.to_dict(), indent=4))
+
+    response = search.execute()
+
+    if log.isEnabledFor(logging.DEBUG):
+        log.debug(json.dumps(response.to_dict(), indent=4))
+
+    return response.to_dict()
+
+
+def do_user_search(query: str, terms: dict, page: int) -> dict:
+    """ return user objects represented in elasicsearch result dict"""
+
+    must, should = _common_user_search(query)
+    search = nested_bool(must, should, terms, index_alias='USER')
+    add_aggs_to_search(search, USER_AGG_TERMS)
+    search = paginate(search, page)
+
+    if log.isEnabledFor(logging.DEBUG):
+        log.debug(json.dumps(search.to_dict(), indent=4))
+
+    response = search.execute()
+
+    if log.isEnabledFor(logging.DEBUG):
+        log.debug(json.dumps(response.to_dict(), indent=4))
+
+    return response.to_dict()
+
+
+def _common_user_search(query: str) -> (typing.List[Query], typing.List[Query]):
+    """Construct (must,shoud) for regular + admin user search."""
+    if not query:
+        return [], []
+
+    should = []
+
+    if '@' in query:
+        should.append({'term': {'email_exact': {'value': query, 'boost': 50}}})
+        email_boost = 25
+    else:
+        email_boost = 1
+
+    should.extend([
+        Q('match', username=query),
+        Q('match', full_name=query),
+        {'match': {'email': {'query': query, 'boost': email_boost}}},
+        {'term': {'username_exact': {'value': query, 'boost': 50}}},
+    ])
+
+    return [], should
+
+
+def do_user_search_admin(query: str, terms: dict, page: int) -> dict:
+    """
+    return users seach result dict object
+    search all user fields and provide aggregation information
+    """
+
+    must, should = _common_user_search(query)
+
+    if query:
+        # We most likely got and id field. we should find it.
+        if len(query) == len('563aca02c379cf0005e8e17d'):
+            should.append({'term': {
+                'objectID': {
+                    'value': query,  # the thing we're looking for
+                    'boost': 100,  # how much more it counts for the score
+                }
+            }})
+
+    search = nested_bool(must, should, terms, index_alias='USER')
+    add_aggs_to_search(search, USER_AGG_TERMS)
+    search = paginate(search, page)
+
+    if log.isEnabledFor(logging.DEBUG):
+        log.debug(json.dumps(search.to_dict(), indent=4))
+
+    response = search.execute()
+
+    if log.isEnabledFor(logging.DEBUG):
+        log.debug(json.dumps(response.to_dict(), indent=4))
+
+    return response.to_dict()
+
+
+def paginate(search: Search, page_idx: int) -> Search:
+    return search[page_idx * ITEMS_PER_PAGE:(page_idx + 1) * ITEMS_PER_PAGE]
+
+
+def setup_app(app):
+    global client
+
+    hosts = app.config['ELASTIC_SEARCH_HOSTS']
+    log.getChild('setup_app').info('Creating ElasticSearch client for %s', hosts)
+    client = Elasticsearch(hosts)

pillar/api/search/routes.py

@@ -0,0 +1,110 @@
+import logging
+
+from flask import Blueprint, request
+import elasticsearch.exceptions as elk_ex
+from werkzeug import exceptions as wz_exceptions
+from pillar.api.utils import authorization, jsonify
+
+from . import queries
+
+log = logging.getLogger(__name__)
+
+blueprint_search = Blueprint('elksearch', __name__)
+
+TERMS = [
+    'node_type', 'media',
+    'tags', 'is_free', 'projectname',
+    'roles',
+]
+
+
+def _term_filters() -> dict:
+    """
+    Check if frontent wants to filter stuff
+    on specific fields AKA facets
+
+    return mapping with term field name
+    and provided user term value
+    """
+    return {term: request.args.get(term, '') for term in TERMS}
+
+
+def _page_index() -> int:
+    """Return the page index from the query string."""
+    try:
+        page_idx = int(request.args.get('page') or '0')
+    except TypeError:
+        log.info('invalid page number %r received', request.args.get('page'))
+        raise wz_exceptions.BadRequest()
+    return page_idx
+
+
+@blueprint_search.route('/')
+def search_nodes():
+    searchword = request.args.get('q', '')
+    project_id = request.args.get('project', '')
+    terms = _term_filters()
+    page_idx = _page_index()
+
+    result = queries.do_node_search(searchword, terms, page_idx, project_id)
+    return jsonify(result)
+
+
+@blueprint_search.route('/user')
+def search_user():
+    searchword = request.args.get('q', '')
+    terms = _term_filters()
+    page_idx = _page_index()
+    # result is the raw elasticseach output.
+    # we need to filter fields in case of user objects.
+
+    try:
+        result = queries.do_user_search(searchword, terms, page_idx)
+    except elk_ex.ElasticsearchException as ex:
+        resp = jsonify({'_message': str(ex)})
+        resp.status_code = 500
+        return resp
+
+    # filter sensitive stuff
+    # we only need. objectID, full_name, username
+    hits = result.get('hits', {})
+
+    new_hits = []
+
+    for hit in hits.get('hits'):
+        source = hit['_source']
+        single_hit = {
+            '_source': {
+                'objectID': source.get('objectID'),
+                'username': source.get('username'),
+                'full_name': source.get('full_name'),
+            }
+        }
+
+        new_hits.append(single_hit)
+
+    # replace search result with safe subset
+    result['hits']['hits'] = new_hits
+
+    return jsonify(result)
+
+
+@blueprint_search.route('/admin/user')
+@authorization.require_login(require_cap='admin')
+def search_user_admin():
+    """
+    User search over all fields.
+    """
+
+    searchword = request.args.get('q', '')
+    terms = _term_filters()
+    page_idx = _page_index()
+
+    try:
+        result = queries.do_user_search_admin(searchword, terms, page_idx)
+    except elk_ex.ElasticsearchException as ex:
+        resp = jsonify({'_message': str(ex)})
+        resp.status_code = 500
+        return resp
+
+    return jsonify(result)

pillar/api/service.py

@@ -1,24 +1,31 @@
 """Service accounts."""
 
 import logging
+import typing
 
 import blinker
+import bson
+
 from flask import Blueprint, current_app, request
-from pillar.api import local_auth
-from pillar.api.utils import mongo
-from pillar.api.utils import authorization, authentication, str2id, jsonify
 from werkzeug import exceptions as wz_exceptions
 
+from pillar.api import local_auth
+from pillar.api.utils import authorization, authentication
+
 blueprint = Blueprint('service', __name__)
 log = logging.getLogger(__name__)
 signal_user_changed_role = blinker.NamedSignal('badger:user_changed_role')
 
-ROLES_WITH_GROUPS = {u'admin', u'demo', u'subscriber'}
+ROLES_WITH_GROUPS = {'admin', 'demo', 'subscriber'}
 
 # Map of role name to group ID, for the above groups.
 role_to_group_id = {}
 
 
+class ServiceAccountCreationError(Exception):
+    """Raised when a service account cannot be created."""
+
+
 @blueprint.before_app_first_request
 def fetch_role_to_group_id_map():
     """Fills the _role_to_group_id mapping upon application startup."""
@@ -38,7 +45,7 @@ def fetch_role_to_group_id_map():
 
 
 @blueprint.route('/badger', methods=['POST'])
-@authorization.require_login(require_roles={u'service', u'badger'}, require_all=True)
+@authorization.require_login(require_roles={'service', 'badger'}, require_all=True)
 def badger():
     if request.mimetype != 'application/json':
         log.debug('Received %s instead of application/json', request.mimetype)
@@ -70,42 +77,76 @@ def badger():
                     action, user_email, role, action, role)
         return 'Role not allowed', 403
 
-    return do_badger(action, user_email, role)
+    return do_badger(action, role=role, user_email=user_email)
 
 
-def do_badger(action, user_email, role):
-    """Performs a badger action, returning a HTTP response."""
+def do_badger(action: str, *,
+              role: str=None, roles: typing.Iterable[str]=None,
+              user_email: str = '', user_id: bson.ObjectId = None):
+    """Performs a badger action, returning a HTTP response.
+
+    Either role or roles must be given.
+    Either user_email or user_id must be given.
+    """
 
     if action not in {'grant', 'revoke'}:
+        log.error('do_badger(%r, %r, %r, %r): action %r not supported.',
+                  action, role, user_email, user_id, action)
         raise wz_exceptions.BadRequest('Action %r not supported' % action)
 
-    if not user_email:
+    if not user_email and user_id is None:
+        log.error('do_badger(%r, %r, %r, %r): neither email nor user_id given.',
+                  action, role, user_email, user_id)
         raise wz_exceptions.BadRequest('User email not given')
 
-    if not role:
-        raise wz_exceptions.BadRequest('Role not given')
+    if bool(role) == bool(roles):
+        log.error('do_badger(%r, role=%r, roles=%r, %r, %r): '
+                  'either "role" or "roles" must be given.',
+                  action, role, roles, user_email, user_id)
+        raise wz_exceptions.BadRequest('Invalid role(s) given')
+
+    # If only a single role was given, handle it as a set of one role.
+    if not roles:
+        roles = {role}
+    del role
 
     users_coll = current_app.data.driver.db['users']
 
     # Fetch the user
-    db_user = users_coll.find_one({'email': user_email}, projection={'roles': 1, 'groups': 1})
+    if user_email:
+        query = {'email': user_email}
+    else:
+        query = user_id
+    db_user = users_coll.find_one(query, projection={'roles': 1, 'groups': 1})
     if db_user is None:
-        log.warning('badger(%s, %s, %s): user not found', action, user_email, role)
+        log.warning('badger(%s, roles=%s, user_email=%s, user_id=%s): user not found',
+                    action, roles, user_email, user_id)
         return 'User not found', 404
 
     # Apply the action
-    roles = set(db_user.get('roles') or [])
+    user_roles = set(db_user.get('roles') or [])
     if action == 'grant':
-        roles.add(role)
+        user_roles |= roles
     else:
-        roles.discard(role)
+        user_roles -= roles
 
+    groups = None
+    for role in roles:
         groups = manage_user_group_membership(db_user, role, action)
 
-    updates = {'roles': list(roles)}
+        if groups is None:
+            # No change for this role
+            continue
+
+        # Also update db_user for the next iteration.
+        db_user['groups'] = groups
+
+    updates = {'roles': list(user_roles)}
     if groups is not None:
         updates['groups'] = list(groups)
 
+    log.debug('badger(%s, %s, user_email=%s, user_id=%s): applying updates %r',
+              action, role, user_email, user_id, updates)
     users_coll.update_one({'_id': db_user['_id']},
                           {'$set': updates})
 
@@ -116,19 +157,6 @@ def do_badger(action, user_email, role):
     return '', 204
 
 
-@blueprint.route('/urler/<project_id>', methods=['GET'])
-@authorization.require_login(require_roles={u'service', u'urler'}, require_all=True)
-def urler(project_id):
-    """Returns the URL of any project."""
-
-    project_id = str2id(project_id)
-    project = mongo.find_one_or_404('projects', project_id,
-                                    projection={'url': 1})
-    return jsonify({
-        '_id': project_id,
-        'url': project['url']})
-
-
 def manage_user_group_membership(db_user, role, action):
     """Some roles have associated groups; this function maintains group & role membership.
 
@@ -162,69 +190,52 @@ def manage_user_group_membership(db_user, role, action):
     return user_groups
 
 
-def create_service_account(email, roles, service, update_existing=None):
+def create_service_account(email: str, roles: typing.Iterable, service: dict,
+                           *, full_name: str=None):
     """Creates a service account with the given roles + the role 'service'.
 
-    :param email: email address associated with the account
-    :type email: str
+    :param email: optional email address associated with the account.
     :param roles: iterable of role names
     :param service: dict of the 'service' key in the user.
-    :type service: dict
-    :param update_existing: callback function that receives an existing user to update
-        for this service, in case the email address is already in use by someone.
-        If not given or None, updating existing users is disallowed, and a ValueError
-        exception is thrown instead.
+    :param full_name: Full name of the service account. If None, will be set to
+        something reasonable.
 
     :return: tuple (user doc, token doc)
     """
 
-    from pillar.api.utils import remove_private_keys
-
-    # Find existing
-    users_coll = current_app.db()['users']
-    user = users_coll.find_one({'email': email})
-    if user:
-        # Check whether updating is allowed at all.
-        if update_existing is None:
-            raise ValueError('User %s already exists' % email)
-
-        # Compute the new roles, and assign.
-        roles = list(set(roles).union({u'service'}).union(user['roles']))
-        user['roles'] = list(roles)
-
-        # Let the caller perform any required updates.
-        log.info('Updating existing user %s to become service account for %s',
-                 email, roles)
-        update_existing(user['service'])
-
-        # Try to store the updated user.
-        result, _, _, status = current_app.put_internal('users',
-                                                        remove_private_keys(user),
-                                                        _id=user['_id'])
-        expected_status = 200
-    else:
     # Create a user with the correct roles.
-        roles = list(set(roles).union({u'service'}))
-        user = {'username': email,
+    roles = sorted(set(roles).union({'service'}))
+    user_id = bson.ObjectId()
+
+    log.info('Creating service account %s with roles %s', user_id, roles)
+    user = {'_id': user_id,
+            'username': f'SRV-{user_id}',
             'groups': [],
             'roles': roles,
             'settings': {'email_communications': 0},
             'auth': [],
-                'full_name': email,
-                'email': email,
+            'full_name': full_name or f'SRV-{user_id}',
             'service': service}
+    if email:
+        user['email'] = email
     result, _, _, status = current_app.post_internal('users', user)
-        expected_status = 201
 
-    if status != expected_status:
-        raise SystemExit('Error creating user {}: {}'.format(email, result))
+    if status != 201:
+        raise ServiceAccountCreationError('Error creating user {}: {}'.format(user_id, result))
     user.update(result)
 
     # Create an authentication token that won't expire for a long time.
-    token = local_auth.generate_and_store_token(user['_id'], days=36500, prefix='SRV')
+    token = generate_auth_token(user['_id'])
 
     return user, token
 
 
+def generate_auth_token(service_account_id) -> dict:
+    """Generates an authentication token for a service account."""
+
+    token_info = local_auth.generate_and_store_token(service_account_id, days=36500, prefix=b'SRV')
+    return token_info
+
+
 def setup_app(app, api_prefix):
     app.register_api_blueprint(blueprint, url_prefix=api_prefix)

pillar/api/users/__init__.py

@@ -1,15 +1,79 @@
+import logging
+
+import bson
+from flask import current_app
+
 from . import hooks
 from .routes import blueprint_api
 
+log = logging.getLogger(__name__)
+
+
+def remove_user_from_group(user_id: bson.ObjectId, group_id: bson.ObjectId):
+    """Removes the user from the given group.
+
+    Directly uses MongoDB, so that it doesn't require any special permissions.
+    """
+
+    log.info('Removing user %s from group %s', user_id, group_id)
+    user_group_action(user_id, group_id, '$pull')
+
+
+def add_user_to_group(user_id: bson.ObjectId, group_id: bson.ObjectId):
+    """Makes the user member of the given group.
+
+    Directly uses MongoDB, so that it doesn't require any special permissions.
+    """
+
+    log.info('Adding user %s to group %s', user_id, group_id)
+    user_group_action(user_id, group_id, '$addToSet')
+
+
+def user_group_action(user_id: bson.ObjectId, group_id: bson.ObjectId, action: str):
+    """Performs a group action (add/remove).
+
+    :param user_id: the user's ObjectID.
+    :param group_id: the group's ObjectID.
+    :param action: either '$pull' to remove from a group, or '$addToSet' to add to a group.
+    """
+
+    from pymongo.results import UpdateResult
+
+    assert isinstance(user_id, bson.ObjectId)
+    assert isinstance(group_id, bson.ObjectId)
+    assert action in {'$pull', '$addToSet'}
+
+    users_coll = current_app.db('users')
+    result: UpdateResult = users_coll.update_one(
+        {'_id': user_id},
+        {action: {'groups': group_id}},
+    )
+
+    if result.matched_count == 0:
+        raise ValueError(f'Unable to {action} user {user_id} membership of group {group_id}; '
+                         f'user not found.')
+
+
+def _update_search_user_changed_role(sender, user: dict):
+    log.debug('Sending updated user %s to Algolia due to role change', user['_id'])
+    hooks.push_updated_user_to_search(user, original=None)
+
 
 def setup_app(app, api_prefix):
+    from pillar.api import service
+
     app.on_pre_GET_users += hooks.check_user_access
     app.on_post_GET_users += hooks.post_GET_user
     app.on_pre_PUT_users += hooks.check_put_access
     app.on_pre_PUT_users += hooks.before_replacing_user
-    app.on_replaced_users += hooks.push_updated_user_to_algolia
+    app.on_replaced_users += hooks.push_updated_user_to_search
     app.on_replaced_users += hooks.send_blinker_signal_roles_changed
     app.on_fetched_item_users += hooks.after_fetching_user
     app.on_fetched_resource_users += hooks.after_fetching_user_resource
 
+    app.on_insert_users += hooks.before_inserting_users
+    app.on_inserted_users += hooks.after_inserting_users
+
     app.register_api_blueprint(blueprint_api, url_prefix=api_prefix)
+
+    service.signal_user_changed_role.connect(_update_search_user_changed_role)

pillar/api/users/hooks.py

@@ -1,88 +1,129 @@
 import copy
 import json
 
+import bson
 from eve.utils import parse_request
-from flask import current_app, g
+from werkzeug import exceptions as wz_exceptions
+
+from pillar import current_app
 from pillar.api.users.routes import log
 from pillar.api.utils.authorization import user_has_role
-from werkzeug.exceptions import Forbidden
+import pillar.auth
+
+USER_EDITABLE_FIELDS = {'full_name', 'username', 'email', 'settings'}
+
+# These fields nobody is allowed to touch directly, not even admins.
+USER_ALWAYS_RESTORE_FIELDS = {'auth'}
 
 
 def before_replacing_user(request, lookup):
-    """Loads the auth field from the database, preventing any changes."""
+    """Prevents changes to any field of the user doc, except USER_EDITABLE_FIELDS."""
 
     # Find the user that is being replaced
     req = parse_request('users')
-    req.projection = json.dumps({'auth': 1})
+    req.projection = json.dumps({key: 0 for key in USER_EDITABLE_FIELDS})
     original = current_app.data.find_one('users', req, **lookup)
 
     # Make sure that the replacement has a valid auth field.
-    updates = request.get_json()
-    assert updates is request.get_json()  # We should get a ref to the cached JSON, and not a copy.
+    put_data = request.get_json()
+    if put_data is None:
+        raise wz_exceptions.BadRequest('No JSON data received')
 
-    if 'auth' in original:
-        updates['auth'] = copy.deepcopy(original['auth'])
+    # We should get a ref to the cached JSON, and not a copy. This will allow us to
+    # modify the cached JSON so that Eve sees our modifications.
+    assert put_data is request.get_json()
+
+    # Reset fields that shouldn't be edited to their original values. This is only
+    # needed when users are editing themselves; admins are allowed to edit much more.
+    if not pillar.auth.current_user.has_cap('admin'):
+        for db_key, db_value in original.items():
+            if db_key[0] == '_' or db_key in USER_EDITABLE_FIELDS:
+                continue
+
+            if db_key in original:
+                put_data[db_key] = copy.deepcopy(original[db_key])
+
+        # Remove fields added by this PUT request, except when they are user-editable.
+        for put_key in list(put_data.keys()):
+            if put_key[0] == '_' or put_key in USER_EDITABLE_FIELDS:
+                continue
+
+            if put_key not in original:
+                del put_data[put_key]
+
+    # Always restore those fields
+    for db_key in USER_ALWAYS_RESTORE_FIELDS:
+        if db_key in original:
+            put_data[db_key] = copy.deepcopy(original[db_key])
         else:
-        updates.pop('auth', None)
+            del put_data[db_key]
+
+    # Regular users should always have an email address
+    if 'service' not in put_data.get('roles', ()):
+        if not put_data.get('email'):
+            raise wz_exceptions.UnprocessableEntity(
+                'email field must be given')
 
 
-def push_updated_user_to_algolia(user, original):
-    """Push an update to the Algolia index when a user item is updated"""
+def push_updated_user_to_search(user, original):
+    """
+    Push an update to the Search index when a user
+    item is updated
+    """
 
-    from algoliasearch.client import AlgoliaException
-    from pillar.api.utils.algolia import algolia_index_user_save
+    from pillar.celery import search_index_tasks as searchindex
 
-    try:
-        algolia_index_user_save(user)
-    except AlgoliaException as ex:
-        log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s',
-                    user.get('username'), user.get('_id'), ex)
+    searchindex.updated_user.delay(str(user['_id']))
 
 
 def send_blinker_signal_roles_changed(user, original):
-    """Sends a Blinker signal that the user roles were changed, so others can respond."""
+    """
+    Sends a Blinker signal that the user roles were
+    changed, so others can respond.
+    """
 
-    if user.get('roles') == original.get('roles'):
+    current_roles = set(user.get('roles', []))
+    original_roles = set(original.get('roles', []))
+
+    if current_roles == original_roles:
         return
 
     from pillar.api.service import signal_user_changed_role
 
     log.info('User %s changed roles to %s, sending Blinker signal',
-             user.get('_id'), user.get('roles'))
+             user.get('_id'), current_roles)
     signal_user_changed_role.send(current_app, user=user)
 
 
 def check_user_access(request, lookup):
     """Modifies the lookup dict to limit returned user info."""
 
-    # No access when not logged in.
-    current_user = g.get('current_user')
-    current_user_id = current_user['user_id'] if current_user else None
+    user = pillar.auth.get_current_user()
 
     # Admins can do anything and get everything, except the 'auth' block.
-    if user_has_role(u'admin'):
+    if user.has_cap('admin'):
         return
 
-    if not lookup and not current_user:
-        raise Forbidden()
+    if not lookup and user.is_anonymous:
+        raise wz_exceptions.Forbidden()
 
     # Add a filter to only return the current user.
     if '_id' not in lookup:
-        lookup['_id'] = current_user['user_id']
+        lookup['_id'] = user.user_id
 
 
 def check_put_access(request, lookup):
     """Only allow PUT to the current user, or all users if admin."""
 
-    if user_has_role(u'admin'):
+    user = pillar.auth.get_current_user()
+    if user.has_cap('admin'):
         return
 
-    current_user = g.get('current_user')
-    if not current_user:
-        raise Forbidden()
+    if user.is_anonymous:
+        raise wz_exceptions.Forbidden()
 
-    if str(lookup['_id']) != str(current_user['user_id']):
-        raise Forbidden()
+    if str(lookup['_id']) != str(user.user_id):
+        raise wz_exceptions.Forbidden()
 
 
 def after_fetching_user(user):
@@ -90,19 +131,18 @@ def after_fetching_user(user):
     # custom end-points.
     user.pop('auth', None)
 
-    current_user = g.get('current_user')
-    current_user_id = current_user['user_id'] if current_user else None
+    current_user = pillar.auth.get_current_user()
 
     # Admins can do anything and get everything, except the 'auth' block.
-    if user_has_role(u'admin'):
+    if current_user.has_cap('admin'):
         return
 
     # Only allow full access to the current user.
-    if str(user['_id']) == str(current_user_id):
+    if current_user.is_authenticated and str(user['_id']) == str(current_user.user_id):
         return
 
     # Remove all fields except public ones.
-    public_fields = {'full_name', 'username', 'email'}
+    public_fields = {'full_name', 'username', 'email', 'extension_props_public'}
     for field in list(user.keys()):
         if field not in public_fields:
             del user[field]
@@ -121,3 +161,46 @@ def post_GET_user(request, payload):
     # json_data['computed_permissions'] = \
     #     compute_permissions(json_data['_id'], app.data.driver)
     payload.data = json.dumps(json_data)
+
+
+def grant_org_roles(user_doc):
+    """Handle any organization this user may be part of."""
+
+    email = user_doc.get('email')
+    if not email:
+        log.info('Unable to check new user for organization membership, no email address: %r',
+                 user_doc)
+        return
+
+    org_roles = current_app.org_manager.unknown_member_roles(email)
+    if not org_roles:
+        log.debug('No organization roles for user %r', email)
+        return
+
+    log.info('Granting organization roles %r to user %r', org_roles, email)
+    new_roles = set(user_doc.get('roles') or []) | org_roles
+    user_doc['roles'] = list(new_roles)
+
+
+def before_inserting_users(user_docs):
+    """Grants organization roles to the created users."""
+
+    for user_doc in user_docs:
+        grant_org_roles(user_doc)
+
+
+def after_inserting_users(user_docs):
+    """Moves the users from the unknown_members to the members list of their organizations."""
+
+    om = current_app.org_manager
+    for user_doc in user_docs:
+        user_id = user_doc.get('_id')
+        user_email = user_doc.get('email')
+
+        if not user_id or not user_email:
+            # Missing emails can happen when creating a service account, it's fine.
+            log.info('User created with _id=%r and email=%r, unable to check organizations',
+                     user_id, user_email)
+            continue
+
+        om.make_member_known(user_id, user_email)

pillar/api/users/routes.py

@@ -1,9 +1,11 @@
 import logging
 
 from eve.methods.get import get
-from flask import g, Blueprint
+from flask import Blueprint
+
 from pillar.api.utils import jsonify
 from pillar.api.utils.authorization import require_login
+from pillar.auth import current_user
 
 log = logging.getLogger(__name__)
 blueprint_api = Blueprint('users_api', __name__)
@@ -12,7 +14,7 @@ blueprint_api = Blueprint('users_api', __name__)
 @blueprint_api.route('/me')
 @require_login()
 def my_info():
-    eve_resp, _, _, status, _ = get('users', {'_id': g.current_user['user_id']})
+    eve_resp, _, _, status, _ = get('users', {'_id': current_user.user_id})
     resp = jsonify(eve_resp['_items'][0], status=status)
     return resp
 

pillar/api/utils/__init__.py

@@ -1,13 +1,16 @@
+import base64
 import copy
-import hashlib
-import json
-import urllib
-
 import datetime
 import functools
+import hashlib
+import json
 import logging
+import random
+import typing
+import urllib.request, urllib.parse, urllib.error
 
 import bson.objectid
+import bson.tz_util
 from eve import RFC1123_DATE_FORMAT
 from flask import current_app
 from werkzeug import exceptions as wz_exceptions
@@ -103,10 +106,11 @@ def skip_when_testing(func):
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         if current_app.config['TESTING']:
-            log.debug('Skipping call to %s(...) due to TESTING', func.func_name)
+            log.debug('Skipping call to %s(...) due to TESTING', func.__name__)
             return None
 
         return func(*args, **kwargs)
+
     return wrapper
 
 
@@ -122,11 +126,9 @@ def project_get_node_type(project_document, node_type_node_name):
                  if node_type['name'] == node_type_node_name), None)
 
 
-def str2id(document_id):
+def str2id(document_id: str) -> bson.ObjectId:
     """Returns the document ID as ObjectID, or raises a BadRequest exception.
 
-    :type document_id: str
-    :rtype: bson.ObjectId
     :raises: wz_exceptions.BadRequest
     """
 
@@ -136,36 +138,41 @@ def str2id(document_id):
 
     try:
         return bson.ObjectId(document_id)
-    except bson.objectid.InvalidId:
+    except (bson.objectid.InvalidId, TypeError):
         log.debug('str2id(%r): Invalid Object ID', document_id)
         raise wz_exceptions.BadRequest('Invalid object ID %r' % document_id)
 
 
-def gravatar(email, size=64):
+def gravatar(email: str, size=64) -> typing.Optional[str]:
+    if email is None:
+        return None
+
     parameters = {'s': str(size), 'd': 'mm'}
     return "https://www.gravatar.com/avatar/" + \
-           hashlib.md5(str(email)).hexdigest() + \
-           "?" + urllib.urlencode(parameters)
-
+           hashlib.md5(email.encode()).hexdigest() + \
+           "?" + urllib.parse.urlencode(parameters)
 
 
 class MetaFalsey(type):
-    def __nonzero__(cls):
+    def __bool__(cls):
         return False
-    __bool__ = __nonzero__  # for Python 3
 
 
-class DoesNotExist(object):
+class DoesNotExistMeta(MetaFalsey):
+    def __repr__(cls) -> str:
+        return 'DoesNotExist'
+
+
+class DoesNotExist(object, metaclass=DoesNotExistMeta):
     """Returned as value by doc_diff if a value does not exist."""
-    __metaclass__ = MetaFalsey
 
 
-def doc_diff(doc1, doc2, falsey_is_equal=True):
+def doc_diff(doc1, doc2, *, falsey_is_equal=True, superkey: str = None):
     """Generator, yields differences between documents.
 
     Yields changes as (key, value in doc1, value in doc2) tuples, where
     the value can also be the DoesNotExist class. Does not report changed
-    private keys (i.e. starting with underscores).
+    private keys (i.e. the standard Eve keys starting with underscores).
 
     Sub-documents (i.e. dicts) are recursed, and dot notation is used
     for the keys if changes are found.
@@ -174,22 +181,68 @@ def doc_diff(doc1, doc2, falsey_is_equal=True):
     function won't report differences between DoesNotExist, False, '', and 0.
     """
 
+    private_keys = {'_id', '_etag', '_deleted', '_updated', '_created'}
+
+    def combine_key(some_key):
+        """Combine this key with the superkey.
+
+        Keep the key type the same, unless we have to combine with a superkey.
+        """
+        if not superkey:
+            return some_key
+        if isinstance(some_key, str) and some_key[0] == '[':
+            return f'{superkey}{some_key}'
+        return f'{superkey}.{some_key}'
+
+    if doc1 is doc2:
+        return
+
+    if falsey_is_equal and not bool(doc1) and not bool(doc2):
+        return
+
+    if isinstance(doc1, dict) and isinstance(doc2, dict):
         for key in set(doc1.keys()).union(set(doc2.keys())):
-        if isinstance(key, basestring) and key[0] == u'_':
+            if key in private_keys:
                 continue
 
             val1 = doc1.get(key, DoesNotExist)
             val2 = doc2.get(key, DoesNotExist)
 
-        # Only recurse if both values are dicts
-        if isinstance(val1, dict) and isinstance(val2, dict):
-            for subkey, subval1, subval2 in doc_diff(val1, val2):
-                yield '%s.%s' % (key, subkey), subval1, subval2
-            continue
+            yield from doc_diff(val1, val2,
+                                falsey_is_equal=falsey_is_equal,
+                                superkey=combine_key(key))
+        return
 
-        if val1 == val2:
-            continue
-        if falsey_is_equal and bool(val1) == bool(val2) == False:
-            continue
+    if isinstance(doc1, list) and isinstance(doc2, list):
+        for idx in range(max(len(doc1), len(doc2))):
+            try:
+                item1 = doc1[idx]
+            except IndexError:
+                item1 = DoesNotExist
+            try:
+                item2 = doc2[idx]
+            except IndexError:
+                item2 = DoesNotExist
 
-        yield key, val1, val2
+            subkey = f'[{idx}]'
+            if item1 is DoesNotExist or item2 is DoesNotExist:
+                yield combine_key(subkey), item1, item2
+            else:
+                yield from doc_diff(item1, item2,
+                                    falsey_is_equal=falsey_is_equal,
+                                    superkey=combine_key(subkey))
+        return
+
+    if doc1 != doc2:
+        yield superkey, doc1, doc2
+
+
+def random_etag() -> str:
+    """Random string usable as etag."""
+
+    randbytes = random.getrandbits(256).to_bytes(32, 'big')
+    return base64.b64encode(randbytes)[:-1].decode()
+
+
+def utcnow() -> datetime.datetime:
+    return datetime.datetime.now(tz=bson.tz_util.utc)

pillar/api/utils/algolia.py

@@ -1,101 +1,33 @@
 import logging
 
 from bson import ObjectId
-from flask import current_app
 
-from pillar.api.file_storage import generate_link
+from pillar import current_app
 from . import skip_when_testing
 
 log = logging.getLogger(__name__)
 
-INDEX_ALLOWED_USER_ROLES = {'admin', 'subscriber', 'demo'}
-INDEX_ALLOWED_NODE_TYPES = {'asset', 'texture', 'group', 'hdri'}
-
 
 @skip_when_testing
-def algolia_index_user_save(user):
-    if current_app.algolia_index_users is None:
+def index_user_save(to_index_user: dict):
+    index_users = current_app.algolia_index_users
+    if not index_users:
+        log.debug('No Algolia index defined, so nothing to do.')
         return
-    # Strip unneeded roles
-    if 'roles' in user:
-        roles = set(user['roles']).intersection(INDEX_ALLOWED_USER_ROLES)
-    else:
-        roles = set()
-    if current_app.algolia_index_users:
+
     # Create or update Algolia index for the user
-        current_app.algolia_index_users.save_object({
-            'objectID': user['_id'],
-            'full_name': user['full_name'],
-            'username': user['username'],
-            'roles': list(roles),
-            'groups': user['groups'],
-            'email': user['email']
-        })
+    index_users.save_object(to_index_user)
 
 
 @skip_when_testing
-def algolia_index_node_save(node):
+def index_node_save(node_to_index):
     if not current_app.algolia_index_nodes:
         return
-    if node['node_type'] not in INDEX_ALLOWED_NODE_TYPES:
-        return
-    # If a nodes does not have status published, do not index
-    if node['properties'].get('status') != 'published':
-        return
-
-    projects_collection = current_app.data.driver.db['projects']
-    project = projects_collection.find_one({'_id': ObjectId(node['project'])})
-
-    users_collection = current_app.data.driver.db['users']
-    user = users_collection.find_one({'_id': ObjectId(node['user'])})
-
-    node_ob = {
-        'objectID': node['_id'],
-        'name': node['name'],
-        'project': {
-            '_id': project['_id'],
-            'name': project['name']
-        },
-        'created': node['_created'],
-        'updated': node['_updated'],
-        'node_type': node['node_type'],
-        'user': {
-            '_id': user['_id'],
-            'full_name': user['full_name']
-        },
-    }
-    if 'description' in node and node['description']:
-        node_ob['description'] = node['description']
-    if 'picture' in node and node['picture']:
-        files_collection = current_app.data.driver.db['files']
-        lookup = {'_id': ObjectId(node['picture'])}
-        picture = files_collection.find_one(lookup)
-        if picture['backend'] == 'gcs':
-            variation_t = next((item for item in picture['variations'] \
-                                if item['size'] == 't'), None)
-            if variation_t:
-                node_ob['picture'] = generate_link(picture['backend'],
-                                                   variation_t['file_path'], project_id=str(picture['project']),
-                                                   is_public=True)
-    # If the node has world permissions, compute the Free permission
-    if 'permissions' in node and 'world' in node['permissions']:
-        if 'GET' in node['permissions']['world']:
-            node_ob['is_free'] = True
-
-    # Append the media key if the node is of node_type 'asset'
-    if node['node_type'] == 'asset':
-        node_ob['media'] = node['properties']['content_type']
-
-    # Add extra properties
-    for prop in ('tags', 'license_notes'):
-        if prop in node['properties']:
-            node_ob[prop] = node['properties'][prop]
-
-    current_app.algolia_index_nodes.save_object(node_ob)
+    current_app.algolia_index_nodes.save_object(node_to_index)
 
 
 @skip_when_testing
-def algolia_index_node_delete(node):
+def index_node_delete(delete_id):
     if current_app.algolia_index_nodes is None:
         return
-    current_app.algolia_index_nodes.delete_object(node['_id'])
+    current_app.algolia_index_nodes.delete_object(delete_id)

pillar/api/utils/authentication.py

@@ -5,21 +5,27 @@ unique usernames from emails. Calls out to the pillar_server.modules.blender_id
 module for Blender ID communication.
 """
 
-import logging
+import base64
 import datetime
+import hmac
+import hashlib
+import logging
+import typing
 
-from bson import tz_util
-from flask import g
+import bson
+from flask import g, current_app
 from flask import request
-from flask import current_app
+from werkzeug import exceptions as wz_exceptions
+
+from pillar.api.utils import remove_private_keys, utcnow
 
 log = logging.getLogger(__name__)
 
-CLI_USER = {
-    'user_id': 'CLI',
-    'groups': [],
-    'roles': {'admin'},
-}
+# Construction is done when requested, since constructing a UserClass instance
+# requires an application context to look up capabilities. We set the initial
+# value to a not-None singleton to be able to differentiate between
+# g.current_user set to "not logged in" or "uninitialised CLI_USER".
+CLI_USER = ...
 
 
 def force_cli_user():
@@ -28,11 +34,76 @@ def force_cli_user():
     This is used as a marker to avoid authorization checks and just allow everything.
     """
 
-    log.warning('Logging in as CLI_USER, circumventing authentication.')
+    global CLI_USER
+
+    from pillar.auth import UserClass
+
+    if CLI_USER is ...:
+        CLI_USER = UserClass.construct('CLI', {
+            '_id': 'CLI',
+            'groups': [],
+            'roles': {'admin'},
+            'email': 'local@nowhere',
+            'username': 'CLI',
+        })
+        log.info('CONSTRUCTED CLI USER %s of type %s', id(CLI_USER), id(type(CLI_USER)))
+
+    log.info('Logging in as CLI_USER (%s) of type %s, circumventing authentication.',
+             id(CLI_USER), id(type(CLI_USER)))
     g.current_user = CLI_USER
 
 
-def validate_token():
+def find_user_in_db(user_info: dict, provider='blender-id') -> dict:
+    """Find the user in our database, creating/updating the returned document where needed.
+
+    First, search for the user using its id from the provider, then try to look the user up via the
+    email address.
+
+    Does NOT update the user in the database.
+    
+    :param user_info: Information (id, email and full_name) from the auth provider
+    :param provider: One of the supported providers
+    """
+
+    users = current_app.data.driver.db['users']
+
+    user_id = user_info['id']
+    query = {'$or': [
+        {'auth': {'$elemMatch': {
+            'user_id': str(user_id),
+            'provider': provider}}},
+        {'email': user_info['email']},
+    ]}
+    log.debug('Querying: %s', query)
+    db_user = users.find_one(query)
+
+    if db_user:
+        log.debug('User with %s id %s already in our database, updating with info from %s',
+                  provider, user_id, provider)
+        db_user['email'] = user_info['email']
+
+        # Find out if an auth entry for the current provider already exists
+        provider_entry = [element for element in db_user['auth'] if element['provider'] == provider]
+        if not provider_entry:
+            db_user['auth'].append({
+                'provider': provider,
+                'user_id': str(user_id),
+                'token': ''})
+    else:
+        log.debug('User %r not yet in our database, create a new one.', user_id)
+        db_user = create_new_user_document(
+            email=user_info['email'],
+            user_id=user_id,
+            username=user_info['full_name'],
+            provider=provider)
+        db_user['username'] = make_unique_username(user_info['email'])
+        if not db_user['full_name']:
+            db_user['full_name'] = db_user['username']
+
+    return db_user
+
+
+def validate_token(*, force=False):
     """Validate the token provided in the request and populate the current_user
     flask.g object, so that permissions and access to a resource can be defined
     from it.
@@ -40,26 +111,38 @@ def validate_token():
     When the token is successfully validated, sets `g.current_user` to contain
     the user information, otherwise it is set to None.
 
-    @returns True iff the user is logged in with a valid Blender ID token.
+    :param force: don't trust g.current_user and force a re-check.
+    :returns: True iff the user is logged in with a valid Blender ID token.
     """
 
+    from pillar.auth import AnonymousUser
+
+    # Trust a pre-existing g.current_user
+    if not force:
+        cur = getattr(g, 'current_user', None)
+        if cur is not None and cur.is_authenticated:
+            log.debug('skipping token check because current user is already set to %s', cur)
+            return True
+
+    auth_header = request.headers.get('Authorization') or ''
     if request.authorization:
         token = request.authorization.username
         oauth_subclient = request.authorization.password
+    elif auth_header.startswith('Bearer '):
+        token = auth_header[7:].strip()
+        oauth_subclient = ''
     else:
         # Check the session, the user might be logged in through Flask-Login.
         from pillar import auth
 
         token = auth.get_blender_id_oauth_token()
-        if token and isinstance(token, (tuple, list)):
-            token = token[0]
         oauth_subclient = None
 
     if not token:
         # If no authorization headers are provided, we are getting a request
         # from a non logged in user. Proceed accordingly.
         log.debug('No authentication headers, so not logged in.')
-        g.current_user = None
+        g.current_user = AnonymousUser()
         return False
 
     return validate_this_token(token, oauth_subclient) is not None
@@ -72,13 +155,15 @@ def validate_this_token(token, oauth_subclient=None):
     :rtype: dict
     """
 
+    from pillar.auth import UserClass, AnonymousUser, user_authenticated
+
     g.current_user = None
     _delete_expired_tokens()
 
     # Check the users to see if there is one with this Blender ID token.
     db_token = find_token(token, oauth_subclient)
     if not db_token:
-        log.debug('Token %s not found in our local database.', token)
+        log.debug('Token %r not found in our local database.', token)
 
         # If no valid token is found in our local database, we issue a new
         # request to the Blender ID server to verify the validity of the token
@@ -94,45 +179,74 @@ def validate_this_token(token, oauth_subclient=None):
 
     if db_user is None:
         log.debug('Validation failed, user not logged in')
+        g.current_user = AnonymousUser()
         return None
 
-    g.current_user = {'user_id': db_user['_id'],
-                      'groups': db_user['groups'],
-                      'roles': set(db_user.get('roles', []))}
+    g.current_user = UserClass.construct(token, db_user)
+    user_authenticated.send(None)
 
     return db_user
 
 
+def remove_token(token: str):
+    """Removes the token from the database."""
+
+    tokens_coll = current_app.db('tokens')
+    token_hashed = hash_auth_token(token)
+
+    # TODO: remove matching on unhashed tokens once all tokens have been hashed.
+    lookup = {'$or': [{'token': token}, {'token_hashed': token_hashed}]}
+    del_res = tokens_coll.delete_many(lookup)
+    log.debug('Removed token %r, matched %d documents', token, del_res.deleted_count)
+
+
 def find_token(token, is_subclient_token=False, **extra_filters):
     """Returns the token document, or None if it doesn't exist (or is expired)."""
 
-    tokens_collection = current_app.data.driver.db['tokens']
+    tokens_coll = current_app.db('tokens')
+    token_hashed = hash_auth_token(token)
 
-    # TODO: remove expired tokens from collection.
-    lookup = {'token': token,
+    # TODO: remove matching on unhashed tokens once all tokens have been hashed.
+    lookup = {'$or': [{'token': token}, {'token_hashed': token_hashed}],
               'is_subclient_token': True if is_subclient_token else {'$in': [False, None]},
-              'expire_time': {"$gt": datetime.datetime.now(tz=tz_util.utc)}}
+              'expire_time': {"$gt": utcnow()}}
     lookup.update(extra_filters)
 
-    db_token = tokens_collection.find_one(lookup)
+    db_token = tokens_coll.find_one(lookup)
     return db_token
 
 
-def store_token(user_id, token, token_expiry, oauth_subclient_id=False):
+def hash_auth_token(token: str) -> str:
+    """Returns the hashed authentication token.
+
+    The token is hashed using HMAC and then base64-encoded.
+    """
+
+    hmac_key = current_app.config['AUTH_TOKEN_HMAC_KEY']
+    token_hmac = hmac.new(hmac_key, msg=token.encode('utf8'), digestmod=hashlib.sha256)
+    digest = token_hmac.digest()
+
+    return base64.b64encode(digest).decode('ascii')
+
+
+def store_token(user_id, token: str, token_expiry, oauth_subclient_id=False,
+                org_roles: typing.Set[str] = frozenset()):
     """Stores an authentication token.
 
     :returns: the token document from MongoDB
     """
 
-    assert isinstance(token, (str, unicode)), 'token must be string type, not %r' % type(token)
+    assert isinstance(token, str), 'token must be string type, not %r' % type(token)
 
     token_data = {
         'user': user_id,
-        'token': token,
+        'token_hashed': hash_auth_token(token),
         'expire_time': token_expiry,
     }
     if oauth_subclient_id:
         token_data['is_subclient_token'] = True
+    if org_roles:
+        token_data['org_roles'] = sorted(org_roles)
 
     r, _, _, status = current_app.post_internal('tokens', token_data)
 
@@ -160,13 +274,13 @@ def create_new_user(email, username, user_id):
 
 
 def create_new_user_document(email, user_id, username, provider='blender-id',
-                             token=''):
+                             token='', *, full_name=''):
     """Creates a new user document, without storing it in MongoDB. The token
     parameter is a password in case provider is "local".
     """
 
     user_data = {
-        'full_name': username,
+        'full_name': full_name or username,
         'username': username,
         'email': email,
         'auth': [{
@@ -219,22 +333,99 @@ def _delete_expired_tokens():
 
     token_coll = current_app.data.driver.db['tokens']
 
-    now = datetime.datetime.now(tz_util.utc)
-    expiry_date = now - datetime.timedelta(days=7)
-
+    expiry_date = utcnow() - datetime.timedelta(days=7)
     result = token_coll.delete_many({'expire_time': {"$lt": expiry_date}})
     # log.debug('Deleted %i expired authentication tokens', result.deleted_count)
 
 
-def current_user_id():
+def current_user_id() -> typing.Optional[bson.ObjectId]:
     """None-safe fetching of user ID. Can return None itself, though."""
 
-    current_user = g.get('current_user') or {}
-    return current_user.get('user_id')
+    user = current_user()
+    return user.user_id
+
+
+def current_user():
+    """Returns the current user, or an AnonymousUser if not logged in.
+
+    :rtype: pillar.auth.UserClass
+    """
+
+    import pillar.auth
+
+    user: pillar.auth.UserClass = g.get('current_user')
+    if user is None:
+        return pillar.auth.AnonymousUser()
+
+    return user
 
 
 def setup_app(app):
     @app.before_request
     def validate_token_at_each_request():
         validate_token()
-        return None
+
+
+def upsert_user(db_user):
+    """Inserts/updates the user in MongoDB.
+
+    Retries a few times when there are uniqueness issues in the username.
+
+    :returns: the user's database ID and the status of the PUT/POST.
+        The status is 201 on insert, and 200 on update.
+    :type: (ObjectId, int)
+    """
+
+    if 'subscriber' in db_user.get('groups', []):
+        log.error('Non-ObjectID string found in user.groups: %s', db_user)
+        raise wz_exceptions.InternalServerError(
+            'Non-ObjectID string found in user.groups: %s' % db_user)
+
+    if not db_user['full_name']:
+        # Blender ID doesn't need a full name, but we do.
+        db_user['full_name'] = db_user['username']
+
+    r = {}
+    for retry in range(5):
+        if '_id' in db_user:
+            # Update the existing user
+            attempted_eve_method = 'PUT'
+            db_id = db_user['_id']
+            r, _, _, status = current_app.put_internal('users', remove_private_keys(db_user),
+                                                       _id=db_id)
+            if status == 422:
+                log.error('Status %i trying to PUT user %s with values %s, should not happen! %s',
+                          status, db_id, remove_private_keys(db_user), r)
+        else:
+            # Create a new user, retry for non-unique usernames.
+            attempted_eve_method = 'POST'
+            r, _, _, status = current_app.post_internal('users', db_user)
+
+            if status not in {200, 201}:
+                log.error('Status %i trying to create user with values %s: %s',
+                          status, db_user, r)
+                raise wz_exceptions.InternalServerError()
+
+            db_id = r['_id']
+            db_user.update(r)  # update with database/eve-generated fields.
+
+        if status == 422:
+            # Probably non-unique username, so retry a few times with different usernames.
+            log.info('Error creating new user: %s', r)
+            username_issue = r.get('_issues', {}).get('username', '')
+            if 'not unique' in username_issue:
+                # Retry
+                db_user['username'] = make_unique_username(db_user['email'])
+                continue
+
+        # Saving was successful, or at least didn't break on a non-unique username.
+        break
+    else:
+        log.error('Unable to create new user %s: %s', db_user, r)
+        raise wz_exceptions.InternalServerError()
+
+    if status not in (200, 201):
+        log.error('internal response from %s to Eve: %r %r', attempted_eve_method, status, r)
+        raise wz_exceptions.InternalServerError()
+
+    return db_id, status

pillar/api/utils/authorization.py

@@ -27,6 +27,12 @@ def check_permissions(collection_name, resource, method, append_allowed_methods=
     :param check_node_type: node type to check. Only valid when collection_name='projects'.
     :type check_node_type: str
     """
+    from pillar.auth import get_current_user
+    from .authentication import CLI_USER
+
+    if get_current_user() is CLI_USER:
+        log.debug('Short-circuiting check_permissions() for CLI user')
+        return
 
     if not has_permissions(collection_name, resource, method, append_allowed_methods,
                            check_node_type):
@@ -45,6 +51,8 @@ def compute_allowed_methods(collection_name, resource, check_node_type=None):
     :rtype: set
     """
 
+    import pillar.auth
+
     # Check some input values.
     if collection_name not in CHECK_PERMISSIONS_IMPLEMENTED_FOR:
         raise ValueError('compute_allowed_methods only implemented for %s, not for %s',
@@ -62,18 +70,18 @@ def compute_allowed_methods(collection_name, resource, check_node_type=None):
 
     # Accumulate allowed methods from the user, group and world level.
     allowed_methods = set()
-    current_user = getattr(g, 'current_user', None)
+    user = pillar.auth.get_current_user()
 
-    if current_user:
-        user_is_admin = is_admin(current_user)
+    if user.is_authenticated:
+        user_is_admin = is_admin(user)
 
         # If the user is authenticated, proceed to compare the group permissions
         for permission in computed_permissions.get('groups', ()):
-            if user_is_admin or permission['group'] in current_user['groups']:
+            if user_is_admin or permission['group'] in user.group_ids:
                 allowed_methods.update(permission['methods'])
 
         for permission in computed_permissions.get('users', ()):
-            if user_is_admin or current_user['user_id'] == permission['user']:
+            if user_is_admin or user.user_id == permission['user']:
                 allowed_methods.update(permission['methods'])
 
     # Check if the node is public or private. This must be set for non logged
@@ -214,6 +222,8 @@ def merge_permissions(*args):
     :returns: combined list of permissions.
     """
 
+    from pillar.auth import current_user
+
     if not args:
         return {}
 
@@ -235,25 +245,35 @@ def merge_permissions(*args):
         from0 = args[0].get(plural_name, [])
         from1 = args[1].get(plural_name, [])
 
+        try:
             asdict0 = {permission[field_name]: permission['methods'] for permission in from0}
+        except KeyError:
+            log.exception('KeyError creating asdict0 for %r permissions; user=%s; args[0]=%r',
+                          field_name, current_user.user_id, args[0])
+            asdict0 = {}
+        try:
             asdict1 = {permission[field_name]: permission['methods'] for permission in from1}
+        except KeyError:
+            log.exception('KeyError creating asdict1 for %r permissions; user=%s; args[1]=%r',
+                          field_name, current_user.user_id, args[1])
+            asdict1 = {}
 
-        keys = set(asdict0.keys() + asdict1.keys())
+        keys = set(asdict0.keys()).union(set(asdict1.keys()))
         for key in maybe_sorted(keys):
             methods0 = asdict0.get(key, [])
             methods1 = asdict1.get(key, [])
             methods = maybe_sorted(set(methods0).union(set(methods1)))
-            effective.setdefault(plural_name, []).append({field_name: key, u'methods': methods})
+            effective.setdefault(plural_name, []).append({field_name: key, 'methods': methods})
 
-    merge(u'user')
-    merge(u'group')
+    merge('user')
+    merge('group')
 
     # Gather permissions for world
     world0 = args[0].get('world', [])
     world1 = args[1].get('world', [])
     world_methods = set(world0).union(set(world1))
     if world_methods:
-        effective[u'world'] = maybe_sorted(world_methods)
+        effective['world'] = maybe_sorted(world_methods)
 
     # Recurse for longer merges
     if len(args) > 2:
@@ -262,39 +282,83 @@ def merge_permissions(*args):
     return effective
 
 
-def require_login(require_roles=set(),
-                  require_all=False):
+def require_login(*, require_roles=set(),
+                  require_cap='',
+                  require_all=False,
+                  redirect_to_login=False,
+                  error_view=None):
     """Decorator that enforces users to authenticate.
 
-    Optionally only allows access to users with a certain role.
+    Optionally only allows access to users with a certain role and/or capability.
+
+    Either check on roles or on a capability, but never on both. There is no
+    require_all check for capabilities; if you need to check for multiple
+    capabilities at once, it's a sign that you need to add another capability
+    and give it to everybody that needs it.
 
     :param require_roles: set of roles.
+    :param require_cap: a capability.
     :param require_all:
         When False (the default): if the user's roles have a
         non-empty intersection with the given roles, access is granted.
         When True: require the user to have all given roles before access is
         granted.
+    :param redirect_to_login: Determines the behaviour when the user is not
+        logged in. When False (the default), a 403 Forbidden response is
+        returned; this is suitable for API calls. When True, the user is
+        redirected to the login page; this is suitable for user-facing web
+        requests, and mimicks the flask_login behaviour.
+    :param error_view: Callable that returns a Flask response object. This is
+        sent back to the client instead of the default 403 Forbidden.
     """
 
+    from flask import request, redirect, url_for, Response
+
     if not isinstance(require_roles, set):
-        raise TypeError('require_roles param should be a set, but is a %r' % type(require_roles))
+        raise TypeError(f'require_roles param should be a set, but is {type(require_roles)!r}')
+
+    if not isinstance(require_cap, str):
+        raise TypeError(f'require_caps param should be a str, but is {type(require_cap)!r}')
+
+    if require_roles and require_cap:
+        raise ValueError('either use require_roles or require_cap, but not both')
 
     if require_all and not require_roles:
         raise ValueError('require_login(require_all=True) cannot be used with empty require_roles.')
 
+    def render_error() -> Response:
+        if error_view is None:
+            abort(403)
+        resp: Response = error_view()
+        resp.status_code = 403
+        return resp
+
     def decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
-            if not user_matches_roles(require_roles, require_all):
-                if g.current_user is None:
+            import pillar.auth
+
+            current_user = pillar.auth.get_current_user()
+            if current_user.is_anonymous:
                 # We don't need to log at a higher level, as this is very common.
                 # Many browsers first try to see whether authentication is needed
                 # at all, before sending the password.
-                    log.debug('Unauthenticated acces to %s attempted.', func)
-                else:
-                    log.warning('User %s is authenticated, but does not have required roles %s to '
-                                'access %s', g.current_user['user_id'], require_roles, func)
-                abort(403)
+                log.debug('Unauthenticated access to %s attempted.', func)
+                if redirect_to_login:
+                    # Redirect using a 303 See Other, since even a POST
+                    # request should cause a GET on the login page.
+                    return redirect(url_for('users.login', next=request.url), 303)
+                return render_error()
+
+            if require_roles and not current_user.matches_roles(require_roles, require_all):
+                log.info('User %s is authenticated, but does not have required roles %s to '
+                         'access %s', current_user.user_id, require_roles, func)
+                return render_error()
+
+            if require_cap and not current_user.has_cap(require_cap):
+                log.info('User %s is authenticated, but does not have required capability %s to '
+                         'access %s', current_user.user_id, require_cap, func)
+                return render_error()
 
             return func(*args, **kwargs)
 
@@ -337,14 +401,36 @@ def ab_testing(require_roles=set(),
 def user_has_role(role, user=None):
     """Returns True iff the user is logged in and has the given role."""
 
-    if user is None:
-        user = g.get('current_user')
+    import pillar.auth
 
     if user is None:
+        user = pillar.auth.get_current_user()
+        if user is not None and not isinstance(user, pillar.auth.UserClass):
+            raise TypeError(f'pillar.auth.current_user should be instance of UserClass, '
+                            f'not {type(user)}')
+    elif not isinstance(user, pillar.auth.UserClass):
+        raise TypeError(f'user should be instance of UserClass, not {type(user)}')
+
+    if user.is_anonymous:
         return False
 
-    roles = user.get('roles') or ()
-    return role in roles
+    return user.has_role(role)
+
+
+def user_has_cap(capability: str, user=None) -> bool:
+    """Returns True iff the user is logged in and has the given capability."""
+
+    import pillar.auth
+
+    assert capability
+
+    if user is None:
+        user = pillar.auth.get_current_user()
+
+    if not isinstance(user, pillar.auth.UserClass):
+        raise TypeError(f'user should be instance of UserClass, not {type(user)}')
+
+    return user.has_cap(capability)
 
 
 def user_matches_roles(require_roles=set(),
@@ -359,25 +445,16 @@ def user_matches_roles(require_roles=set(),
         returning True.
     """
 
-    if not isinstance(require_roles, set):
-        raise TypeError('require_roles param should be a set, but is a %r' % type(require_roles))
+    import pillar.auth
 
-    if require_all and not require_roles:
-        raise ValueError('require_login(require_all=True) cannot be used with empty require_roles.')
+    user = pillar.auth.get_current_user()
+    if not isinstance(user, pillar.auth.UserClass):
+        raise TypeError(f'user should be instance of UserClass, not {type(user)}')
 
-    current_user = g.get('current_user')
-
-    if current_user is None:
-        return False
-
-    intersection = require_roles.intersection(current_user['roles'])
-    if require_all:
-        return len(intersection) == len(require_roles)
-
-    return not bool(require_roles) or bool(intersection)
+    return user.matches_roles(require_roles, require_all)
 
 
 def is_admin(user):
-    """Returns True iff the given user has the admin role."""
+    """Returns True iff the given user has the admin capability."""
 
-    return user_has_role(u'admin', user)
+    return user_has_cap('admin', user)

pillar/api/utils/cdn.py

@@ -1,5 +1,7 @@
 import datetime
 from hashlib import md5
+import base64
+
 from flask import current_app
 
 
@@ -17,19 +19,20 @@ def hash_file_path(file_path, expiry_timestamp=None):
     if current_app.config['CDN_USE_URL_SIGNING']:
 
         url_signing_key = current_app.config['CDN_URL_SIGNING_KEY']
-        hash_string = domain_subfolder + file_path + url_signing_key
+        to_hash = domain_subfolder + file_path + url_signing_key
 
         if not expiry_timestamp:
             expiry_timestamp = datetime.datetime.now() + datetime.timedelta(hours=24)
             expiry_timestamp = expiry_timestamp.strftime('%s')
 
-        hash_string = expiry_timestamp + hash_string
+        to_hash = expiry_timestamp + to_hash
+        if isinstance(to_hash, str):
+            to_hash = to_hash.encode()
 
         expiry_timestamp = "," + str(expiry_timestamp)
 
-        hashed_file_path = md5(hash_string).digest().encode('base64')[:-1]
-        hashed_file_path = hashed_file_path.replace('+', '-')
-        hashed_file_path = hashed_file_path.replace('/', '_')
+        hashed_file_path = base64.b64encode(md5(to_hash).digest())[:-1].decode()
+        hashed_file_path = hashed_file_path.replace('+', '-').replace('/', '_')
 
         asset_url = asset_url + \
                     '?secure=' + \

pillar/api/utils/encoding.py

@@ -31,7 +31,10 @@ class Encoder:
         options = dict(notifications=current_app.config['ZENCODER_NOTIFICATIONS_URL'])
 
         outputs = [{'format': v['format'],
-                    'url': os.path.join(storage_base, v['file_path'])}
+                    'url': os.path.join(storage_base, v['file_path']),
+                    'upscale': False,
+                    'size': '{width}x{height}'.format(**v),
+                    }
                    for v in src_file['variations']]
         r = current_app.encoding_service_client.job.create(file_input,
                                                            outputs=outputs,

pillar/api/utils/gcs.py

@@ -1,251 +0,0 @@
-import os
-import time
-import datetime
-import logging
-
-from bson import ObjectId
-from gcloud.storage.client import Client
-from gcloud.exceptions import NotFound
-from flask import current_app, g
-from werkzeug.local import LocalProxy
-
-log = logging.getLogger(__name__)
-
-
-def get_client():
-    """Stores the GCS client on the global Flask object.
-
-    The GCS client is not user-specific anyway.
-
-    :rtype: Client
-    """
-
-    _gcs = getattr(g, '_gcs_client', None)
-    if _gcs is None:
-        _gcs = g._gcs_client = Client()
-    return _gcs
-
-
-# This hides the specifics of how/where we store the GCS client,
-# and allows the rest of the code to use 'gcs' as a simple variable
-# that does the right thing.
-gcs = LocalProxy(get_client)
-
-
-class GoogleCloudStorageBucket(object):
-    """Cloud Storage bucket interface. We create a bucket for every project. In
-    the bucket we create first level subdirs as follows:
-    - '_' (will contain hashed assets, and stays on top of default listing)
-    - 'svn' (svn checkout mirror)
-    - 'shared' (any additional folder of static folder that is accessed via a
-      node of 'storage' node_type)
-
-    :type bucket_name: string
-    :param bucket_name: Name of the bucket.
-
-    :type subdir: string
-    :param subdir: The local entry point to browse the bucket.
-
-    """
-
-    def __init__(self, bucket_name, subdir='_/'):
-        try:
-            self.bucket = gcs.get_bucket(bucket_name)
-        except NotFound:
-            self.bucket = gcs.bucket(bucket_name)
-            # Hardcode the bucket location to EU
-            self.bucket.location = 'EU'
-            # Optionally enable CORS from * (currently only used for vrview)
-            # self.bucket.cors = [
-            #     {
-            #       "origin": ["*"],
-            #       "responseHeader": ["Content-Type"],
-            #       "method": ["GET", "HEAD", "DELETE"],
-            #       "maxAgeSeconds": 3600
-            #     }
-            # ]
-            self.bucket.create()
-
-        self.subdir = subdir
-
-    def List(self, path=None):
-        """Display the content of a subdir in the project bucket. If the path
-        points to a file the listing is simply empty.
-
-        :type path: string
-        :param path: The relative path to the directory or asset.
-        """
-        if path and not path.endswith('/'):
-            path += '/'
-        prefix = os.path.join(self.subdir, path)
-
-        fields_to_return = 'nextPageToken,items(name,size,contentType),prefixes'
-        req = self.bucket.list_blobs(fields=fields_to_return, prefix=prefix,
-                                     delimiter='/')
-
-        files = []
-        for f in req:
-            filename = os.path.basename(f.name)
-            if filename != '':  # Skip own folder name
-                files.append(dict(
-                    path=os.path.relpath(f.name, self.subdir),
-                    text=filename,
-                    type=f.content_type))
-
-        directories = []
-        for dir_path in req.prefixes:
-            directory_name = os.path.basename(os.path.normpath(dir_path))
-            directories.append(dict(
-                text=directory_name,
-                path=os.path.relpath(dir_path, self.subdir),
-                type='group_storage',
-                children=True))
-            # print os.path.basename(os.path.normpath(path))
-
-        list_dict = dict(
-            name=os.path.basename(os.path.normpath(path)),
-            type='group_storage',
-            children=files + directories
-        )
-
-        return list_dict
-
-    def blob_to_dict(self, blob):
-        blob.reload()
-        expiration = datetime.datetime.now() + datetime.timedelta(days=1)
-        expiration = int(time.mktime(expiration.timetuple()))
-        return dict(
-            updated=blob.updated,
-            name=os.path.basename(blob.name),
-            size=blob.size,
-            content_type=blob.content_type,
-            signed_url=blob.generate_signed_url(expiration),
-            public_url=blob.public_url)
-
-    def Get(self, path, to_dict=True):
-        """Get selected file info if the path matches.
-
-        :type path: string
-        :param path: The relative path to the file.
-        :type to_dict: bool
-        :param to_dict: Return the object as a dictionary.
-        """
-        path = os.path.join(self.subdir, path)
-        blob = self.bucket.blob(path)
-        if blob.exists():
-            if to_dict:
-                return self.blob_to_dict(blob)
-            else:
-                return blob
-        else:
-            return None
-
-    def Post(self, full_path, path=None):
-        """Create new blob and upload data to it.
-        """
-        path = path if path else os.path.join('_', os.path.basename(full_path))
-        blob = self.bucket.blob(path)
-        if blob.exists():
-            return None
-        blob.upload_from_filename(full_path)
-        return blob
-        # return self.blob_to_dict(blob) # Has issues with threading
-
-    def Delete(self, path):
-        """Delete blob (when removing an asset or replacing a preview)"""
-
-        # We want to get the actual blob to delete
-        blob = self.Get(path, to_dict=False)
-        try:
-            blob.delete()
-            return True
-        except NotFound:
-            return None
-
-    def update_name(self, blob, name):
-        """Set the ContentDisposition metadata so that when a file is downloaded
-        it has a human-readable name.
-        """
-        blob.content_disposition = u'attachment; filename="{0}"'.format(name)
-        blob.patch()
-
-    def copy_blob(self, blob, to_bucket):
-        """Copies the given blob from this bucket to the other bucket.
-
-        Returns the new blob.
-        """
-
-        assert isinstance(to_bucket, GoogleCloudStorageBucket)
-        return self.bucket.copy_blob(blob, to_bucket.bucket)
-
-
-def update_file_name(node):
-    """Assign to the CGS blob the same name of the asset node. This way when
-    downloading an asset we get a human-readable name.
-    """
-
-    # Process only files that are not processing
-    if node['properties'].get('status', '') == 'processing':
-        return
-
-    def _format_name(name, override_ext, size=None, map_type=u''):
-        root, _ = os.path.splitext(name)
-        size = u'-{}'.format(size) if size else u''
-        map_type = u'-{}'.format(map_type) if map_type else u''
-        return u'{}{}{}{}'.format(root, size, map_type, override_ext)
-
-    def _update_name(file_id, file_props):
-        files_collection = current_app.data.driver.db['files']
-        file_doc = files_collection.find_one({'_id': ObjectId(file_id)})
-
-        if file_doc is None or file_doc.get('backend') != 'gcs':
-            return
-
-        # For textures -- the map type should be part of the name.
-        map_type = file_props.get('map_type', u'')
-
-        storage = GoogleCloudStorageBucket(str(node['project']))
-        blob = storage.Get(file_doc['file_path'], to_dict=False)
-        if blob is None:
-            log.warning('Unable to find blob for file %s in project %s',
-                        file_doc['file_path'], file_doc['project'])
-            return
-
-        # Pick file extension from original filename
-        _, ext = os.path.splitext(file_doc['filename'])
-        name = _format_name(node['name'], ext, map_type=map_type)
-        storage.update_name(blob, name)
-
-        # Assign the same name to variations
-        for v in file_doc.get('variations', []):
-            _, override_ext = os.path.splitext(v['file_path'])
-            name = _format_name(node['name'], override_ext, v['size'], map_type=map_type)
-            blob = storage.Get(v['file_path'], to_dict=False)
-            if blob is None:
-                log.info('Unable to find blob for file %s in project %s. This can happen if the '
-                         'video encoding is still processing.', v['file_path'], node['project'])
-                continue
-            storage.update_name(blob, name)
-
-    # Currently we search for 'file' and 'files' keys in the object properties.
-    # This could become a bit more flexible and realy on a true reference of the
-    # file object type from the schema.
-    if 'file' in node['properties']:
-        _update_name(node['properties']['file'], {})
-
-    if 'files' in node['properties']:
-        for file_props in node['properties']['files']:
-            _update_name(file_props['file'], file_props)
-
-
-def copy_to_bucket(file_path, src_project_id, dest_project_id):
-    """Copies a file from one bucket to the other."""
-
-    log.info('Copying %s from project bucket %s to %s',
-             file_path, src_project_id, dest_project_id)
-
-    src_storage = GoogleCloudStorageBucket(str(src_project_id))
-    dest_storage = GoogleCloudStorageBucket(str(dest_project_id))
-
-    blob = src_storage.Get(file_path, to_dict=False)
-    src_storage.copy_blob(blob, dest_storage)

pillar/api/utils/imaging.py

@@ -1,47 +1,61 @@
-import os
 import json
+import typing
+
+import os
+import pathlib
 import subprocess
+
 from PIL import Image
 from flask import current_app
 
+# Images with these modes will be thumbed to PNG, others to JPEG.
+MODES_FOR_PNG = {'RGBA', 'LA'}
 
-def generate_local_thumbnails(name_base, src):
+
+def generate_local_thumbnails(fp_base: str, src: pathlib.Path):
     """Given a source image, use Pillow to generate thumbnails according to the
     application settings.
 
-    :param name_base: the thumbnail will get a field 'name': '{basename}-{thumbsize}.jpg'
-    :type name_base: str
+    :param fp_base: the thumbnail will get a field
+        'file_path': '{fp_base}-{thumbsize}.{ext}'
     :param src: the path of the image to be thumbnailed
-    :type src: str
     """
 
     thumbnail_settings = current_app.config['UPLOADS_LOCAL_STORAGE_THUMBNAILS']
     thumbnails = []
 
-    save_to_base, _ = os.path.splitext(src)
-    name_base, _ = os.path.splitext(name_base)
+    for size, settings in thumbnail_settings.items():
+        im = Image.open(src)
+        extra_args = {}
 
-    for size, settings in thumbnail_settings.iteritems():
-        dst = '{0}-{1}{2}'.format(save_to_base, size, '.jpg')
-        name = '{0}-{1}{2}'.format(name_base, size, '.jpg')
+        # If the source image has transparency, save as PNG
+        if im.mode in MODES_FOR_PNG:
+            suffix = '.png'
+            imformat = 'PNG'
+        else:
+            suffix = '.jpg'
+            imformat = 'JPEG'
+            extra_args = {'quality': 95}
+        dst = src.with_name(f'{src.stem}-{size}{suffix}')
 
         if settings['crop']:
-            resize_and_crop(src, dst, settings['size'])
-            width, height = settings['size']
+            im = resize_and_crop(im, settings['size'])
         else:
-            im = Image.open(src).convert('RGB')
-            im.thumbnail(settings['size'])
-            im.save(dst, "JPEG")
+            im.thumbnail(settings['size'], resample=Image.LANCZOS)
         width, height = im.size
 
+        if imformat == 'JPEG':
+            im = im.convert('RGB')
+        im.save(dst, format=imformat, optimize=True, **extra_args)
+
         thumb_info = {'size': size,
-                      'file_path': name,
-                      'local_path': dst,
-                      'length': os.stat(dst).st_size,
+                      'file_path': f'{fp_base}-{size}{suffix}',
+                      'local_path': str(dst),
+                      'length': dst.stat().st_size,
                       'width': width,
                       'height': height,
                       'md5': '',
-                      'content_type': 'image/jpeg'}
+                      'content_type': f'image/{imformat.lower()}'}
 
         if size == 't':
             thumb_info['is_public'] = True
@@ -51,63 +65,40 @@ def generate_local_thumbnails(name_base, src):
     return thumbnails
 
 
-def resize_and_crop(img_path, modified_path, size, crop_type='middle'):
-    """
-    Resize and crop an image to fit the specified size. Thanks to:
-    https://gist.github.com/sigilioso/2957026
+def resize_and_crop(img: Image, size: typing.Tuple[int, int]) -> Image:
+    """Resize and crop an image to fit the specified size.
 
-    args:
-    img_path: path for the image to resize.
-    modified_path: path to store the modified image.
-    size: `(width, height)` tuple.
-    crop_type: can be 'top', 'middle' or 'bottom', depending on this
-    value, the image will cropped getting the 'top/left', 'middle' or
-    'bottom/right' of the image to fit the size.
-    raises:
-    Exception: if can not open the file in img_path of there is problems
-    to save the image.
-    ValueError: if an invalid `crop_type` is provided.
+    Thanks to: https://gist.github.com/sigilioso/2957026
 
+    :param img: opened PIL.Image to work on
+    :param size: `(width, height)` tuple.
     """
     # If height is higher we resize vertically, if not we resize horizontally
-    img = Image.open(img_path).convert('RGB')
     # Get current and desired ratio for the images
-    img_ratio = img.size[0] / float(img.size[1])
-    ratio = size[0] / float(size[1])
+    cur_w, cur_h = img.size  # current
+    img_ratio = cur_w / cur_h
+
+    w, h = size  # desired
+    ratio = w / h
+
     # The image is scaled/cropped vertically or horizontally depending on the ratio
     if ratio > img_ratio:
-        img = img.resize((size[0], int(round(size[0] * img.size[1] / img.size[0]))),
-                         Image.ANTIALIAS)
-        # Crop in the top, middle or bottom
-        if crop_type == 'top':
-            box = (0, 0, img.size[0], size[1])
-        elif crop_type == 'middle':
-            box = (0, int(round((img.size[1] - size[1]) / 2)), img.size[0],
-                   int(round((img.size[1] + size[1]) / 2)))
-        elif crop_type == 'bottom':
-            box = (0, img.size[1] - size[1], img.size[0], img.size[1])
-        else:
-            raise ValueError('ERROR: invalid value for crop_type')
+        uncropped_h = (w * cur_h) // cur_w
+        img = img.resize((w, uncropped_h), Image.ANTIALIAS)
+        box = (0, (uncropped_h - h) // 2,
+               w, (uncropped_h + h) // 2)
         img = img.crop(box)
     elif ratio < img_ratio:
-        img = img.resize((int(round(size[1] * img.size[0] / img.size[1])), size[1]),
-                         Image.ANTIALIAS)
-        # Crop in the top, middle or bottom
-        if crop_type == 'top':
-            box = (0, 0, size[0], img.size[1])
-        elif crop_type == 'middle':
-            box = (int(round((img.size[0] - size[0]) / 2)), 0,
-                   int(round((img.size[0] + size[0]) / 2)), img.size[1])
-        elif crop_type == 'bottom':
-            box = (img.size[0] - size[0], 0, img.size[0], img.size[1])
-        else:
-            raise ValueError('ERROR: invalid value for crop_type')
+        uncropped_w = (h * cur_w) // cur_h
+        img = img.resize((uncropped_w, h), Image.ANTIALIAS)
+        box = ((uncropped_w - w) // 2, 0,
+               (uncropped_w + w) // 2, h)
         img = img.crop(box)
     else:
-        img = img.resize((size[0], size[1]),
-                         Image.ANTIALIAS)
+        img = img.resize((w, h), Image.ANTIALIAS)
+
     # If the scale is the same, we do not need to crop
-    img.save(modified_path, "JPEG")
+    return img
 
 
 def get_video_data(filepath):
@@ -143,7 +134,7 @@ def get_video_data(filepath):
             res_y=video_stream['height'],
         )
         if video_stream['sample_aspect_ratio'] != '1:1':
-            print '[warning] Pixel aspect ratio is not square!'
+            print('[warning] Pixel aspect ratio is not square!')
 
     return outdata
 
@@ -190,14 +181,14 @@ def ffmpeg_encode(src, format, res_y=720):
     dst = os.path.splitext(src)
     dst = "{0}-{1}p.{2}".format(dst[0], res_y, format)
     args.append(dst)
-    print "Encoding {0} to {1}".format(src, format)
+    print("Encoding {0} to {1}".format(src, format))
     returncode = subprocess.call([current_app.config['BIN_FFMPEG']] + args)
     if returncode == 0:
-        print "Successfully encoded {0}".format(dst)
+        print("Successfully encoded {0}".format(dst))
     else:
-        print "Error during encode"
-        print "Code:    {0}".format(returncode)
-        print "Command: {0}".format(current_app.config['BIN_FFMPEG'] + " " + " ".join(args))
+        print("Error during encode")
+        print("Code:    {0}".format(returncode))
+        print("Command: {0}".format(current_app.config['BIN_FFMPEG'] + " " + " ".join(args)))
         dst = None
     # return path of the encoded video
     return dst

pillar/api/utils/node_type_utils.py

@@ -27,9 +27,11 @@ def assign_permissions(project, node_types, permission_callback):
         permissions = {}
 
         for key in ('users', 'groups'):
-            perms = proj_perms[key]
-            singular = key.rstrip('s')
+            perms = proj_perms.get(key)
+            if not perms:
+                continue
 
+            singular = key.rstrip('s')
             for perm in perms:
                 assert isinstance(perm, dict), 'perm should be dict, but is %r' % perm
                 ident = perm[singular]  # group or user ID.

pillar/api/utils/rating.py

@@ -0,0 +1,87 @@
+# These functions come from Reddit
+# https://github.com/reddit/reddit/blob/master/r2/r2/lib/db/_sorts.pyx
+
+# Additional resources
+# http://www.redditblog.com/2009/10/reddits-new-comment-sorting-system.html
+# http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
+# http://amix.dk/blog/post/19588
+
+from datetime import datetime, timezone
+from math import log
+from math import sqrt
+
+epoch = datetime(1970, 1, 1, 0, 0, 0, 0, timezone.utc)
+
+
+def epoch_seconds(date):
+    """Returns the number of seconds from the epoch to date."""
+    td = date - epoch
+    return td.days * 86400 + td.seconds + (float(td.microseconds) / 1000000)
+
+
+def score(ups, downs):
+    return ups - downs
+
+
+def hot(ups, downs, date):
+    """The hot formula. Reddit's hot ranking uses the logarithm function to
+    weight the first votes higher than the rest.
+    The first 10 upvotes have the same weight as the next 100 upvotes which
+    have the same weight as the next 1000, etc.
+
+    Dillo authors: we modified the formula to give more weight to negative
+    votes when an entry is controversial.
+
+    TODO: make this function more dynamic so that different defaults can be
+    specified depending on the item that is being rated.
+    """
+
+    s = score(ups, downs)
+    order = log(max(abs(s), 1), 10)
+    sign = 1 if s > 0 else -1 if s < 0 else 0
+    seconds = epoch_seconds(date) - 1134028003
+    base_hot = round(sign * order + seconds / 45000, 7)
+
+    if downs > 1:
+        rating_delta = 100 * (downs - ups) / downs
+        if rating_delta < 25:
+            # The post is controversial
+            return base_hot
+        base_hot = base_hot - (downs * 6)
+
+    return base_hot
+
+
+def _confidence(ups, downs):
+    n = ups + downs
+
+    if n == 0:
+        return 0
+
+    z = 1.0 #1.0 = 85%, 1.6 = 95%
+    phat = float(ups) / n
+    return sqrt(phat+z*z/(2*n)-z*((phat*(1-phat)+z*z/(4*n))/n))/(1+z*z/n)
+
+
+def confidence(ups, downs):
+    if ups + downs == 0:
+        return 0
+    else:
+        return _confidence(ups, downs)
+
+
+def update_hot(document):
+    """Update the hotness of a document given its current ratings.
+
+    We expect the document to implement the ratings_embedded_schema in
+    a 'ratings' property.
+    """
+
+    dt = document['_created']
+    dt = dt.replace(tzinfo=timezone.utc)
+
+    document['properties']['ratings']['hot'] = hot(
+        document['properties']['ratings']['positive'],
+        document['properties']['ratings']['negative'],
+        dt,
+    )

pillar/api/utils/storage.py

@@ -1,83 +1 @@
-import subprocess
-
-import os
-from flask import current_app
-from pillar.api.utils.gcs import GoogleCloudStorageBucket
-
-
-def get_sizedata(filepath):
-    outdata = dict(
-        size=int(os.stat(filepath).st_size)
-    )
-    return outdata
-
-
-def rsync(path, remote_dir=''):
-    BIN_SSH = current_app.config['BIN_SSH']
-    BIN_RSYNC = current_app.config['BIN_RSYNC']
-
-    DRY_RUN = False
-    arguments = ['--verbose', '--ignore-existing', '--recursive', '--human-readable']
-    logs_path = current_app.config['CDN_SYNC_LOGS']
-    storage_address = current_app.config['CDN_STORAGE_ADDRESS']
-    user = current_app.config['CDN_STORAGE_USER']
-    rsa_key_path = current_app.config['CDN_RSA_KEY']
-    known_hosts_path = current_app.config['CDN_KNOWN_HOSTS']
-
-    if DRY_RUN:
-        arguments.append('--dry-run')
-    folder_arguments = list(arguments)
-    if rsa_key_path:
-        folder_arguments.append(
-            '-e ' + BIN_SSH + ' -i ' + rsa_key_path + ' -o "StrictHostKeyChecking=no"')
-    # if known_hosts_path:
-    #     folder_arguments.append("-o UserKnownHostsFile " + known_hosts_path)
-    folder_arguments.append("--log-file=" + logs_path + "/rsync.log")
-    folder_arguments.append(path)
-    folder_arguments.append(user + "@" + storage_address + ":/public/" + remote_dir)
-    # print (folder_arguments)
-    devnull = open(os.devnull, 'wb')
-    # DEBUG CONFIG
-    # print folder_arguments
-    # proc = subprocess.Popen(['rsync'] + folder_arguments)
-    # stdout, stderr = proc.communicate()
-    subprocess.Popen(['nohup', BIN_RSYNC] + folder_arguments, stdout=devnull, stderr=devnull)
-
-
-def remote_storage_sync(path):  # can be both folder and file
-    if os.path.isfile(path):
-        filename = os.path.split(path)[1]
-        rsync(path, filename[:2] + '/')
-    else:
-        if os.path.exists(path):
-            rsync(path)
-        else:
-            raise IOError('ERROR: path not found')
-
-
-def push_to_storage(project_id, full_path, backend='cgs'):
-    """Move a file from temporary/processing local storage to a storage endpoint.
-    By default we store items in a Google Cloud Storage bucket named after the
-    project id.
-    """
-
-    def push_single_file(project_id, full_path, backend):
-        if backend == 'cgs':
-            storage = GoogleCloudStorageBucket(project_id, subdir='_')
-            blob = storage.Post(full_path)
-            # XXX Make public on the fly if it's an image and small preview.
-            # This should happen by reading the database (push to storage
-            # should change to accomodate it).
-            if blob is not None and full_path.endswith('-t.jpg'):
-                blob.make_public()
-            os.remove(full_path)
-
-    if os.path.isfile(full_path):
-        push_single_file(project_id, full_path, backend)
-    else:
-        if os.path.exists(full_path):
-            for root, dirs, files in os.walk(full_path):
-                for name in files:
-                    push_single_file(project_id, os.path.join(root, name), backend)
-        else:
-            raise IOError('ERROR: path not found')
+"""Utility for managing storage backends and files."""

pillar/attrs_extra.py

@@ -1,9 +1,12 @@
 """Extra functionality for attrs."""
 
+import functools
 import logging
 
 import attr
 
+string = functools.partial(attr.ib, validator=attr.validators.instance_of(str))
+
 
 def log(name):
     """Returns a logger attr.ib

pillar/auth/__init__.py

@@ -1,27 +1,105 @@
 """Authentication code common to the web and api modules."""
 
+import collections
 import logging
+import typing
 
-from flask import current_app, session
+import blinker
+import bson
+from flask import session, g
 import flask_login
-import flask_oauthlib.client
+from werkzeug.local import LocalProxy
 
-from ..api import utils, blender_id
-from ..api.utils import authentication
+from pillar import current_app
 
+user_authenticated = blinker.Signal('Sent whenever a user was authenticated')
 log = logging.getLogger(__name__)
 
+# Mapping from user role to capabilities obtained by users with that role.
+CAPABILITIES = collections.defaultdict(**{
+    'subscriber': {'subscriber', 'home-project'},
+    'demo': {'subscriber', 'home-project'},
+    'admin': {'video-encoding', 'admin',
+              'view-pending-nodes', 'edit-project-node-types'},
+}, default_factory=frozenset)
+
 
 class UserClass(flask_login.UserMixin):
-    def __init__(self, token):
+    def __init__(self, token: typing.Optional[str]):
         # We store the Token instead of ID
         self.id = token
-        self.username = None
-        self.full_name = None
-        self.objectid = None
-        self.gravatar = None
-        self.email = None
-        self.roles = []
+        self.username: str = None
+        self.full_name: str = None
+        self.user_id: bson.ObjectId = None
+        self.objectid: str = None
+        self.gravatar: str = None
+        self.email: str = None
+        self.roles: typing.List[str] = []
+        self.groups: typing.List[str] = []  # NOTE: these are stringified object IDs.
+        self.group_ids: typing.List[bson.ObjectId] = []
+        self.capabilities: typing.Set[str] = set()
+
+        # Lazily evaluated
+        self._has_organizations: typing.Optional[bool] = None
+
+    @classmethod
+    def construct(cls, token: str, db_user: dict) -> 'UserClass':
+        """Constructs a new UserClass instance from a Mongo user document."""
+
+        from ..api import utils
+
+        user = cls(token)
+
+        user.user_id = db_user.get('_id')
+        user.roles = db_user.get('roles') or []
+        user.group_ids = db_user.get('groups') or []
+        user.email = db_user.get('email') or ''
+        user.username = db_user.get('username') or ''
+        user.full_name = db_user.get('full_name') or ''
+
+        # Derived properties
+        user.objectid = str(user.user_id or '')
+        user.gravatar = utils.gravatar(user.email)
+        user.groups = [str(g) for g in user.group_ids]
+        user.collect_capabilities()
+
+        return user
+
+    def __repr__(self):
+        return f'UserClass(user_id={self.user_id})'
+
+    def __str__(self):
+        return f'{self.__class__.__name__}(id={self.user_id}, email={self.email!r}'
+
+    def __getitem__(self, item):
+        """Compatibility layer with old dict-based g.current_user object."""
+
+        if item == 'user_id':
+            return self.user_id
+        if item == 'groups':
+            return self.group_ids
+        if item == 'roles':
+            return set(self.roles)
+
+        raise KeyError(f'No such key {item!r}')
+
+    def get(self, key, default=None):
+        """Compatibility layer with old dict-based g.current_user object."""
+
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def collect_capabilities(self):
+        """Constructs the capabilities set given the user's current roles.
+
+        Requires an application context to be active.
+        """
+
+        app_caps = current_app.user_caps
+
+        self.capabilities = set().union(*(app_caps[role] for role in self.roles))
 
     def has_role(self, *roles):
         """Returns True iff the user has one or more of the given roles."""
@@ -31,38 +109,84 @@ class UserClass(flask_login.UserMixin):
 
         return bool(set(self.roles).intersection(set(roles)))
 
+    def has_cap(self, *capabilities: typing.Iterable[str]) -> bool:
+        """Returns True iff the user has one or more of the given capabilities."""
 
-class AnonymousUser(flask_login.AnonymousUserMixin):
-    @property
-    def objectid(self):
-        """Anonymous user has no settable objectid."""
-        return None
+        if not self.capabilities:
+            return False
+
+        return bool(set(self.capabilities).intersection(set(capabilities)))
+
+    def matches_roles(self,
+                      require_roles=set(),
+                      require_all=False) -> bool:
+        """Returns True iff the user's roles matches the query.
+
+        :param require_roles: set of roles.
+        :param require_all:
+            When False (the default): if the user's roles have a
+            non-empty intersection with the given roles, returns True.
+            When True: require the user to have all given roles before
+            returning True.
+        """
+
+        if not isinstance(require_roles, set):
+            raise TypeError(f'require_roles param should be a set, but is {type(require_roles)!r}')
+
+        if require_all and not require_roles:
+            raise ValueError('require_login(require_all=True) cannot be used with '
+                             'empty require_roles.')
+
+        intersection = require_roles.intersection(self.roles)
+        if require_all:
+            return len(intersection) == len(require_roles)
+
+        return not bool(require_roles) or bool(intersection)
+
+    def has_organizations(self) -> bool:
+        """Returns True iff this user administers or is member of any organization."""
+
+        if self._has_organizations is None:
+            assert self.user_id
+            self._has_organizations = current_app.org_manager.user_has_organizations(self.user_id)
+
+        return bool(self._has_organizations)
+
+
+class AnonymousUser(flask_login.AnonymousUserMixin, UserClass):
+    def __init__(self):
+        super().__init__(token=None)
 
     def has_role(self, *roles):
         return False
 
+    def has_cap(self, *capabilities):
+        return False
 
-def _load_user(token):
+    def has_organizations(self) -> bool:
+        return False
+
+
+def _load_user(token) -> typing.Union[UserClass, AnonymousUser]:
     """Loads a user by their token.
 
     :returns: returns a UserClass instance if logged in, or an AnonymousUser() if not.
-    :rtype: UserClass
     """
 
+    from ..api.utils import authentication
+
+    if not token:
+        return AnonymousUser()
+
     db_user = authentication.validate_this_token(token)
     if not db_user:
+        # There is a token, but it's not valid. We should reset the user's session.
+        session.clear()
         return AnonymousUser()
 
-    login_user = UserClass(token)
-    login_user.email = db_user['email']
-    login_user.objectid = unicode(db_user['_id'])
-    login_user.username = db_user['username']
-    login_user.gravatar = utils.gravatar(db_user['email'])
-    login_user.roles = db_user.get('roles', [])
-    login_user.groups = [unicode(g) for g in db_user['groups'] or ()]
-    login_user.full_name = db_user.get('full_name', '')
+    user = UserClass.construct(token, db_user)
 
-    return login_user
+    return user
 
 
 def config_login_manager(app):
@@ -71,6 +195,7 @@ def config_login_manager(app):
     login_manager = flask_login.LoginManager()
     login_manager.init_app(app)
     login_manager.login_view = "users.login"
+    login_manager.login_message = ''
     login_manager.anonymous_user = AnonymousUser
     # noinspection PyTypeChecker
     login_manager.user_loader(_load_user)
@@ -78,39 +203,72 @@ def config_login_manager(app):
     return login_manager
 
 
-def login_user(oauth_token):
+def login_user(oauth_token: str, *, load_from_db=False):
     """Log in the user identified by the given token."""
 
+    if load_from_db:
+        user = _load_user(oauth_token)
+    else:
         user = UserClass(oauth_token)
-    flask_login.login_user(user)
+    flask_login.login_user(user, remember=True)
+    g.current_user = user
+    user_authenticated.send(None)
 
 
-def get_blender_id_oauth_token():
-    """Returns a tuple (token, ''), for use with flask_oauthlib."""
-    return session.get('blender_id_oauth_token')
+def logout_user():
+    """Forces a logout of the current user."""
+
+    from ..api.utils import authentication
+
+    token = get_blender_id_oauth_token()
+    if token:
+        authentication.remove_token(token)
+
+    session.clear()
+    flask_login.logout_user()
+    g.current_user = AnonymousUser()
 
 
-def config_oauth_login(app):
-    config = app.config
-    if not config.get('SOCIAL_BLENDER_ID'):
-        log.info('OAuth Blender-ID login not setup.')
-        return None
+def get_blender_id_oauth_token() -> str:
+    """Returns the Blender ID auth token, or an empty string if there is none."""
 
-    oauth = flask_oauthlib.client.OAuth(app)
-    social_blender_id = config.get('SOCIAL_BLENDER_ID')
+    from flask import request
 
-    oauth_blender_id = oauth.remote_app(
-        'blender_id',
-        consumer_key=social_blender_id['app_id'],
-        consumer_secret=social_blender_id['app_secret'],
-        request_token_params={'scope': 'email'},
-        base_url=config['BLENDER_ID_OAUTH_URL'],
-        request_token_url=None,
-        access_token_url=config['BLENDER_ID_BASE_ACCESS_TOKEN_URL'],
-        authorize_url=config['BLENDER_ID_AUTHORIZE_URL']
-    )
+    token = session.get('blender_id_oauth_token')
+    if token:
+        if isinstance(token, (tuple, list)):
+            # In a past version of Pillar we accidentally stored tuples in the session.
+            # Such sessions should be actively fixed.
+            # TODO(anyone, after 2017-12-01): refactor this if-block so that it just converts
+            # the token value to a string and use that instead.
+            token = token[0]
+            session['blender_id_oauth_token'] = token
+        return token
 
-    oauth_blender_id.tokengetter(get_blender_id_oauth_token)
-    log.info('OAuth Blender-ID login setup as %s', social_blender_id['app_id'])
+    if request.authorization and request.authorization.username:
+        return request.authorization.username
 
-    return oauth_blender_id
+    if current_user.is_authenticated and current_user.id:
+        return current_user.id
+
+    return ''
+
+
+def get_current_user() -> UserClass:
+    """Returns the current user as a UserClass instance.
+
+    Never returns None; returns an AnonymousUser() instance instead.
+
+    This function is intended to be used when pillar.auth.current_user is
+    accessed many times in the same scope. Calling this function is then
+    more efficient, since it doesn't have to resolve the LocalProxy for
+    each access to the returned object.
+    """
+
+    from ..api.utils.authentication import current_user
+
+    return current_user()
+
+
+current_user: UserClass = LocalProxy(get_current_user)
+"""The current user."""

pillar/auth/oauth.py

@@ -0,0 +1,219 @@
+import abc
+import attr
+import json
+import logging
+
+from rauth import OAuth2Service
+from flask import current_app, url_for, request, redirect, session, Response
+
+
+@attr.s
+class OAuthUserResponse:
+    """Represents user information requested to an OAuth provider after
+    authenticating.
+    """
+
+    id = attr.ib(validator=attr.validators.instance_of(str))
+    email = attr.ib(validator=attr.validators.instance_of(str))
+
+
+class OAuthError(Exception):
+    """Superclass of all exceptions raised by this module."""
+
+
+class ProviderConfigurationMissing(OAuthError):
+    """Raised when an OAuth provider is used but not configured."""
+
+
+class ProviderNotImplemented(OAuthError):
+    """Raised when a provider is requested that does not exist."""
+
+
+class OAuthCodeNotProvided(OAuthError):
+    """Raised when the 'code' arg is not provided in the OAuth callback."""
+
+
+class ProviderNotConfigured:
+    """Dummy class that indicates a provider isn't configured."""
+
+
+class OAuthSignIn(metaclass=abc.ABCMeta):
+    provider_name: str = None  # set in each subclass.
+
+    _providers = None  # initialized in get_provider()
+    _log = logging.getLogger(f'{__name__}.OAuthSignIn')
+
+    def __init__(self):
+        credentials = current_app.config['OAUTH_CREDENTIALS'].get(self.provider_name)
+        if not credentials:
+            raise ProviderConfigurationMissing(
+                f'Missing OAuth credentials for {self.provider_name}')
+
+        self.consumer_id = credentials['id']
+        self.consumer_secret = credentials['secret']
+
+        # Set in a subclass
+        self.service: OAuth2Service = None
+
+    @abc.abstractmethod
+    def authorize(self) -> Response:
+        """Redirect to the correct authorization endpoint for the current provider.
+
+        Depending on the provider, we sometimes have to specify a different
+        'scope'.
+        """
+        pass
+
+    @abc.abstractmethod
+    def callback(self) -> OAuthUserResponse:
+        """Callback performed after authorizing the user.
+
+        This is usually a request to a protected /me endpoint to query for
+        user information, such as user id and email address.
+        """
+        pass
+
+    def get_callback_url(self):
+        return url_for('users.oauth_callback', provider=self.provider_name,
+                       _external=True, _scheme=current_app.config['SCHEME'])
+
+    @staticmethod
+    def auth_code_from_request() -> str:
+        try:
+            return request.args['code']
+        except KeyError:
+            raise OAuthCodeNotProvided('A code argument was not provided in the request')
+
+    @staticmethod
+    def decode_json(payload):
+        return json.loads(payload.decode('utf-8'))
+
+    def make_oauth_session(self):
+        return self.service.get_auth_session(
+            data={'code': self.auth_code_from_request(),
+                  'grant_type': 'authorization_code',
+                  'redirect_uri': self.get_callback_url()},
+            decoder=self.decode_json
+        )
+
+    @classmethod
+    def get_provider(cls, provider_name) -> 'OAuthSignIn':
+        if cls._providers is None:
+            cls._init_providers()
+
+        try:
+            provider = cls._providers[provider_name]
+        except KeyError:
+            raise ProviderNotImplemented(f'No such OAuth provider {provider_name}')
+
+        if provider is ProviderNotConfigured:
+            raise ProviderConfigurationMissing(f'OAuth provider {provider_name} not configured')
+
+        return provider
+
+    @classmethod
+    def _init_providers(cls):
+        cls._providers = {}
+
+        for provider_class in cls.__subclasses__():
+            try:
+                provider = provider_class()
+            except ProviderConfigurationMissing:
+                cls._log.info('OAuth provider %s not configured',
+                              provider_class.provider_name)
+                provider = ProviderNotConfigured
+            cls._providers[provider_class.provider_name] = provider
+
+
+class BlenderIdSignIn(OAuthSignIn):
+    provider_name = 'blender-id'
+
+    def __init__(self):
+        super().__init__()
+
+        base_url = current_app.config['BLENDER_ID_ENDPOINT']
+
+        self.service = OAuth2Service(
+            name='blender-id',
+            client_id=self.consumer_id,
+            client_secret=self.consumer_secret,
+            authorize_url='%s/oauth/authorize' % base_url,
+            access_token_url='%s/oauth/token' % base_url,
+            base_url='%s/api/' % base_url
+        )
+
+    def authorize(self):
+        return redirect(self.service.get_authorize_url(
+            scope='email',
+            response_type='code',
+            redirect_uri=self.get_callback_url())
+        )
+
+    def callback(self):
+        oauth_session = self.make_oauth_session()
+
+        # TODO handle exception for failed oauth or not authorized
+        access_token = oauth_session.access_token
+        assert isinstance(access_token, str), f'oauth token must be str, not {type(access_token)}'
+
+        session['blender_id_oauth_token'] = access_token
+        me = oauth_session.get('user').json()
+        return OAuthUserResponse(str(me['id']), me['email'])
+
+
+class FacebookSignIn(OAuthSignIn):
+    provider_name = 'facebook'
+
+    def __init__(self):
+        super().__init__()
+        self.service = OAuth2Service(
+            name='facebook',
+            client_id=self.consumer_id,
+            client_secret=self.consumer_secret,
+            authorize_url='https://graph.facebook.com/oauth/authorize',
+            access_token_url='https://graph.facebook.com/oauth/access_token',
+            base_url='https://graph.facebook.com/'
+        )
+
+    def authorize(self):
+        return redirect(self.service.get_authorize_url(
+            scope='email',
+            response_type='code',
+            redirect_uri=self.get_callback_url())
+        )
+
+    def callback(self):
+        oauth_session = self.make_oauth_session()
+
+        me = oauth_session.get('me?fields=id,email').json()
+        # TODO handle case when user chooses not to disclose en email
+        # see https://developers.facebook.com/docs/graph-api/reference/user/
+        return OAuthUserResponse(me['id'], me.get('email'))
+
+
+class GoogleSignIn(OAuthSignIn):
+    provider_name = 'google'
+
+    def __init__(self):
+        super().__init__()
+        self.service = OAuth2Service(
+            name='google',
+            client_id=self.consumer_id,
+            client_secret=self.consumer_secret,
+            authorize_url='https://accounts.google.com/o/oauth2/auth',
+            access_token_url='https://accounts.google.com/o/oauth2/token',
+            base_url='https://www.googleapis.com/oauth2/v1/'
+        )
+
+    def authorize(self):
+        return redirect(self.service.get_authorize_url(
+            scope='https://www.googleapis.com/auth/userinfo.email',
+            response_type='code',
+            redirect_uri=self.get_callback_url())
+        )
+
+    def callback(self):
+        oauth_session = self.make_oauth_session()
+
+        me = oauth_session.get('userinfo').json()
+        return OAuthUserResponse(str(me['id']), me['email'])

pillar/auth/subscriptions.py

@@ -1,51 +0,0 @@
-"""Cloud subscription info.
-
-Connects to the external subscription server to obtain user info.
-"""
-
-import logging
-
-from flask import current_app
-import requests
-from requests.adapters import HTTPAdapter
-
-log = logging.getLogger(__name__)
-
-
-def fetch_user(email):
-    """Returns the user info dict from the external subscriptions management server.
-
-    :returns: the store user info, or None if the user can't be found or there
-        was an error communicating. A dict like this is returned:
-        {
-            "shop_id": 700,
-            "cloud_access": 1,
-            "paid_balance": 314.75,
-            "balance_currency": "EUR",
-            "start_date": "2014-08-25 17:05:46",
-            "expiration_date": "2016-08-24 13:38:45",
-            "subscription_status": "wc-active",
-            "expiration_date_approximate": true
-        }
-    :rtype: dict
-    """
-
-    external_subscriptions_server = current_app.config['EXTERNAL_SUBSCRIPTIONS_MANAGEMENT_SERVER']
-
-    log.debug('Connecting to store at %s?blenderid=%s', external_subscriptions_server, email)
-
-    # Retry a few times when contacting the store.
-    s = requests.Session()
-    s.mount(external_subscriptions_server, HTTPAdapter(max_retries=5))
-    r = s.get(external_subscriptions_server, params={'blenderid': email},
-              verify=current_app.config['TLS_CERT_FILE'])
-
-    if r.status_code != 200:
-        log.warning("Error communicating with %s, code=%i, unable to check "
-                    "subscription status of user %s",
-                    external_subscriptions_server, r.status_code, email)
-        return None
-
-    store_user = r.json()
-    return store_user
-

pillar/bugsnag_extra.py

@@ -0,0 +1,52 @@
+# Keys in the user's session dictionary that are removed before sending to Bugsnag.
+SESSION_KEYS_TO_REMOVE = ('blender_id_oauth_token', 'user_id')
+
+
+def add_pillar_request_to_notification(notification):
+    """Adds request metadata to the Bugsnag notifications.
+
+    This basically copies bugsnag.flask.add_flask_request_to_notification,
+    but is altered to include Pillar-specific metadata.
+    """
+    from flask import request, session
+    from bugsnag.wsgi import request_path
+    import pillar.auth
+
+    if not request:
+        return
+
+    notification.context = "%s %s" % (request.method,
+                                      request_path(request.environ))
+
+    if 'id' not in notification.user:
+        user: pillar.auth.UserClass = pillar.auth.current_user._get_current_object()
+        notification.set_user(id=user.user_id,
+                              email=user.email,
+                              name=user.username)
+        notification.user['roles'] = sorted(user.roles)
+        notification.user['capabilities'] = sorted(user.capabilities)
+
+    session_dict = dict(session)
+    for key in SESSION_KEYS_TO_REMOVE:
+        try:
+            del session_dict[key]
+        except KeyError:
+            pass
+    notification.add_tab("session", session_dict)
+    notification.add_tab("environment", dict(request.environ))
+
+    remote_addr = request.remote_addr
+    forwarded_for = request.headers.get('X-Forwarded-For')
+    if forwarded_for:
+        remote_addr = f'{forwarded_for} (proxied via {remote_addr})'
+
+    notification.add_tab("request", {
+        "method": request.method,
+        "url": request.base_url,
+        "headers": dict(request.headers),
+        "params": dict(request.form),
+        "data": {'request.data': request.data,
+                 'request.json': request.get_json()},
+        "endpoint": request.endpoint,
+        "remote_addr": remote_addr,
+    })

pillar/celery/__init__.py

@@ -0,0 +1,6 @@
+"""Tasks to be run by the Celery worker.
+
+If you create a new submodule/subpackage, be sure to add it to
+PillarServer._config_celery() too.
+
+"""

pillar/celery/algolia_indexing.py

@@ -0,0 +1,38 @@
+import logging
+
+from algoliasearch.helpers import AlgoliaException
+
+log = logging.getLogger(__name__)
+
+
+def push_updated_user(user_to_index: dict):
+    """Push an update to the Algolia index when a user item is updated"""
+
+    from pillar.api.utils.algolia import index_user_save
+
+    try:
+        index_user_save(user_to_index)
+    except AlgoliaException as ex:
+        log.warning(
+            'Unable to push user info to Algolia for user "%s", id=%s; %s',  # noqa
+            user_to_index.get('username'),
+            user_to_index.get('objectID'), ex)
+
+
+def index_node_save(node_to_index: dict):
+    from pillar.api.utils import algolia
+
+    try:
+        algolia.index_node_save(node_to_index)
+    except AlgoliaException as ex:
+        log.warning(
+            'Unable to push node info to Algolia for node %s; %s', node_to_index, ex)  # noqa
+
+
+def index_node_delete(delete_id: str):
+
+    from pillar.api.utils import algolia
+    try:
+        algolia.index_node_delete(delete_id)
+    except AlgoliaException as ex:
+        log.warning('Unable to delete node info to Algolia for node %s; %s', delete_id, ex)  # noqa

pillar/celery/email_tasks.py

@@ -0,0 +1,50 @@
+"""Deferred email support.
+
+Note that this module can only be imported when an application context is
+active. Best to late-import this in the functions where it's needed.
+"""
+from email.message import EmailMessage
+from email.headerregistry import Address
+import logging
+import smtplib
+
+import celery
+
+from pillar import current_app
+
+log = logging.getLogger(__name__)
+
+
+@current_app.celery.task(bind=True, ignore_result=True, acks_late=True)
+def send_email(self: celery.Task, to_name: str, to_addr: str, subject: str, text: str, html: str):
+    """Send an email to a single address."""
+    # WARNING: when changing the signature of this function, also change the
+    # self.retry() call below.
+    cfg = current_app.config
+
+    # Construct the message
+    msg = EmailMessage()
+    msg['Subject'] = subject
+    msg['From'] = Address(cfg['MAIL_DEFAULT_FROM_NAME'], addr_spec=cfg['MAIL_DEFAULT_FROM_ADDR'])
+    msg['To'] = (Address(to_name, addr_spec=to_addr),)
+    msg.set_content(text)
+    msg.add_alternative(html, subtype='html')
+
+    # Refuse to send mail when we're testing.
+    if cfg['TESTING']:
+        log.warning('not sending mail to %s <%s> because we are TESTING', to_name, to_addr)
+        return
+    log.info('sending email to %s <%s>', to_name, to_addr)
+
+    # Send the message via local SMTP server.
+    try:
+        with smtplib.SMTP(cfg['SMTP_HOST'], cfg['SMTP_PORT'], timeout=cfg['SMTP_TIMEOUT']) as smtp:
+            if cfg.get('SMTP_USERNAME') and cfg.get('SMTP_PASSWORD'):
+                smtp.login(cfg['SMTP_USERNAME'], cfg['SMTP_PASSWORD'])
+            smtp.send_message(msg)
+    except (IOError, OSError) as ex:
+        log.exception('error sending email to %s <%s>, will retry later: %s',
+                      to_name, to_addr, ex)
+        self.retry((to_name, to_addr, subject, text, html), countdown=cfg['MAIL_RETRY'])
+    else:
+        log.info('mail to %s <%s> successfully sent', to_name, to_addr)

pillar/celery/file_link_tasks.py

@@ -0,0 +1,19 @@
+from pillar import current_app
+
+
+@current_app.celery.task(ignore_result=True)
+def regenerate_all_expired_links(backend_name: str, chunk_size: int):
+    """Regenerate all expired links for all non-deleted file documents.
+
+    Probably only works on Google Cloud Storage ('gcs') backends at
+    the moment, since those are the only links that actually expire.
+
+    :param backend_name: name of the backend to refresh for.
+    :param chunk_size: the maximum number of files to refresh in this run.
+    """
+    from pillar.api import file_storage
+
+    # Refresh all files that already have expired or will expire in the next
+    # two hours. Since this task is intended to run every hour, this should
+    # result in all regular file requests having a valid link.
+    file_storage.refresh_links_for_backend(backend_name, chunk_size, expiry_seconds=7200)

pillar/celery/search_index_tasks.py

@@ -0,0 +1,198 @@
+import logging
+from bson import ObjectId
+
+from pillar import current_app
+from pillar.api.file_storage import generate_link
+from pillar.api.search import elastic_indexing
+from pillar.api.search import algolia_indexing
+
+
+log = logging.getLogger(__name__)
+
+
+INDEX_ALLOWED_NODE_TYPES = {'asset', 'texture', 'group', 'hdri'}
+
+
+SEARCH_BACKENDS = {
+    'algolia': algolia_indexing,
+    'elastic': elastic_indexing
+}
+
+
+def _get_node_from_id(node_id: str):
+    node_oid = ObjectId(node_id)
+
+    nodes_coll = current_app.db('nodes')
+    node = nodes_coll.find_one({'_id': node_oid})
+
+    return node
+
+
+def _handle_picture(node: dict, to_index: dict):
+    """Add picture URL in-place to the to-be-indexed node."""
+
+    picture_id = node.get('picture')
+    if not picture_id:
+        return
+
+    files_collection = current_app.data.driver.db['files']
+    lookup = {'_id': ObjectId(picture_id)}
+    picture = files_collection.find_one(lookup)
+
+    for item in picture.get('variations', []):
+        if item['size'] != 't':
+            continue
+
+        # Not all files have a project...
+        pid = picture.get('project')
+        if pid:
+            link = generate_link(picture['backend'],
+                                 item['file_path'],
+                                 str(pid),
+                                 is_public=True)
+        else:
+            link = item['link']
+        to_index['picture'] = link
+        break
+
+
+def prepare_node_data(node_id: str, node: dict=None) -> dict:
+    """Given a node id or a node document, return an indexable version of it.
+
+    Returns an empty dict when the node shouldn't be indexed.
+    """
+
+    if node_id and node:
+        raise ValueError("Do not provide node and node_id together")
+
+    if node_id:
+        node = _get_node_from_id(node_id)
+
+    if node is None:
+        log.warning('Unable to find node %s, not updating.', node_id)
+        return {}
+
+    if node['node_type'] not in INDEX_ALLOWED_NODE_TYPES:
+        log.debug('Node of type %s is not indexable by Pillar', node['node_type'])
+        return {}
+    # If a nodes does not have status published, do not index
+    if node['properties'].get('status') != 'published':
+        log.debug('Node %s is does not have published status', node_id)
+        return {}
+
+    projects_collection = current_app.data.driver.db['projects']
+    project = projects_collection.find_one({'_id': ObjectId(node['project'])})
+
+    users_collection = current_app.data.driver.db['users']
+    user = users_collection.find_one({'_id': ObjectId(node['user'])})
+
+    to_index = {
+        'objectID': node['_id'],
+        'name': node['name'],
+        'project': {
+            '_id': project['_id'],
+            'name': project['name']
+        },
+        'created': node['_created'],
+        'updated': node['_updated'],
+        'node_type': node['node_type'],
+        'user': {
+            '_id': user['_id'],
+            'full_name': user['full_name']
+        },
+        'description': node.get('description'),
+    }
+
+    _handle_picture(node, to_index)
+
+    # If the node has world permissions, compute the Free permission
+    if 'world' in node.get('permissions', {}):
+        if 'GET' in node['permissions']['world']:
+            to_index['is_free'] = True
+
+    # Append the media key if the node is of node_type 'asset'
+    if node['node_type'] == 'asset':
+        to_index['media'] = node['properties']['content_type']
+
+    # Add extra properties
+    for prop in ('tags', 'license_notes'):
+        if prop in node['properties']:
+            to_index[prop] = node['properties'][prop]
+
+    return to_index
+
+
+def prepare_user_data(user_id: str, user=None) -> dict:
+    """
+    Prepare data to index for user node.
+
+    Returns an empty dict if the user should not be indexed.
+    """
+
+    if not user:
+        user_oid = ObjectId(user_id)
+        log.info('Retrieving user %s', user_oid)
+        users_coll = current_app.db('users')
+        user = users_coll.find_one({'_id': user_oid})
+
+    if user is None:
+        log.warning('Unable to find user %s, not updating search index.', user_id)
+        return {}
+
+    user_roles = set(user.get('roles', ()))
+
+    if 'service' in user_roles:
+        return {}
+
+    # Strip unneeded roles
+    index_roles = user_roles.intersection(current_app.user_roles_indexable)
+
+    log.debug('Push user %r to Search index', user['_id'])
+
+    user_to_index = {
+        'objectID': user['_id'],
+        'full_name': user['full_name'],
+        'username': user['username'],
+        'roles': list(index_roles),
+        'groups': user['groups'],
+        'email': user['email']
+    }
+
+    return user_to_index
+
+
+@current_app.celery.task(ignore_result=True)
+def updated_user(user_id: str):
+    """Push an update to the index when a user item is updated"""
+
+    user_to_index = prepare_user_data(user_id)
+
+    for searchoption in current_app.config['SEARCH_BACKENDS']:
+        searchmodule = SEARCH_BACKENDS[searchoption]
+        searchmodule.push_updated_user(user_to_index)
+
+
+@current_app.celery.task(ignore_result=True)
+def node_save(node_id: str):
+
+    to_index = prepare_node_data(node_id)
+
+    if not to_index:
+        log.debug('Node %s will not be indexed', node_id)
+        return
+
+    for searchoption in current_app.config['SEARCH_BACKENDS']:
+        searchmodule = SEARCH_BACKENDS[searchoption]
+        searchmodule.index_node_save(to_index)
+
+
+@current_app.celery.task(ignore_result=True)
+def node_delete(node_id: str):
+
+    # Deleting a node takes nothing more than the ID anyway.
+    # No need to fetch anything from Mongo.
+    delete_id = ObjectId(node_id)
+
+    for searchoption in current_app.config['SEARCH_BACKENDS']:
+        searchmodule = SEARCH_BACKENDS[searchoption]
+        searchmodule.index_node_delete(delete_id)

pillar/celery/tasks.py

@@ -0,0 +1,20 @@
+import logging
+import typing
+
+from pillar import current_app
+
+log = logging.getLogger(__name__)
+
+
+@current_app.celery.task(track_started=True)
+def long_task(numbers: typing.List[int]):
+    _log = log.getChild('long_task')
+    _log.info('Computing sum of %i items', len(numbers))
+
+    import time
+    time.sleep(6)
+    thesum = sum(numbers)
+
+    _log.info('Computed sum of %i items', len(numbers))
+
+    return thesum

pillar/cli.py

@@ -1,795 +0,0 @@
-"""Commandline interface.
-
-Run commands with 'flask <command>'
-"""
-
-from __future__ import print_function, division
-
-import copy
-import logging
-
-from bson.objectid import ObjectId, InvalidId
-from eve.methods.put import put_internal
-from eve.methods.post import post_internal
-
-from flask import current_app
-from flask_script import Manager
-
-log = logging.getLogger(__name__)
-manager = Manager(current_app)
-
-manager_maintenance = Manager(
-    current_app, usage="Maintenance scripts, to update user groups")
-manager_setup = Manager(
-    current_app, usage="Setup utilities, like setup_db() or create_blog()")
-manager_operations = Manager(
-    current_app, usage="Backend operations, like moving nodes across projects")
-
-
-@manager_setup.command
-def setup_db(admin_email):
-    """Setup the database
-    - Create admin, subscriber and demo Group collection
-    - Create admin user (must use valid blender-id credentials)
-    - Create one project
-    """
-
-    # Create default groups
-    groups_list = []
-    for group in ['admin', 'subscriber', 'demo']:
-        g = {'name': group}
-        g = current_app.post_internal('groups', g)
-        groups_list.append(g[0]['_id'])
-        print("Creating group {0}".format(group))
-
-    # Create admin user
-    user = {'username': admin_email,
-            'groups': groups_list,
-            'roles': ['admin', 'subscriber', 'demo'],
-            'settings': {'email_communications': 1},
-            'auth': [],
-            'full_name': admin_email,
-            'email': admin_email}
-    result, _, _, status = current_app.post_internal('users', user)
-    if status != 201:
-        raise SystemExit('Error creating user {}: {}'.format(admin_email, result))
-    user.update(result)
-    print("Created user {0}".format(user['_id']))
-
-    # Create a default project by faking a POST request.
-    with current_app.test_request_context(data={'project_name': u'Default Project'}):
-        from flask import g
-        from pillar.api.projects import routes as proj_routes
-
-        g.current_user = {'user_id': user['_id'],
-                          'groups': user['groups'],
-                          'roles': set(user['roles'])}
-
-        proj_routes.create_project(overrides={'url': 'default-project',
-                                              'is_private': False})
-
-
-@manager_maintenance.command
-def find_duplicate_users():
-    """Finds users that have the same BlenderID user_id."""
-
-    from collections import defaultdict
-
-    users_coll = current_app.data.driver.db['users']
-    nodes_coll = current_app.data.driver.db['nodes']
-    projects_coll = current_app.data.driver.db['projects']
-
-    found_users = defaultdict(list)
-
-    for user in users_coll.find():
-        blender_ids = [auth['user_id'] for auth in user['auth']
-                       if auth['provider'] == 'blender-id']
-        if not blender_ids:
-            continue
-        blender_id = blender_ids[0]
-        found_users[blender_id].append(user)
-
-    for blender_id, users in found_users.iteritems():
-        if len(users) == 1:
-            continue
-
-        usernames = ', '.join(user['username'] for user in users)
-        print('Blender ID: %5s has %i users: %s' % (
-            blender_id, len(users), usernames))
-
-        for user in users:
-            print('  %s owns %i nodes and %i projects' % (
-                user['username'],
-                nodes_coll.count({'user': user['_id']}),
-                projects_coll.count({'user': user['_id']}),
-            ))
-
-
-@manager_maintenance.command
-def sync_role_groups(do_revoke_groups):
-    """For each user, synchronizes roles and group membership.
-
-    This ensures that everybody with the 'subscriber' role is also member of the 'subscriber'
-    group, and people without the 'subscriber' role are not member of that group. Same for
-    admin and demo groups.
-
-    When do_revoke_groups=False (the default), people are only added to groups.
-    when do_revoke_groups=True, people are also removed from groups.
-    """
-
-    from pillar.api import service
-
-    if do_revoke_groups not in {'true', 'false'}:
-        print('Use either "true" or "false" as first argument.')
-        print('When passing "false", people are only added to groups.')
-        print('when passing "true", people are also removed from groups.')
-        raise SystemExit()
-    do_revoke_groups = do_revoke_groups == 'true'
-
-    service.fetch_role_to_group_id_map()
-
-    users_coll = current_app.data.driver.db['users']
-    groups_coll = current_app.data.driver.db['groups']
-
-    group_names = {}
-
-    def gname(gid):
-        try:
-            return group_names[gid]
-        except KeyError:
-            name = groups_coll.find_one(gid, projection={'name': 1})['name']
-            name = str(name)
-            group_names[gid] = name
-            return name
-
-    ok_users = bad_users = 0
-    for user in users_coll.find():
-        grant_groups = set()
-        revoke_groups = set()
-        current_groups = set(user.get('groups', []))
-        user_roles = user.get('roles', set())
-
-        for role in service.ROLES_WITH_GROUPS:
-            action = 'grant' if role in user_roles else 'revoke'
-            groups = service.manage_user_group_membership(user, role, action)
-
-            if groups is None:
-                # No changes required
-                continue
-
-            if groups == current_groups:
-                continue
-
-            grant_groups.update(groups.difference(current_groups))
-            revoke_groups.update(current_groups.difference(groups))
-
-        if grant_groups or revoke_groups:
-            bad_users += 1
-
-            expected_groups = current_groups.union(grant_groups).difference(revoke_groups)
-
-            print('Discrepancy for user %s/%s:' % (user['_id'], user['full_name'].encode('utf8')))
-            print('    - actual groups  :', sorted(gname(gid) for gid in user.get('groups')))
-            print('    - expected groups:', sorted(gname(gid) for gid in expected_groups))
-            print('    - will grant     :', sorted(gname(gid) for gid in grant_groups))
-
-            if do_revoke_groups:
-                label = 'WILL REVOKE '
-            else:
-                label = 'could revoke'
-            print('    - %s   :' % label, sorted(gname(gid) for gid in revoke_groups))
-
-            if grant_groups and revoke_groups:
-                print('        ------ CAREFUL this one has BOTH grant AND revoke -----')
-
-            # Determine which changes we'll apply
-            final_groups = current_groups.union(grant_groups)
-            if do_revoke_groups:
-                final_groups.difference_update(revoke_groups)
-            print('    - final groups   :', sorted(gname(gid) for gid in final_groups))
-
-            # Perform the actual update
-            users_coll.update_one({'_id': user['_id']},
-                                  {'$set': {'groups': list(final_groups)}})
-        else:
-            ok_users += 1
-
-    print('%i bad and %i ok users seen.' % (bad_users, ok_users))
-
-
-@manager_maintenance.command
-def sync_project_groups(user_email, fix):
-    """Gives the user access to their self-created projects."""
-
-    if fix.lower() not in {'true', 'false'}:
-        print('Use either "true" or "false" as second argument.')
-        print('When passing "false", only a report is produced.')
-        print('when passing "true", group membership is fixed.')
-        raise SystemExit()
-    fix = fix.lower() == 'true'
-
-    users_coll = current_app.data.driver.db['users']
-    proj_coll = current_app.data.driver.db['projects']
-    groups_coll = current_app.data.driver.db['groups']
-
-    # Find by email or by user ID
-    if '@' in user_email:
-        where = {'email': user_email}
-    else:
-        try:
-            where = {'_id': ObjectId(user_email)}
-        except InvalidId:
-            log.warning('Invalid ObjectID: %s', user_email)
-            return
-
-    user = users_coll.find_one(where, projection={'_id': 1, 'groups': 1})
-    if user is None:
-        log.error('User %s not found', where)
-        raise SystemExit()
-
-    user_groups = set(user['groups'])
-    user_id = user['_id']
-    log.info('Updating projects for user %s', user_id)
-
-    ok_groups = missing_groups = 0
-    for proj in proj_coll.find({'user': user_id}):
-        project_id = proj['_id']
-        log.info('Investigating project %s (%s)', project_id, proj['name'])
-
-        # Find the admin group
-        admin_group = groups_coll.find_one({'name': str(project_id)}, projection={'_id': 1})
-        if admin_group is None:
-            log.warning('No admin group for project %s', project_id)
-            continue
-        group_id = admin_group['_id']
-
-        # Check membership
-        if group_id not in user_groups:
-            log.info('Missing group membership')
-            missing_groups += 1
-            user_groups.add(group_id)
-        else:
-            ok_groups += 1
-
-    log.info('User %s was missing %i group memberships; %i projects were ok.',
-             user_id, missing_groups, ok_groups)
-
-    if missing_groups > 0 and fix:
-        log.info('Updating database.')
-        result = users_coll.update_one({'_id': user_id},
-                                       {'$set': {'groups': list(user_groups)}})
-        log.info('Updated %i user.', result.modified_count)
-
-
-@manager_maintenance.command
-def check_home_project_groups():
-    """Checks all users' group membership of their home project admin group."""
-
-    users_coll = current_app.data.driver.db['users']
-    proj_coll = current_app.data.driver.db['projects']
-
-    good = bad = 0
-    for proj in proj_coll.find({'category': 'home'}):
-        try:
-            admin_group_perms = proj['permissions']['groups'][0]
-        except IndexError:
-            log.error('Project %s has no admin group', proj['_id'])
-            return 255
-        except KeyError:
-            log.error('Project %s has no group permissions at all', proj['_id'])
-            return 255
-
-        user = users_coll.find_one({'_id': proj['user']},
-                                   projection={'groups': 1})
-        if user is None:
-            log.error('Project %s has non-existing owner %s', proj['user'])
-            return 255
-
-        user_groups = set(user['groups'])
-        admin_group_id = admin_group_perms['group']
-        if admin_group_id in user_groups:
-            # All is fine!
-            good += 1
-            continue
-
-        log.warning('User %s has no admin rights to home project %s -- needs group %s',
-                    proj['user'], proj['_id'], admin_group_id)
-        bad += 1
-
-    log.info('%i projects OK, %i projects in error', good, bad)
-    return bad
-
-
-@manager_setup.command
-def badger(action, user_email, role):
-    from pillar.api import service
-
-    with current_app.app_context():
-        service.fetch_role_to_group_id_map()
-        response, status = service.do_badger(action, user_email, role)
-
-    if status == 204:
-        log.info('Done.')
-    else:
-        log.info('Response: %s', response)
-        log.info('Status  : %i', status)
-
-
-def create_service_account(email, service_roles, service_definition, update_existing=None):
-    from pillar.api import service
-    from pillar.api.utils import dumps
-
-    account, token = service.create_service_account(
-        email,
-        service_roles,
-        service_definition,
-        update_existing=update_existing
-    )
-
-    print('Service account information:')
-    print(dumps(account, indent=4, sort_keys=True))
-    print()
-    print('Access token: %s' % token['token'])
-    print('  expires on: %s' % token['expire_time'])
-    return account, token
-
-
-@manager_setup.command
-def create_badger_account(email, badges):
-    """
-    Creates a new service account that can give badges (i.e. roles).
-
-    :param email: email address associated with the account
-    :param badges: single space-separated argument containing the roles
-        this account can assign and revoke.
-    """
-
-    create_service_account(email, [u'badger'], {'badger': badges.strip().split()})
-
-
-@manager_setup.command
-def create_urler_account(email):
-    """Creates a new service account that can fetch all project URLs."""
-
-    create_service_account(email, [u'urler'], {})
-
-
-@manager_setup.command
-def create_local_user_account(email, password):
-    from pillar.api.local_auth import create_local_user
-    create_local_user(email, password)
-
-
-@manager_maintenance.command
-@manager_maintenance.option('-c', '--chunk', dest='chunk_size', default=50,
-                help='Number of links to update, use 0 to update all.')
-@manager_maintenance.option('-q', '--quiet', dest='quiet', action='store_true', default=False)
-@manager_maintenance.option('-w', '--window', dest='window', default=12,
-                help='Refresh links that expire in this many hours.')
-def refresh_backend_links(backend_name, chunk_size=50, quiet=False, window=12):
-    """Refreshes all file links that are using a certain storage backend.
-
-    Use `--chunk 0` to refresh all links.
-    """
-
-    chunk_size = int(chunk_size)
-    window = int(window)
-
-    loglevel = logging.WARNING if quiet else logging.DEBUG
-    logging.getLogger('pillar.api.file_storage').setLevel(loglevel)
-
-    chunk_size = int(chunk_size)  # CLI parameters are passed as strings
-    from pillar.api import file_storage
-
-    file_storage.refresh_links_for_backend(backend_name, chunk_size, window * 3600)
-
-
-@manager_maintenance.command
-def expire_all_project_links(project_uuid):
-    """Expires all file links for a certain project without refreshing.
-
-    This is just for testing.
-    """
-
-    import datetime
-    import bson.tz_util
-
-    files_collection = current_app.data.driver.db['files']
-
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
-    expires = now - datetime.timedelta(days=1)
-
-    result = files_collection.update_many(
-        {'project': ObjectId(project_uuid)},
-        {'$set': {'link_expires': expires}}
-    )
-
-    print('Expired %i links' % result.matched_count)
-
-
-@manager_operations.command
-def file_change_backend(file_id, dest_backend='gcs'):
-    """Given a file document, move it to the specified backend (if not already
-    there) and update the document to reflect that.
-    Files on the original backend are not deleted automatically.
-    """
-
-    from pillar.api.file_storage.moving import change_file_storage_backend
-    change_file_storage_backend(file_id, dest_backend)
-
-
-@manager_operations.command
-def mass_copy_between_backends(src_backend='cdnsun', dest_backend='gcs'):
-    """Copies all files from one backend to the other, updating them in Mongo.
-
-    Files on the original backend are not deleted.
-    """
-
-    import requests.exceptions
-
-    from pillar.api.file_storage import moving
-
-    logging.getLogger('pillar').setLevel(logging.INFO)
-    log.info('Mass-moving all files from backend %r to %r',
-             src_backend, dest_backend)
-
-    files_coll = current_app.data.driver.db['files']
-
-    fdocs = files_coll.find({'backend': src_backend},
-                            projection={'_id': True})
-    copied_ok = 0
-    copy_errs = 0
-    try:
-        for fdoc in fdocs:
-            try:
-                moving.change_file_storage_backend(fdoc['_id'], dest_backend)
-            except moving.PrerequisiteNotMetError as ex:
-                log.error('Error copying %s: %s', fdoc['_id'], ex)
-                copy_errs += 1
-            except requests.exceptions.HTTPError as ex:
-                log.error('Error copying %s (%s): %s',
-                          fdoc['_id'], ex.response.url, ex)
-                copy_errs += 1
-            except Exception:
-                log.exception('Unexpected exception handling file %s', fdoc['_id'])
-                copy_errs += 1
-            else:
-                copied_ok += 1
-    except KeyboardInterrupt:
-        log.error('Stopping due to keyboard interrupt')
-
-    log.info('%i files copied ok', copied_ok)
-    log.info('%i files we did not copy', copy_errs)
-
-
-@manager_operations.command
-@manager_operations.option('-p', '--project', dest='dest_proj_url',
-                help='Destination project URL')
-@manager_operations.option('-f', '--force', dest='force', action='store_true', default=False,
-                help='Move even when already at the given project.')
-@manager_operations.option('-s', '--skip-gcs', dest='skip_gcs', action='store_true', default=False,
-                help='Skip file handling on GCS, just update the database.')
-def move_group_node_project(node_uuid, dest_proj_url, force=False, skip_gcs=False):
-    """Copies all files from one project to the other, then moves the nodes.
-
-    The node and all its children are moved recursively.
-    """
-
-    from pillar.api.nodes import moving
-    from pillar.api.utils import str2id
-
-    logging.getLogger('pillar').setLevel(logging.INFO)
-
-    db = current_app.db()
-    nodes_coll = db['nodes']
-    projs_coll = db['projects']
-
-    # Parse CLI args and get the node, source and destination projects.
-    node_uuid = str2id(node_uuid)
-    node = nodes_coll.find_one({'_id': node_uuid})
-    if node is None:
-        log.error("Node %s can't be found!", node_uuid)
-        return 1
-
-    if node.get('parent', None):
-        log.error('Node cannot have a parent, it must be top-level.')
-        return 4
-
-    src_proj = projs_coll.find_one({'_id': node['project']})
-    dest_proj = projs_coll.find_one({'url': dest_proj_url})
-
-    if src_proj is None:
-        log.warning("Node's source project %s doesn't exist!", node['project'])
-    if dest_proj is None:
-        log.error("Destination project url='%s' doesn't exist.", dest_proj_url)
-        return 2
-    if src_proj['_id'] == dest_proj['_id']:
-        if force:
-            log.warning("Node is already at project url='%s'!", dest_proj_url)
-        else:
-            log.error("Node is already at project url='%s'!", dest_proj_url)
-            return 3
-
-    log.info("Mass-moving %s (%s) and children from project '%s' (%s) to '%s' (%s)",
-             node_uuid, node['name'], src_proj['url'], src_proj['_id'], dest_proj['url'],
-             dest_proj['_id'])
-
-    mover = moving.NodeMover(db=db, skip_gcs=skip_gcs)
-    mover.change_project(node, dest_proj)
-
-    log.info('Done moving.')
-
-
-@manager_maintenance.command
-@manager_maintenance.option('-p', '--project', dest='proj_url', nargs='?',
-                help='Project URL')
-@manager_maintenance.option('-a', '--all', dest='all_projects', action='store_true', default=False,
-                help='Replace on all projects.')
-def replace_pillar_node_type_schemas(proj_url=None, all_projects=False):
-    """Replaces the project's node type schemas with the standard Pillar ones.
-
-    Non-standard node types are left alone.
-    """
-
-    if bool(proj_url) == all_projects:
-        log.error('Use either --project or --all.')
-        return 1
-
-    from pillar.api.utils.authentication import force_cli_user
-    force_cli_user()
-
-    from pillar.api.node_types import PILLAR_NAMED_NODE_TYPES
-    from pillar.api.utils import remove_private_keys
-
-    projects_collection = current_app.db()['projects']
-
-    def handle_project(project):
-        log.info('Handling project %s', project['url'])
-        is_public_proj = not project.get('is_private', True)
-
-        for proj_nt in project['node_types']:
-            nt_name = proj_nt['name']
-            try:
-                pillar_nt = PILLAR_NAMED_NODE_TYPES[nt_name]
-            except KeyError:
-                log.info('   - skipping non-standard node type "%s"', nt_name)
-                continue
-
-            log.info('   - replacing schema on node type "%s"', nt_name)
-
-            # This leaves node type keys intact that aren't in Pillar's node_type_xxx definitions,
-            # such as permissions.
-            proj_nt.update(copy.deepcopy(pillar_nt))
-
-            # On our own public projects we want to be able to set license stuff.
-            if is_public_proj:
-                proj_nt['form_schema'].pop('license_type', None)
-                proj_nt['form_schema'].pop('license_notes', None)
-
-        # Use Eve to PUT, so we have schema checking.
-        db_proj = remove_private_keys(project)
-        r, _, _, status = put_internal('projects', db_proj, _id=project['_id'])
-        if status != 200:
-            log.error('Error %i storing altered project %s %s', status, project['_id'], r)
-            raise SystemExit('Error storing project, see log.')
-        log.info('Project saved succesfully.')
-
-    if all_projects:
-        for project in projects_collection.find():
-            handle_project(project)
-        return
-
-    project = projects_collection.find_one({'url': proj_url})
-    if not project:
-        log.error('Project url=%s not found', proj_url)
-        return 3
-
-    handle_project(project)
-
-
-@manager_maintenance.command
-def remarkdown_comments():
-    """Retranslates all Markdown to HTML for all comment nodes.
-    """
-
-    from pillar.api.nodes import convert_markdown
-
-    nodes_collection = current_app.db()['nodes']
-    comments = nodes_collection.find({'node_type': 'comment'},
-                                     projection={'properties.content': 1,
-                                                 'node_type': 1})
-
-    updated = identical = skipped = errors = 0
-    for node in comments:
-        convert_markdown(node)
-        node_id = node['_id']
-
-        try:
-            content_html = node['properties']['content_html']
-        except KeyError:
-            log.warning('Node %s has no content_html', node_id)
-            skipped += 1
-            continue
-
-        result = nodes_collection.update_one(
-            {'_id': node_id},
-            {'$set': {'properties.content_html': content_html}}
-        )
-        if result.matched_count != 1:
-            log.error('Unable to update node %s', node_id)
-            errors += 1
-            continue
-
-        if result.modified_count:
-            updated += 1
-        else:
-            identical += 1
-
-    log.info('updated  : %i', updated)
-    log.info('identical: %i', identical)
-    log.info('skipped  : %i', skipped)
-    log.info('errors   : %i', errors)
-
-
-@manager_maintenance.command
-@manager_maintenance.option('-p', '--project', dest='proj_url', nargs='?',
-                help='Project URL')
-@manager_maintenance.option('-a', '--all', dest='all_projects', action='store_true', default=False,
-                help='Replace on all projects.')
-def upgrade_attachment_schema(proj_url=None, all_projects=False):
-    """Replaces the project's attachments with the new schema.
-
-    Updates both the schema definition and the nodes with attachments (asset, page, post).
-    """
-
-    if bool(proj_url) == all_projects:
-        log.error('Use either --project or --all.')
-        return 1
-
-    from pillar.api.utils.authentication import force_cli_user
-    force_cli_user()
-
-    from pillar.api.node_types.asset import node_type_asset
-    from pillar.api.node_types.page import node_type_page
-    from pillar.api.node_types.post import node_type_post
-    from pillar.api.node_types import _attachments_embedded_schema
-    from pillar.api.utils import remove_private_keys
-
-    # Node types that support attachments
-    node_types = (node_type_asset, node_type_page, node_type_post)
-    nts_by_name = {nt['name']: nt for nt in node_types}
-
-    db = current_app.db()
-    projects_coll = db['projects']
-    nodes_coll = db['nodes']
-
-    def handle_project(project):
-        log.info('Handling project %s', project['url'])
-
-        replace_schemas(project)
-        replace_attachments(project)
-
-    def replace_schemas(project):
-        for proj_nt in project['node_types']:
-            nt_name = proj_nt['name']
-            if nt_name not in nts_by_name:
-                continue
-
-            log.info('   - replacing attachment schema on node type "%s"', nt_name)
-            pillar_nt = nts_by_name[nt_name]
-            proj_nt['dyn_schema']['attachments'] = copy.deepcopy(_attachments_embedded_schema)
-
-            # Get the form schema the same as the official Pillar one, but only for attachments.
-            try:
-                pillar_form_schema = pillar_nt['form_schema']['attachments']
-            except KeyError:
-                proj_nt['form_schema'].pop('attachments', None)
-            else:
-                proj_nt['form_schema']['attachments'] = pillar_form_schema
-
-        # Use Eve to PUT, so we have schema checking.
-        db_proj = remove_private_keys(project)
-        r, _, _, status = put_internal('projects', db_proj, _id=project['_id'])
-        if status != 200:
-            log.error('Error %i storing altered project %s %s', status, project['_id'], r)
-            raise SystemExit('Error storing project, see log.')
-        log.info('Project saved succesfully.')
-
-    def replace_attachments(project):
-        log.info('Upgrading nodes for project %s', project['url'])
-        nodes = nodes_coll.find({
-            '_deleted': False,
-            'project': project['_id'],
-            'node_type': {'$in': list(nts_by_name)},
-            'properties.attachments': {'$exists': True},
-        })
-        for node in nodes:
-            attachments = node[u'properties'][u'attachments']
-            if isinstance(attachments, dict):
-                # This node has already been upgraded.
-                continue
-
-            log.info('    - Updating schema on node %s (%s)', node['_id'], node.get('name'))
-            new_atts = {}
-            for field_info in attachments:
-                for attachment in field_info.get('files', []):
-                    new_atts[attachment[u'slug']] = {u'oid': attachment[u'file']}
-
-            node[u'properties'][u'attachments'] = new_atts
-
-            # Use Eve to PUT, so we have schema checking.
-            db_node = remove_private_keys(node)
-            r, _, _, status = put_internal('nodes', db_node, _id=node['_id'])
-            if status != 200:
-                log.error('Error %i storing altered node %s %s', status, node['_id'], r)
-                raise SystemExit('Error storing node; see log.')
-
-    if all_projects:
-        for proj in projects_coll.find():
-            handle_project(proj)
-        return
-
-    proj = projects_coll.find_one({'url': proj_url})
-    if not proj:
-        log.error('Project url=%s not found', proj_url)
-        return 3
-
-    handle_project(proj)
-
-
-@manager_setup.command
-def create_blog(proj_url):
-    """Adds a blog to the project."""
-
-    from pillar.api.utils.authentication import force_cli_user
-    from pillar.api.utils import node_type_utils
-    from pillar.api.node_types.blog import node_type_blog
-    from pillar.api.node_types.post import node_type_post
-    from pillar.api.utils import remove_private_keys
-
-    force_cli_user()
-
-    db = current_app.db()
-
-    # Add the blog & post node types to the project.
-    projects_coll = db['projects']
-    proj = projects_coll.find_one({'url': proj_url})
-    if not proj:
-        log.error('Project url=%s not found', proj_url)
-        return 3
-
-    node_type_utils.add_to_project(proj,
-                                   (node_type_blog, node_type_post),
-                                   replace_existing=False)
-
-    proj_id = proj['_id']
-    r, _, _, status = put_internal('projects', remove_private_keys(proj), _id=proj_id)
-    if status != 200:
-        log.error('Error %i storing altered project %s %s', status, proj_id, r)
-        return 4
-    log.info('Project saved succesfully.')
-
-    # Create a blog node.
-    nodes_coll = db['nodes']
-    blog = nodes_coll.find_one({'node_type': 'blog', 'project': proj_id})
-    if not blog:
-        blog = {
-            u'node_type': node_type_blog['name'],
-            u'name': u'Blog',
-            u'description': u'',
-            u'properties': {},
-            u'project': proj_id,
-        }
-        r, _, _, status = post_internal('nodes', blog)
-        if status != 201:
-            log.error('Error %i storing blog node: %s', status, r)
-            return 4
-        log.info('Blog node saved succesfully: %s', r)
-    else:
-        log.info('Blog node already exists: %s', blog)
-
-    return 0
-
-manager.add_command("maintenance", manager_maintenance)
-manager.add_command("setup", manager_setup)
-manager.add_command("operations", manager_operations)

pillar/cli/__init__.py

@@ -0,0 +1,26 @@
+"""Commandline interface.
+
+Run commands with 'flask <command>'
+"""
+
+import logging
+
+from flask_script import Manager
+
+from pillar import current_app
+from pillar.cli.celery import manager_celery
+from pillar.cli.maintenance import manager_maintenance
+from pillar.cli.operations import manager_operations
+from pillar.cli.setup import manager_setup
+from pillar.cli.elastic import manager_elastic
+
+from pillar.cli import translations
+
+log = logging.getLogger(__name__)
+manager = Manager(current_app)
+
+manager.add_command('celery', manager_celery)
+manager.add_command("maintenance", manager_maintenance)
+manager.add_command("setup", manager_setup)
+manager.add_command("operations", manager_operations)
+manager.add_command("elastic", manager_elastic)

pillar/cli/celery.py

@@ -0,0 +1,69 @@
+import logging
+
+from flask_script import Manager
+from pillar import current_app
+
+log = logging.getLogger(__name__)
+
+manager_celery = Manager(
+    current_app, usage="Celery operations, like starting a worker or showing the queue")
+
+
+@manager_celery.option('args', nargs='*')
+def worker(args):
+    """Runs a Celery worker."""
+
+    import sys
+
+    argv0 = f'{sys.argv[0]} operations worker'
+    argvother = [
+        '-E',
+        '-l', 'INFO',
+        '--concurrency', '1',
+        '--pool', 'solo',  # No preforking, as PyMongo can't handle connect-before-fork.
+                           # We might get rid of this and go for the default Celery worker
+                           # preforking concurrency model, *if* we can somehow reset the
+                           # PyMongo client and reconnect after forking.
+    ] + list(args)
+
+    current_app.celery.worker_main([argv0] + argvother)
+
+
+@manager_celery.command
+def queue():
+    """Shows queued Celery tasks."""
+
+    from pprint import pprint
+
+    # Inspect all nodes.
+    i = current_app.celery.control.inspect()
+
+    print(50 * '=')
+    print('Tasks that have an ETA or are scheduled for later processing:')
+    pprint(i.scheduled())
+
+    print()
+    print('Tasks that are currently active:')
+    pprint(i.active())
+
+    print()
+    print('Tasks that have been claimed by workers:')
+    pprint(i.reserved())
+    print(50 * '=')
+
+
+@manager_celery.command
+def purge():
+    """Deletes queued Celery tasks."""
+
+    log.warning('Purging all pending Celery tasks.')
+    current_app.celery.control.purge()
+
+
+@manager_celery.option('args', nargs='*')
+def beat(args):
+    """Runs the Celery beat."""
+
+    from celery.bin.beat import beat
+
+    return beat(app=current_app.celery).run_from_argv('je moeder', args, command='beat')

pillar/cli/elastic.py

@@ -0,0 +1,164 @@
+import concurrent.futures
+import logging
+import typing
+
+import bson
+from flask_script import Manager
+
+from pillar import current_app
+from pillar.api.search import index
+
+log = logging.getLogger(__name__)
+
+manager_elastic = Manager(
+    current_app, usage="Elastic utilities")
+
+name_to_task = {
+    'nodes': index.ResetNodeIndex,
+    'users': index.ResetUserIndex,
+}
+REINDEX_THREAD_COUNT = 5
+
+
+@manager_elastic.option('indices', nargs='*')
+def reset_index(indices: typing.List[str]):
+    """
+    Destroy and recreate elastic indices
+
+    nodes, users
+    """
+
+    with current_app.app_context():
+        if not indices:
+            indices = name_to_task.keys()
+
+        for elk_index in indices:
+            try:
+                task = name_to_task[elk_index]()
+            except KeyError:
+                raise SystemError('Unknown elk_index, choose from %s' %
+                                  (', '.join(name_to_task.keys())))
+            task.execute()
+
+
+def _reindex_users():
+    db = current_app.db()
+    users_coll = db['users']
+
+    # Note that this also finds service accounts, which are filtered out
+    # in prepare_user_data(…)
+    users = users_coll.find()
+    user_count = users.count()
+    indexed = 0
+
+    log.info('Reindexing %d users in Elastic', user_count)
+
+    from pillar.celery.search_index_tasks import prepare_user_data
+    from pillar.api.search import elastic_indexing
+
+    app = current_app.real_app
+
+    def do_work(work_idx_user):
+        nonlocal indexed
+        idx, user = work_idx_user
+
+        with app.app_context():
+            if idx % 100 == 0:
+                log.info('Processing user %d/%d', idx+1, user_count)
+            to_index = prepare_user_data('', user=user)
+            if not to_index:
+                log.debug('not indexing user %s', user)
+                return
+
+            try:
+                elastic_indexing.push_updated_user(to_index)
+            except(KeyError, AttributeError):
+                log.exception('Field is missing for %s', user)
+            else:
+                indexed += 1
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=REINDEX_THREAD_COUNT) as executor:
+        result = executor.map(do_work, enumerate(users))
+
+        # When an exception occurs, it's enough to just iterate over the results.
+        # That will re-raise the exception in the main thread.
+        for ob in result:
+            log.debug('result: %s', ob)
+    log.info('Reindexed %d/%d users', indexed, user_count)
+
+
+def _public_project_ids() -> typing.List[bson.ObjectId]:
+    """Returns a list of ObjectIDs of public projects.
+
+    Memoized in setup_app().
+    """
+
+    proj_coll = current_app.db('projects')
+    result = proj_coll.find({'is_private': False}, {'_id': 1})
+    return [p['_id'] for p in result]
+
+
+def _reindex_nodes():
+    db = current_app.db()
+    nodes_coll = db['nodes']
+    nodes = nodes_coll.find({
+        'project': {'$in': _public_project_ids()},
+        '_deleted': {'$ne': True},
+    })
+    node_count = nodes.count()
+    indexed = 0
+
+    log.info('Nodes %d will be reindexed in Elastic', node_count)
+    app = current_app.real_app
+
+    from pillar.celery.search_index_tasks import prepare_node_data
+    from pillar.api.search import elastic_indexing
+
+    def do_work(work_idx_node):
+        nonlocal indexed
+
+        idx, node = work_idx_node
+        with app.app_context():
+            if idx % 100 == 0:
+                log.info('Processing node %d/%d', idx+1, node_count)
+            try:
+                to_index = prepare_node_data('', node=node)
+                elastic_indexing.index_node_save(to_index)
+            except (KeyError, AttributeError):
+                log.exception('Node %s is missing a field', node)
+            else:
+                indexed += 1
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=REINDEX_THREAD_COUNT) as executor:
+        result = executor.map(do_work, enumerate(nodes))
+        # When an exception occurs, it's enough to just iterate over the results.
+        # That will re-raise the exception in the main thread.
+        for ob in result:
+            log.debug('result: %s', ob)
+    log.info('Reindexed %d/%d nodes', indexed, node_count)
+
+
+@manager_elastic.option('indexname', nargs='?')
+@manager_elastic.option('-r', '--reset', default=False, action='store_true')
+def reindex(indexname='', reset=False):
+    import time
+    import datetime
+
+    start = time.time()
+
+    if reset:
+        log.info('Resetting first')
+        reset_index([indexname] if indexname else [])
+
+    if not indexname:
+        log.info('reindex everything..')
+        _reindex_nodes()
+        _reindex_users()
+    elif indexname == 'users':
+        log.info('Indexing %s', indexname)
+        _reindex_users()
+    elif indexname == 'nodes':
+        log.info('Indexing %s', indexname)
+        _reindex_nodes()
+    duration = time.time() - start
+    log.info('Reindexing took %s', datetime.timedelta(seconds=duration))

pillar/cli/operations.py

@@ -0,0 +1,250 @@
+import logging
+
+from flask_script import Manager
+from pillar import current_app
+
+log = logging.getLogger(__name__)
+
+manager_operations = Manager(
+    current_app, usage="Backend operations, like moving nodes across projects")
+
+
+@manager_operations.command
+def file_change_backend(file_id, dest_backend='gcs'):
+    """Given a file document, move it to the specified backend (if not already
+    there) and update the document to reflect that.
+    Files on the original backend are not deleted automatically.
+    """
+
+    from pillar.api.file_storage.moving import change_file_storage_backend
+    change_file_storage_backend(file_id, dest_backend)
+
+
+@manager_operations.command
+def mass_copy_between_backends(src_backend='cdnsun', dest_backend='gcs'):
+    """Copies all files from one backend to the other, updating them in Mongo.
+
+    Files on the original backend are not deleted.
+    """
+
+    import requests.exceptions
+
+    from pillar.api.file_storage import moving
+
+    logging.getLogger('pillar').setLevel(logging.INFO)
+    log.info('Mass-moving all files from backend %r to %r',
+             src_backend, dest_backend)
+
+    files_coll = current_app.data.driver.db['files']
+
+    fdocs = files_coll.find({'backend': src_backend},
+                            projection={'_id': True})
+    copied_ok = 0
+    copy_errs = 0
+    try:
+        for fdoc in fdocs:
+            try:
+                moving.change_file_storage_backend(fdoc['_id'], dest_backend)
+            except moving.PrerequisiteNotMetError as ex:
+                log.error('Error copying %s: %s', fdoc['_id'], ex)
+                copy_errs += 1
+            except requests.exceptions.HTTPError as ex:
+                log.error('Error copying %s (%s): %s',
+                          fdoc['_id'], ex.response.url, ex)
+                copy_errs += 1
+            except Exception:
+                log.exception('Unexpected exception handling file %s', fdoc['_id'])
+                copy_errs += 1
+            else:
+                copied_ok += 1
+    except KeyboardInterrupt:
+        log.error('Stopping due to keyboard interrupt')
+
+    log.info('%i files copied ok', copied_ok)
+    log.info('%i files we did not copy', copy_errs)
+
+
+@manager_operations.option('dest_proj_url', help='Destination project URL')
+@manager_operations.option('node_uuid', help='ID of the node to move')
+@manager_operations.option('-f', '--force', dest='force', action='store_true', default=False,
+                           help='Move even when already at the given project.')
+@manager_operations.option('-s', '--skip-gcs', dest='skip_gcs', action='store_true', default=False,
+                           help='Skip file handling on GCS, just update the database.')
+def move_group_node_project(node_uuid, dest_proj_url, force=False, skip_gcs=False):
+    """Copies all files from one project to the other, then moves the nodes.
+
+    The node and all its children are moved recursively.
+    """
+
+    from pillar.api.nodes import moving
+    from pillar.api.utils import str2id
+
+    logging.getLogger('pillar').setLevel(logging.INFO)
+
+    db = current_app.db()
+    nodes_coll = db['nodes']
+    projs_coll = db['projects']
+
+    # Parse CLI args and get the node, source and destination projects.
+    node_uuid = str2id(node_uuid)
+    node = nodes_coll.find_one({'_id': node_uuid})
+    if node is None:
+        log.error("Node %s can't be found!", node_uuid)
+        return 1
+
+    if node.get('parent', None):
+        log.error('Node cannot have a parent, it must be top-level.')
+        return 4
+
+    src_proj = projs_coll.find_one({'_id': node['project']})
+    dest_proj = projs_coll.find_one({'url': dest_proj_url})
+
+    if src_proj is None:
+        log.warning("Node's source project %s doesn't exist!", node['project'])
+    if dest_proj is None:
+        log.error("Destination project url='%s' doesn't exist.", dest_proj_url)
+        return 2
+    if src_proj['_id'] == dest_proj['_id']:
+        if force:
+            log.warning("Node is already at project url='%s'!", dest_proj_url)
+        else:
+            log.error("Node is already at project url='%s'!", dest_proj_url)
+            return 3
+
+    log.info("Mass-moving %s (%s) and children from project '%s' (%s) to '%s' (%s)",
+             node_uuid, node['name'], src_proj['url'], src_proj['_id'], dest_proj['url'],
+             dest_proj['_id'])
+
+    mover = moving.NodeMover(db=db, skip_gcs=skip_gcs)
+    mover.change_project(node, dest_proj)
+
+    log.info('Done moving.')
+
+
+@manager_operations.command
+def merge_project(src_proj_url, dest_proj_url):
+    """Move all nodes and files from one project to the other."""
+
+    from pillar.api.projects import merging
+
+    logging.getLogger('pillar').setLevel(logging.INFO)
+
+    log.info('Current server name is %s', current_app.config['SERVER_NAME'])
+    if not current_app.config['SERVER_NAME']:
+        log.fatal('SERVER_NAME configuration is missing, would result in malformed file links.')
+        return 5
+
+    # Parse CLI args and get source and destination projects.
+    projs_coll = current_app.db('projects')
+    src_proj = projs_coll.find_one({'url': src_proj_url}, projection={'_id': 1})
+    dest_proj = projs_coll.find_one({'url': dest_proj_url}, projection={'_id': 1})
+
+    if src_proj is None:
+        log.fatal("Source project url='%s' doesn't exist.", src_proj_url)
+        return 1
+    if dest_proj is None:
+        log.fatal("Destination project url='%s' doesn't exist.", dest_proj_url)
+        return 2
+    dpid = dest_proj['_id']
+    spid = src_proj['_id']
+    if spid == dpid:
+        log.fatal("Source and destination projects are the same!")
+        return 3
+
+    print()
+    try:
+        input(f'Press ENTER to start moving ALL NODES AND FILES '
+              f'from {src_proj_url} to {dest_proj_url}')
+    except KeyboardInterrupt:
+        print()
+        print('Aborted')
+        return 4
+    print()
+
+    merging.merge_project(spid, dpid)
+    log.info('Done moving.')
+
+
+@manager_operations.command
+def index_users_rebuild():
+    """Clear users index, update settings and reindex all users."""
+
+    import concurrent.futures
+
+    from pillar.api.utils.algolia import algolia_index_user_save
+
+    users_index = current_app.algolia_index_users
+    if users_index is None:
+        log.error('Algolia is not configured properly, unable to do anything!')
+        return 1
+
+    log.info('Dropping existing index: %s', users_index)
+    users_index.clear_index()
+    index_users_update_settings()
+
+    db = current_app.db()
+    users = db['users'].find({'_deleted': {'$ne': True}})
+    user_count = users.count()
+
+    log.info('Reindexing all %i users', user_count)
+
+    real_current_app = current_app._get_current_object()._get_current_object()
+
+    def do_user(user):
+        with real_current_app.app_context():
+            algolia_index_user_save(user)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+        future_to_user = {executor.submit(do_user, user): user
+                          for user in users}
+        for idx, future in enumerate(concurrent.futures.as_completed(future_to_user)):
+            user = future_to_user[future]
+            user_ident = user.get('email') or user.get('_id')
+            try:
+                future.result()
+            except Exception:
+                log.exception('Error updating user %i/%i %s', idx + 1, user_count, user_ident)
+            else:
+                log.info('Updated user %i/%i %s', idx + 1, user_count, user_ident)
+
+
+@manager_operations.command
+def index_users_update_settings():
+    """Configure indexing backend as required by the project"""
+    users_index = current_app.algolia_index_users
+
+    # Automatically creates index if it does not exist
+    users_index.set_settings({
+        'searchableAttributes': [
+            'full_name',
+            'username',
+            'email',
+            'unordered(roles)'
+        ]
+    })
+
+
+@manager_operations.command
+def hash_auth_tokens():
+    """Hashes all unhashed authentication tokens."""
+
+    from pymongo.results import UpdateResult
+    from pillar.api.utils.authentication import hash_auth_token
+
+    tokens_coll = current_app.db('tokens')
+    query = {'token': {'$exists': True}}
+    cursor = tokens_coll.find(query, projection={'token': 1, '_id': 1})
+    log.info('Updating %d tokens', cursor.count())
+
+    for token_doc in cursor:
+        hashed_token = hash_auth_token(token_doc['token'])
+        token_id = token_doc['_id']
+        res: UpdateResult = tokens_coll.update_one(
+            {'_id': token_id},
+            {'$set': {'token_hashed': hashed_token},
+             '$unset': {'token': 1}},
+        )
+        if res.modified_count != 1:
+            raise ValueError(f'Unable to update token {token_id}!')
+
+    log.info('Done')

pillar/cli/setup.py

@@ -0,0 +1,160 @@
+import logging
+
+from flask_script import Manager
+
+from pillar import current_app
+
+log = logging.getLogger(__name__)
+
+manager_setup = Manager(
+    current_app, usage="Setup utilities, like setup_db() or create_blog()")
+
+
+@manager_setup.command
+def setup_db(admin_email):
+    """Setup the database
+    - Create admin, subscriber and demo Group collection
+    - Create admin user (must use valid blender-id credentials)
+    - Create one project
+    """
+
+    # Create default groups
+    groups_list = []
+    for group in ['admin', 'subscriber', 'demo']:
+        g = {'name': group}
+        g = current_app.post_internal('groups', g)
+        groups_list.append(g[0]['_id'])
+        print("Creating group {0}".format(group))
+
+    # Create admin user
+    user = {'username': admin_email,
+            'groups': groups_list,
+            'roles': ['admin', 'subscriber', 'demo'],
+            'settings': {'email_communications': 1},
+            'auth': [],
+            'full_name': admin_email,
+            'email': admin_email}
+    result, _, _, status = current_app.post_internal('users', user)
+    if status != 201:
+        raise SystemExit('Error creating user {}: {}'.format(admin_email, result))
+    user.update(result)
+    print("Created user {0}".format(user['_id']))
+
+    # Create a default project by faking a POST request.
+    with current_app.test_request_context(data={'project_name': 'Default Project'}):
+        from flask import g
+        from pillar.auth import UserClass
+        from pillar.api.projects import routes as proj_routes
+
+        g.current_user = UserClass.construct('', user)
+
+        proj_routes.create_project(overrides={'url': 'default-project',
+                                              'is_private': False})
+
+
+@manager_setup.command
+def create_badger_account(email, badges):
+    """
+    Creates a new service account that can give badges (i.e. roles).
+
+    :param email: email address associated with the account
+    :param badges: single space-separated argument containing the roles
+        this account can assign and revoke.
+    """
+
+    create_service_account(email, ['badger'], {'badger': badges.strip().split()})
+
+
+@manager_setup.command
+def create_local_user_account(email, password):
+    from pillar.api.local_auth import create_local_user
+    create_local_user(email, password)
+
+
+@manager_setup.command
+def badger(action, user_email, role):
+    from pillar.api import service
+
+    with current_app.app_context():
+        service.fetch_role_to_group_id_map()
+        response, status = service.do_badger(action, role=role, user_email=user_email)
+
+    if status == 204:
+        log.info('Done.')
+    else:
+        log.info('Response: %s', response)
+        log.info('Status  : %i', status)
+
+
+@manager_setup.command
+def create_blog(proj_url):
+    """Adds a blog to the project."""
+
+    from pillar.api.utils.authentication import force_cli_user
+    from pillar.api.utils import node_type_utils
+    from pillar.api.node_types.blog import node_type_blog
+    from pillar.api.node_types.post import node_type_post
+    from pillar.api.utils import remove_private_keys
+
+    force_cli_user()
+
+    db = current_app.db()
+
+    # Add the blog & post node types to the project.
+    projects_coll = db['projects']
+    proj = projects_coll.find_one({'url': proj_url})
+    if not proj:
+        log.error('Project url=%s not found', proj_url)
+        return 3
+
+    node_type_utils.add_to_project(proj,
+                                   (node_type_blog, node_type_post),
+                                   replace_existing=False)
+
+    proj_id = proj['_id']
+    r, _, _, status = current_app.put_internal('projects', remove_private_keys(proj), _id=proj_id)
+    if status != 200:
+        log.error('Error %i storing altered project %s %s', status, proj_id, r)
+        return 4
+    log.info('Project saved succesfully.')
+
+    # Create a blog node.
+    nodes_coll = db['nodes']
+    blog = nodes_coll.find_one({'node_type': 'blog', 'project': proj_id})
+    if not blog:
+        blog = {
+            'node_type': node_type_blog['name'],
+            'name': 'Blog',
+            'description': '',
+            'properties': {},
+            'project': proj_id,
+        }
+        r, _, _, status = current_app.post_internal('nodes', blog)
+        if status != 201:
+            log.error('Error %i storing blog node: %s', status, r)
+            return 4
+        log.info('Blog node saved succesfully: %s', r)
+    else:
+        log.info('Blog node already exists: %s', blog)
+
+    return 0
+
+
+def create_service_account(email, service_roles, service_definition,
+                           *, full_name: str=None):
+    from pillar.api import service
+    from pillar.api.utils import dumps
+
+    account, token = service.create_service_account(
+        email,
+        service_roles,
+        service_definition,
+        full_name=full_name,
+    )
+
+    print('Service account information:')
+    print(dumps(account, indent=4, sort_keys=True))
+    print()
+    print('Access token: %s' % token['token'])
+    print('  expires on: %s' % token['expire_time'])
+    return account, token

pillar/cli/translations.py

@@ -0,0 +1,104 @@
+import argparse
+import contextlib
+import pathlib
+import subprocess
+import sys
+
+BABEL_CONFIG = pathlib.Path('translations.cfg')
+
+
+@contextlib.contextmanager
+def create_messages_pot() -> pathlib.Path:
+    """Extract the translatable strings from the source code
+
+    This creates a temporary messages.pot file, to be used to init or
+    update the translation .mo files.
+
+    It works as a generator, yielding the temporarily created pot file.
+    The messages.pot file will be deleted at the end of it if all went well.
+
+    :return The path of the messages.pot file created.
+    """
+    if not BABEL_CONFIG.is_file():
+        print("No translations config file found: %s" % (BABEL_CONFIG))
+        sys.exit(-1)
+        return
+
+    messages_pot = pathlib.Path('messages.pot')
+    subprocess.run(('pybabel', 'extract', '-F', BABEL_CONFIG, '-k', 'lazy_gettext', '-o', messages_pot, '.'))
+    yield messages_pot
+    messages_pot.unlink()
+
+
+def init(locale):
+    """
+    Initialize the translations for a new language.
+    """
+    with create_messages_pot() as messages_pot:
+        subprocess.run(('pybabel', 'init', '-i', messages_pot, '-d', 'translations', '-l', locale))
+
+
+def update():
+    """
+    Update the strings to be translated.
+    """
+    with create_messages_pot() as messages_pot:
+        subprocess.run(('pybabel', 'update', '-i', messages_pot, '-d', 'translations'))
+
+
+def compile():
+    """
+    Compile the translation to be used.
+    """
+    if pathlib.Path('translations').is_dir():
+        subprocess.run(('pybabel', 'compile','-d', 'translations'))
+    else:
+        print("No translations folder available")
+
+
+def parse_arguments() -> argparse.Namespace:
+    """
+    Parse command-line arguments.
+    """
+    parser = argparse.ArgumentParser(description='Translate Pillar')
+
+    parser.add_argument(
+            'mode',
+            type=str,
+            help='Init once, update often, compile before deploying.',
+            choices=['init', 'update', 'compile'])
+
+    parser.add_argument(
+            'languages',
+            nargs='*',
+            type=str,
+            help='Languages to initialize: pt it es ...')
+
+    args = parser.parse_args()
+    if args.mode == 'init' and not args.languages:
+        parser.error("init requires languages")
+
+    return args
+
+
+def main():
+    """
+    When calling from the setup.py entry-point we need to parse the arguments
+    and init/update/compile the translations strings
+    """
+    args = parse_arguments()
+
+    if args.mode == 'init':
+        for language in args.languages:
+            init(language)
+
+    elif args.mode == 'update':
+        update()
+
+    else: # mode == 'compile'
+        compile()
+
+
+if __name__ == '__main__':
+    main()
+

pillar/config.py

@@ -13,17 +13,26 @@ RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
 PILLAR_SERVER_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 SCHEME = 'https'
+PREFERRED_URL_SCHEME = SCHEME
+
+# Be sure to set this in your config_local:
+# SERVER_NAME = 'pillar.local:5000'
+# PILLAR_SERVER_ENDPOINT = f'{SCHEME}://{SERVER_NAME}/api/'
+
 STORAGE_DIR = getenv('PILLAR_STORAGE_DIR', '/data/storage/pillar')
 PORT = 5000
 HOST = '0.0.0.0'
 DEBUG = False
 
-SECRET_KEY = '123'
+# Flask and CSRF secret key; generate local one with:
+# python3 -c 'import secrets; print(secrets.token_urlsafe(128))'
+SECRET_KEY = ''
+
+# Authentication token hashing key. If empty falls back to UTF8-encoded SECRET_KEY with a warning.
+AUTH_TOKEN_HMAC_KEY = b''
 
 # Authentication settings
-BLENDER_ID_ENDPOINT = 'http://blender_id:8000/'
-
-PILLAR_SERVER_ENDPOINT = 'http://pillar:5001/api/'
+BLENDER_ID_ENDPOINT = 'http://id.local:8000'
 
 CDN_USE_URL_SIGNING = True
 CDN_SERVICE_DOMAIN_PROTOCOL = 'https'
@@ -53,17 +62,33 @@ BIN_RSYNC = '/usr/bin/rsync'
 
 GCLOUD_APP_CREDENTIALS = 'google_app.json'
 GCLOUD_PROJECT = '-SECRET-'
+# Used for cross-verification on various Google sites (eg. YouTube)
+GOOGLE_SITE_VERIFICATION = ''
 
 ADMIN_USER_GROUP = '5596e975ea893b269af85c0e'
 SUBSCRIBER_USER_GROUP = '5596e975ea893b269af85c0f'
-BUGSNAG_API_KEY = ''
+
+SENTRY_CONFIG = {
+    'dsn': '-set-in-config-local-',
+    # 'release': raven.fetch_git_sha(os.path.dirname(__file__)),
+}
+# See https://docs.sentry.io/clients/python/integrations/flask/#settings
+SENTRY_USER_ATTRS = ['username', 'full_name', 'email', 'objectid']
 
 ALGOLIA_USER = '-SECRET-'
 ALGOLIA_API_KEY = '-SECRET-'
 ALGOLIA_INDEX_USERS = 'dev_Users'
 ALGOLIA_INDEX_NODES = 'dev_Nodes'
 
-SEARCH_BACKEND = 'algolia'  # algolia, elastic
+SEARCH_BACKENDS = ('elastic', )
+
+ELASTIC_INDICES = {
+    'NODE': 'nodes',
+    'USER': 'users',
+}
+
+ELASTIC_SEARCH_HOSTS = ['elastic:9200']
+
 
 ZENCODER_API_KEY = '-SECRET-'
 ZENCODER_NOTIFICATIONS_SECRET = '-SECRET-'
@@ -71,6 +96,10 @@ ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/'
 
 ENCODING_BACKEND = 'zencoder'  # local, flamenco
 
+# Storage solution for uploaded files. If 'local' is selected, make sure you specify the SERVER_NAME
+# config value as well, since it will help building correct URLs when indexing.
+STORAGE_BACKEND = 'local'  # gcs
+
 # Validity period of links, per file storage backend. Expressed in seconds.
 # Shouldn't be more than a year, as this isn't supported by HTTP/1.1.
 FILE_LINK_VALIDITY = defaultdict(
@@ -78,13 +107,33 @@ FILE_LINK_VALIDITY = defaultdict(
     gcs=3600 * 23,  # 23 hours for Google Cloud Storage.
 )
 
-# Roles with full GET-access to all variations of files.
-FULL_FILE_ACCESS_ROLES = {u'admin', u'subscriber', u'demo'}
+# Capability with GET-access to all variations of files.
+FULL_FILE_ACCESS_CAP = 'subscriber'
 
 # Client and Subclient IDs for Blender ID
 BLENDER_ID_CLIENT_ID = 'SPECIAL-SNOWFLAKE-57'
 BLENDER_ID_SUBCLIENT_ID = 'PILLAR'
 
+# Blender ID user info API endpoint URL and auth token, used for
+# reconciling subscribers and updating their info from /u/.
+# The token requires the 'userinfo' scope.
+BLENDER_ID_USER_INFO_API = 'http://blender-id:8000/api/user/'
+BLENDER_ID_USER_INFO_TOKEN = '-set-in-config-local-'
+
+# Collection of supported OAuth providers (Blender ID, Facebook and Google).
+# Example entry:
+# OAUTH_CREDENTIALS = {
+#    'blender-id': {
+#        'id': 'CLOUD-OF-SNOWFLAKES-42',
+#        'secret': 'thesecret',
+#     }
+# }
+# OAuth providers are defined in pillar.auth.oauth
+OAUTH_CREDENTIALS = {
+    'blender-id': {},
+    'facebook': {},
+    'google': {},
+}
 
 # See https://docs.python.org/2/library/logging.config.html#configuration-dictionary-schema
 LOGGING = {
@@ -117,8 +166,9 @@ SHORT_CODE_LENGTH = 6  # characters
 # People are allowed this many bytes per uploaded file.
 FILESIZE_LIMIT_BYTES_NONSUBS = 32 * 2 ** 20
 # Unless they have one of those roles.
-ROLES_FOR_UNLIMITED_UPLOADS = {u'subscriber', u'demo', u'admin'}
+ROLES_FOR_UNLIMITED_UPLOADS = {'subscriber', 'demo', 'admin'}
 
+ROLES_FOR_COMMENT_VOTING = {'subscriber', 'demo'}
 
 #############################################
 # Old pillar-web config:
@@ -134,16 +184,76 @@ GIT = 'git'
 RENDER_HOME_AS_REGULAR_PROJECT = False
 
 
-# Authentication token for the Urler service. If None, defaults
-# to the authentication token of the current user.
-URLER_SERVICE_AUTH_TOKEN = None
-
-
 # Blender Cloud add-on version. This updates the value in all places in the
 # front-end.
 BLENDER_CLOUD_ADDON_VERSION = '1.4'
 
-EXTERNAL_SUBSCRIPTIONS_MANAGEMENT_SERVER = 'https://store.blender.org/api/'
-
 # Certificate file for communication with other systems.
 TLS_CERT_FILE = requests.certs.where()
+
+CELERY_BACKEND = 'redis://redis/1'
+CELERY_BROKER = 'amqp://guest:guest@rabbit//'
+
+# This configures the Celery task scheduler in such a way that we don't
+# have to import the pillar.celery.XXX modules. Remember to run
+# 'manage.py celery beat' too, otherwise those will never run.
+CELERY_BEAT_SCHEDULE = {
+    'regenerate-expired-links': {
+        'task': 'pillar.celery.file_link_tasks.regenerate_all_expired_links',
+        'schedule': 600,  # every N seconds
+        'args': ('gcs', 100)
+    },
+}
+
+# Mapping from user role to capabilities obtained by users with that role.
+USER_CAPABILITIES = defaultdict(**{
+    'subscriber': {'subscriber', 'home-project'},
+    'demo': {'subscriber', 'home-project'},
+    'admin': {'encode-video', 'admin',
+              'view-pending-nodes', 'edit-project-node-types', 'create-organization'},
+    'video-encoder': {'encode-video'},
+    'org-subscriber': {'subscriber', 'home-project'},
+}, default_factory=frozenset)
+
+
+# Internationalization and localization
+
+# The default locale is US English.
+# A locale can include a territory, a codeset and a modifier.
+# We only support locale strings with or without territories though.
+# For example, nl_NL and pt_BR are not the same language as nl_BE, and pt_PT.
+# However we can have a nl, or a pt translation, to be used as a common
+# translation when no territorial specific locale is available.
+# All translations should be in UTF-8.
+# This setting is used as a fallback when there is no good match between the
+# browser language and the available translations.
+DEFAULT_LOCALE = 'en_US'
+# All the available languages will be determined based on available translations
+# in the //translations/ folder. The exception is English, since all the text is
+# originally in English already. That said, if rare occasions we may want to
+# never show the site in English.
+SUPPORT_ENGLISH = True
+
+
+# Mail options, see pillar.celery.email_tasks.
+SMTP_HOST = 'localhost'
+SMTP_PORT = 2525
+SMTP_USERNAME = ''
+SMTP_PASSWORD = ''
+SMTP_TIMEOUT = 30  # timeout in seconds, https://docs.python.org/3/library/smtplib.html#smtplib.SMTP
+MAIL_RETRY = 180  # in seconds, delay until trying to send an email again.
+MAIL_DEFAULT_FROM_NAME = 'Blender Cloud'
+MAIL_DEFAULT_FROM_ADDR = 'cloudsupport@localhost'
+
+SEND_FILE_MAX_AGE_DEFAULT = 3600 * 24 * 365  # seconds
+
+# MUST be 8 characters long, see pillar.flask_extra.HashedPathConverter
+# Intended to be changed for every deploy. If it is empty, a random hash will
+# be used. Note that this causes extra traffic, since every time the process
+# restarts the URLs will be different.
+STATIC_FILE_HASH = ''
+
+# Disable default CSRF protection for all views, since most web endpoints and
+# all API endpoints do not need it. On the views that require it, we use the
+# current_app.csrf.protect() method.
+WTF_CSRF_CHECK_DEFAULT = False

pillar/extension.py

@@ -16,12 +16,34 @@ can then be registered to the application at app creation time:
 """
 
 import abc
+import inspect
+import pathlib
+import typing
+
+import flask
+import pillarsdk
 
 
-class PillarExtension(object):
-    __metaclass__ = abc.ABCMeta
+class PillarExtension(object, metaclass=abc.ABCMeta):
+    # Set to True when your extension implements the project_settings() method.
+    has_project_settings = False
 
-    @abc.abstractproperty
+    # Set to True when your extension implements the context_processor() method.
+    has_context_processor = False
+
+    # List of Celery task modules introduced by this extension.
+    celery_task_modules: typing.List[str] = []
+
+    # Set of user roles used/introduced by this extension.
+    user_roles: typing.Set[str] = set()
+    user_roles_indexable: typing.Set[str] = set()
+
+    # User capabilities introduced by this extension. The final set of
+    # capabilities is the union of all app-level and extension-level caps.
+    user_caps: typing.Mapping[str, typing.FrozenSet] = {}
+
+    @property
+    @abc.abstractmethod
     def name(self):
         """The name of this extension.
 
@@ -33,6 +55,14 @@ class PillarExtension(object):
         :rtype: unicode
         """
 
+    @property
+    def icon(self) -> str:
+        """Returns the icon HTML class, for use like i.pi-{{ext.icon}}
+
+        Defaults to the extension name.
+        """
+        return self.name
+
     @abc.abstractmethod
     def flask_config(self):
         """Returns extension-specific defaults for the Flask configuration.
@@ -84,13 +114,48 @@ class PillarExtension(object):
         """
         return None
 
+    @property
+    def translations_path(self) -> typing.Union[pathlib.Path, None]:
+        """Returns the path where the translations for this extension are stored.
+
+        This is top folder that contains a "translations" sub-folder
+
+        May return None, in which case English will always be used for this extension.
+        """
+        class_filename = pathlib.Path(inspect.getfile(self.__class__))
+
+        # Pillar extensions instantiate the PillarExtension from a sub-folder in
+        # the main project (e.g. //blender_cloud/blender_cloud/__init__.py), but
+        # the translations folders is in the main project folder.
+        translations_path = class_filename.parents[1] / 'translations'
+
+        return translations_path if translations_path.is_dir() else None
+
     def setup_app(self, app):
         """Called during app startup, after all extensions have loaded."""
 
-    def sidebar_links(self, project):
+    def sidebar_links(self, project: pillarsdk.Project) -> str:
         """Returns the sidebar link(s) for the given projects.
 
         :returns: HTML as a string for the sidebar.
         """
 
         return ''
+
+    def project_settings(self, project: pillarsdk.Project, **template_args: dict) -> flask.Response:
+        """Renders the project settings page for this extension.
+
+        Set YourExtension.has_project_settings = True and Pillar will call this function.
+
+        :param project: the project for which to render the settings.
+        :param template_args: additional template arguments.
+        :returns: a Flask HTTP response
+        """
+
+    def context_processor(self) -> dict:
+        """Returns a dictionary that gets injected into the Flask Jinja2 namespace.
+
+        Set has_context_processor  to True when your extension implements this method.
+        """
+
+        return {}

pillar/flask_extra.py

@@ -0,0 +1,76 @@
+import re
+import functools
+
+import flask
+import werkzeug.routing
+
+
+class HashedPathConverter(werkzeug.routing.PathConverter):
+    """Allows for files `xxx.yyy.js` to be served as `xxx.yyy.abc123.js`.
+
+    The hash code is placed before the last extension.
+    """
+    weight = 300
+    # Hash length is hard-coded to 8 characters for now.
+    hash_re = re.compile(r'\.([a-zA-Z0-9]{8})(?=\.[^.]+$)')
+
+    @functools.lru_cache(maxsize=1024)
+    def to_python(self, from_url: str) -> str:
+        return self.hash_re.sub('', from_url)
+
+    @functools.lru_cache(maxsize=1024)
+    def to_url(self, filepath: str) -> str:
+        try:
+            dotidx = filepath.rindex('.')
+        except ValueError:
+            # Happens when there is no dot. Very unlikely.
+            return filepath
+
+        current_hash = flask.current_app.config['STATIC_FILE_HASH']
+        before, after = filepath[:dotidx], filepath[dotidx:]
+        return f'{before}.{current_hash}{after}'
+
+
+def add_response_headers(headers: dict):
+    """This decorator adds the headers passed in to the response"""
+
+    def decorator(f):
+        @functools.wraps(f)
+        def decorated_function(*args, **kwargs):
+            resp = flask.make_response(f(*args, **kwargs))
+            h = resp.headers
+            for header, value in headers.items():
+                h[header] = value
+            return resp
+
+        return decorated_function
+
+    return decorator
+
+
+def vary_xhr():
+    """View function decorator; adds HTTP header "Vary: X-Requested-With" to the response"""
+
+    def decorator(f):
+        header_adder = add_response_headers({'Vary': 'X-Requested-With'})
+        return header_adder(f)
+
+    return decorator
+
+
+def ensure_schema(url: str) -> str:
+    """Return the same URL with the configured PREFERRED_URL_SCHEME."""
+    import urllib.parse
+
+    if not url:
+        return url
+
+    bits = urllib.parse.urlsplit(url, allow_fragments=True)
+
+    if not bits[0] and not bits[1]:
+        # don't replace the schema if there is not even a hostname.
+        return url
+
+    scheme = flask.current_app.config.get('PREFERRED_URL_SCHEME', 'https')
+    bits = (scheme, *bits[1:])
+    return urllib.parse.urlunsplit(bits)

pillar/markdown.py

@@ -3,11 +3,11 @@
 This is for user-generated stuff, like comments.
 """
 
-from __future__ import absolute_import
-
 import bleach
 import CommonMark
 
+from . import shortcodes
+
 ALLOWED_TAGS = [
     'a',
     'abbr',
@@ -17,13 +17,14 @@ ALLOWED_TAGS = [
     'del', 'kbd',
     'dl', 'dt', 'dd',
     'blockquote',
-    'code',
+    'code', 'pre',
     'li', 'ol', 'ul',
     'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
     'p', 'br', 'hr',
     'sup', 'sub', 'strike',
     'img',
     'iframe',
+    'video',
 ]
 
 ALLOWED_ATTRIBUTES = {
@@ -32,6 +33,7 @@ ALLOWED_ATTRIBUTES = {
     'acronym': ['title'],
     'img': ['src', 'alt', 'width', 'height', 'title'],
     'iframe': ['src', 'width', 'height', 'frameborder', 'allowfullscreen'],
+    'video': ['autoplay', 'controls', 'loop', 'muted', 'src'],
     '*': ['style'],
 }
 
@@ -40,10 +42,24 @@ ALLOWED_STYLES = [
 ]
 
 
-def markdown(s):
-    tainted_html = CommonMark.commonmark(s)
-    safe_html = bleach.clean(tainted_html,
-                             tags=ALLOWED_TAGS,
+def markdown(s: str) -> str:
+    commented_shortcodes = shortcodes.comment_shortcodes(s)
+    tainted_html = CommonMark.commonmark(commented_shortcodes)
+
+    # Create a Cleaner that supports parsing of bare links (see filters).
+    cleaner = bleach.Cleaner(tags=ALLOWED_TAGS,
                              attributes=ALLOWED_ATTRIBUTES,
-                             styles=ALLOWED_STYLES)
+                             styles=ALLOWED_STYLES,
+                             strip_comments=False,
+                             filters=[bleach.linkifier.LinkifyFilter])
+
+    safe_html = cleaner.clean(tainted_html)
     return safe_html
+
+
+def cache_field_name(field_name: str) -> str:
+    """Return the field name containing the cached HTML.
+
+    See ValidateCustomFields._normalize_coerce_markdown().
+    """
+    return f'_{field_name}_html'

pillar/sdk.py

@@ -1,7 +1,7 @@
 """PillarSDK subclass for direct Flask-internal calls."""
 
 import logging
-import urlparse
+import urllib.parse
 from flask import current_app
 
 import pillarsdk
@@ -23,8 +23,8 @@ class FlaskInternalApi(pillarsdk.Api):
         self.requests_to_flask_kwargs(kwargs)
 
         # Leave out the query string and fragment from the URL.
-        split_url = urlparse.urlsplit(url)
-        path = urlparse.urlunsplit(split_url[:-2] + (None, None))
+        split_url = urllib.parse.urlsplit(url)
+        path = urllib.parse.urlunsplit(split_url[:-2] + (None, None))
         try:
             response = client.open(path=path, query_string=split_url.query, method=method,
                                    **kwargs)
@@ -41,8 +41,8 @@ class FlaskInternalApi(pillarsdk.Api):
         try:
             content = self.handle_response(response, response.data)
         except:
-            log.warning("%s: Response[%s]: %s", url, response.status_code,
-                        response.data)
+            log.debug("%s: Response[%s]: %s", url, response.status_code,
+                      response.data, exc_info=True)
             raise
 
         return content
@@ -94,8 +94,9 @@ class FlaskInternalApi(pillarsdk.Api):
             return response
 
         exception = exceptions.exception_for_status(response.status_code)
+        text = getattr(response, 'text', '')
         if exception:
-            raise exception(response, response.text)
+            raise exception(response, text)
 
-        raise exceptions.ConnectionError(response, response.text,
+        raise exceptions.ConnectionError(response, text,
                                          "Unknown response code: %s" % response.status_code)

pillar/sentry_extra.py

@@ -0,0 +1,56 @@
+from raven.contrib.flask import Sentry
+
+from .auth import current_user
+from . import current_app
+
+
+class PillarSentry(Sentry):
+    """Flask Sentry with Pillar support.
+
+    This is mostly for obtaining user information on API calls,
+    and for preventing the auth tokens to be logged as user ID.
+    """
+
+    def init_app(self, app, *args, **kwargs):
+        super().init_app(app, *args, **kwargs)
+
+        # We perform authentication of the user while handling the request,
+        # so Sentry calls get_user_info() too early.
+
+    def get_user_context_again(self, ):
+        from flask import request
+
+        try:
+            self.client.user_context(self.get_user_info(request))
+        except Exception as e:
+            self.client.logger.exception(str(e))
+
+    def get_user_info(self, request):
+        user_info = super().get_user_info(request)
+
+        # The auth token is stored as the user ID in the flask_login
+        # current_user object, so don't send that to Sentry.
+        user_info.pop('id', None)
+
+        if len(user_info) > 1:
+            # Sentry always includes the IP address, but when they find a
+            # logged-in user, they add more info. In that case we're done.
+            return user_info
+
+        # This is pretty much a copy-paste from Sentry, except that it uses
+        # pillar.auth.current_user instead.
+        try:
+            if not current_user.is_authenticated:
+                return user_info
+        except AttributeError:
+            # HACK: catch the attribute error thrown by flask-login is not attached
+            # >   current_user = LocalProxy(lambda: _request_ctx_stack.top.user)
+            # E   AttributeError: 'RequestContext' object has no attribute 'user'
+            return user_info
+
+        if 'SENTRY_USER_ATTRS' in current_app.config:
+            for attr in current_app.config['SENTRY_USER_ATTRS']:
+                if hasattr(current_user, attr):
+                    user_info[attr] = getattr(current_user, attr)
+
+        return user_info

pillar/shortcodes.py

@@ -0,0 +1,349 @@
+"""Shortcode rendering.
+
+Shortcodes are little snippets between square brackets, which can be rendered
+into HTML. Markdown passes such snippets unchanged to its HTML output, so this
+module assumes its input is HTML-with-shortcodes.
+
+See mulholland.xyz/docs/shortcodes/.
+
+{iframe src='http://hey' has-cap='subscriber'}
+
+NOTE: nested braces fail, so something like {shortcode abc='{}'} is not
+supported.
+
+NOTE: only single-line shortcodes are supported for now, due to the need to
+pass them though Markdown unscathed.
+
+See https://pillarframework.org/shortcodes/ for documentation.
+"""
+import html as html_module  # I want to be able to use the name 'html' in local scope.
+import logging
+import re
+import typing
+import urllib.parse
+
+import shortcodes
+import pillarsdk
+
+_parser: shortcodes.Parser = None
+_commented_parser: shortcodes.Parser = None
+log = logging.getLogger(__name__)
+
+
+def shortcode(name: str):
+    """Class decorator for shortcodes."""
+
+    def decorator(decorated):
+        assert hasattr(decorated, '__call__'), '@shortcode should be used on callables.'
+        if isinstance(decorated, type):
+            as_callable = decorated()
+        else:
+            as_callable = decorated
+        shortcodes.register(name)(as_callable)
+        return decorated
+
+    return decorator
+
+
+class capcheck:
+    """Decorator for shortcodes.
+
+    On call, check for capabilities before calling the function. If the user does not
+    have a capability, display a message insdead of the content.
+
+    kwargs:
+        - 'cap': Capability required for viewing.
+        - 'nocap': Optional, text shown when the user does not have this capability.
+        - others: Passed to the decorated shortcode.
+    """
+
+    def __init__(self, decorated):
+        assert hasattr(decorated, '__call__'), '@capcheck should be used on callables.'
+        if isinstance(decorated, type):
+            as_callable = decorated()
+        else:
+            as_callable = decorated
+        self.decorated = as_callable
+
+    def __call__(self,
+                 context: typing.Any,
+                 content: str,
+                 pargs: typing.List[str],
+                 kwargs: typing.Dict[str, str]) -> str:
+        from pillar.auth import current_user
+
+        cap = kwargs.pop('cap', '')
+        if cap:
+            nocap = kwargs.pop('nocap', '')
+            if not current_user.has_cap(cap):
+                if not nocap:
+                    return ''
+                html = html_module.escape(nocap)
+                return f'<p class="shortcode nocap">{html}</p>'
+
+        return self.decorated(context, content, pargs, kwargs)
+
+
+@shortcode('test')
+class Test:
+    def __call__(self,
+                 context: typing.Any,
+                 content: str,
+                 pargs: typing.List[str],
+                 kwargs: typing.Dict[str, str]) -> str:
+        """Just for testing.
+
+        "{test abc='def'}" → "<dl><dt>test</dt><dt>abc</dt><dd>def</dd></dl>"
+        """
+
+        parts = ['<dl><dt>test</dt>']
+
+        e = html_module.escape
+        parts.extend([
+            f'<dt>{e(key)}</dt><dd>{e(value)}</dd>' for key, value in kwargs.items()
+        ])
+        parts.append('</dl>')
+        return ''.join(parts)
+
+
+@shortcode('youtube')
+@capcheck
+class YouTube:
+    log = log.getChild('YouTube')
+
+    def video_id(self, url: str) -> str:
+        """Find the video ID from a YouTube URL.
+
+        :raise ValueError: when the ID cannot be determined.
+        """
+
+        if re.fullmatch(r'[a-zA-Z0-9_\-]+', url):
+            return url
+
+        try:
+            parts = urllib.parse.urlparse(url)
+            if parts.netloc == 'youtu.be':
+                return parts.path.split('/')[1]
+            if parts.netloc in {'www.youtube.com', 'youtube.com'}:
+                if parts.path.startswith('/embed/'):
+                    return parts.path.split('/')[2]
+                if parts.path.startswith('/watch'):
+                    qs = urllib.parse.parse_qs(parts.query)
+                    return qs['v'][0]
+        except (ValueError, IndexError, KeyError) as ex:
+            pass
+
+        raise ValueError(f'Unable to parse YouTube URL {url!r}')
+
+    def __call__(self,
+                 context: typing.Any,
+                 content: str,
+                 pargs: typing.List[str],
+                 kwargs: typing.Dict[str, str]) -> str:
+        """Embed a YouTube video.
+
+        The first parameter must be the YouTube video ID or URL. The width and
+        height can be passed in the equally named keyword arguments.
+        """
+
+        width = kwargs.get('width', '560')
+        height = kwargs.get('height', '315')
+
+        # Figure out the embed URL for the video.
+        try:
+            youtube_src = pargs[0]
+        except IndexError:
+            return html_module.escape('{youtube missing YouTube ID/URL}')
+
+        try:
+            youtube_id = self.video_id(youtube_src)
+        except ValueError as ex:
+            return html_module.escape('{youtube %s}' % "; ".join(ex.args))
+        if not youtube_id:
+            return html_module.escape('{youtube invalid YouTube ID/URL}')
+
+        src = f'https://www.youtube.com/embed/{youtube_id}?rel=0'
+        iframe_tag = f'<iframe class="shortcode youtube embed-responsive-item" width="{width}"' \
+                     f' height="{height}" src="{src}" frameborder="0" allow="autoplay; encrypted-media"' \
+                     f' allowfullscreen></iframe>'
+        # Embed the iframe in a container, to allow easier styling
+        html = f'<div class="embed-responsive embed-responsive-16by9">{iframe_tag}</div>'
+        return html
+
+
+@shortcode('iframe')
+@capcheck
+def iframe(context: typing.Any,
+           content: str,
+           pargs: typing.List[str],
+           kwargs: typing.Dict[str, str]) -> str:
+    """Show an iframe to users with the required capability.
+
+    kwargs:
+        - 'cap': Capability required for viewing.
+        - others: Turned into attributes for the iframe element.
+    """
+    import xml.etree.ElementTree as ET
+
+    kwargs['class'] = f'shortcode {kwargs.get("class", "")}'.strip()
+    element = ET.Element('iframe', kwargs)
+    html = ET.tostring(element, encoding='unicode', method='html', short_empty_elements=True)
+    return html
+
+
+@shortcode('attachment')
+class Attachment:
+    class NoSuchSlug(ValueError):
+        """Raised when there is no attachment with the given slug."""
+
+    class NoSuchFile(ValueError):
+        """Raised when the file pointed to by the attachment doesn't exist."""
+
+    class NotSupported(ValueError):
+        """Raised when an attachment is not pointing to a file."""
+
+    def __call__(self,
+                 context: typing.Any,
+                 content: str,
+                 pargs: typing.List[str],
+                 kwargs: typing.Dict[str, str]) -> str:
+        if isinstance(context, pillarsdk.Resource):
+            context = context.to_dict()
+        if not isinstance(context, dict):
+            return '{attachment context not a dictionary}'
+
+        try:
+            slug = pargs[0]
+        except KeyError:
+            return '{attachment No slug given}'
+
+        try:
+            file_doc = self.sdk_file(slug, context)
+        except self.NoSuchSlug:
+            return html_module.escape('{attachment %r does not exist}' % slug)
+        except self.NoSuchFile:
+            return html_module.escape('{attachment file for %r does not exist}' % slug)
+        except self.NotSupported as ex:
+            return html_module.escape('{attachment %s}' % ex)
+
+        return self.render(file_doc, pargs, kwargs)
+
+    def sdk_file(self, slug: str, node_properties: dict) -> pillarsdk.File:
+        """Return the file document for the attachment with this slug."""
+
+        from pillar.web import system_util
+
+        attachments = node_properties.get('attachments', {})
+        attachment = attachments.get(slug)
+        if not attachment:
+            raise self.NoSuchSlug(slug)
+
+        object_id = attachment.get('oid')
+        if not object_id:
+            raise self.NoSuchFile(object_id)
+
+        # In theory attachments can also point to other collections.
+        # There is no support for that yet, though.
+        collection = attachment.get('collection', 'files')
+        if collection != 'files':
+            log.warning('Attachment %r points to ObjectID %s in unsupported collection %r',
+                        slug, object_id, collection)
+            raise self.NotSupported(f'unsupported collection {collection!r}')
+
+        api = system_util.pillar_api()
+        sdk_file = pillarsdk.File.find(object_id, api=api)
+        return sdk_file
+
+    def render(self, sdk_file: pillarsdk.File,
+               pargs: typing.List[str],
+               kwargs: typing.Dict[str, str]) -> str:
+        file_renderers = {
+            'image': self.render_image,
+            'video': self.render_video,
+        }
+
+        mime_type_cat, _ = sdk_file.content_type.split('/', 1)
+        renderer = file_renderers.get(mime_type_cat, self.render_generic)
+        return renderer(sdk_file, pargs, kwargs)
+
+    def render_generic(self, sdk_file,
+                       pargs: typing.List[str],
+                       kwargs: typing.Dict[str, str]):
+        import flask
+        return flask.render_template('nodes/attachments/file_generic.html',
+                                     file=sdk_file, tag_args=kwargs)
+
+    def render_image(self, sdk_file,
+                     pargs: typing.List[str],
+                     kwargs: typing.Dict[str, str]):
+        """Renders an image file."""
+        import flask
+        if 'link' in pargs:
+            kwargs['link'] = 'self'
+        variations = {var.size: var for var in sdk_file.variations}
+        return flask.render_template('nodes/attachments/file_image.html',
+                                     file=sdk_file, vars=variations, tag_args=kwargs)
+
+    def render_video(self, sdk_file,
+                     pargs: typing.List[str],
+                     kwargs: typing.Dict[str, str]):
+        """Renders a video file."""
+        import flask
+        try:
+            # The very first variation is an mp4 file with max width of 1920px
+            default_variation = sdk_file.variations[0]
+        except IndexError:
+            log.error('Could not find variations for file %s' % sdk_file._id)
+            return flask.render_template('nodes/attachments/file_generic.html', file=sdk_file)
+
+        return flask.render_template('nodes/attachments/file_video.html',
+                                     file=sdk_file, var=default_variation, tag_args=kwargs)
+
+
+def _get_parser() -> typing.Tuple[shortcodes.Parser, shortcodes.Parser]:
+    """Return the shortcodes parser, create it if necessary."""
+    global _parser, _commented_parser
+    if _parser is None:
+        start, end = '{}'
+        _parser = shortcodes.Parser(start, end)
+        _commented_parser = shortcodes.Parser(f'<!-- {start}', f'{end} -->')
+    return _parser, _commented_parser
+
+
+def render_commented(text: str, context: typing.Any = None) -> str:
+    """Parse and render HTML-commented shortcodes.
+
+    Expects shortcodes like "<!-- {shortcode abc='def'} -->", as output by
+    escape_html().
+    """
+    _, parser = _get_parser()
+
+    # TODO(Sybren): catch exceptions and handle those gracefully in the response.
+    try:
+        return parser.parse(text, context)
+    except shortcodes.InvalidTagError as ex:
+        return html_module.escape('{%s}' % ex)
+    except shortcodes.RenderingError as ex:
+        log.info('Error rendering tag', exc_info=True)
+        return html_module.escape('{unable to render tag: %s}' % str(ex.__cause__ or ex))
+
+
+def render(text: str, context: typing.Any = None) -> str:
+    """Parse and render shortcodes."""
+    parser, _ = _get_parser()
+
+    # TODO(Sybren): catch exceptions and handle those gracefully in the response.
+    return parser.parse(text, context)
+
+
+def comment_shortcodes(html: str) -> str:
+    """Escape shortcodes in HTML comments.
+
+    This is required to pass the shortcodes as-is through Markdown. Render the
+    shortcodes afterwards with render_commented().
+
+    >>> comment_shortcodes("text\\n{shortcode abc='def'}\\n")
+    "text\\n<!-- {shortcode abc='def'} -->\\n"
+    """
+    parser, _ = _get_parser()
+    return parser.regex.sub(r'<!-- \g<0> -->', html)

pillar/tests/__init__.py

@@ -1,22 +1,20 @@
 # -*- encoding: utf-8 -*-
 
-from __future__ import print_function
-
-from __future__ import absolute_import
-
 import base64
 import copy
+import datetime
 import json
 import logging
-
-import datetime
 import os
+import pathlib
 import sys
+import typing
+import unittest.mock
 
 try:
     from urllib.parse import urlencode
 except ImportError:
-    from urllib import urlencode
+    from urllib.parse import urlencode
 
 from bson import ObjectId, tz_util
 
@@ -28,31 +26,23 @@ eve_test_settings.override_eve()
 from eve.tests import TestMinimal
 import pymongo.collection
 from flask.testing import FlaskClient
+import flask.ctx
 import responses
 
 import pillar
 from . import common_test_data as ctd
 
-# from six:
-PY3 = sys.version_info[0] == 3
-if PY3:
-    string_type = str
-    text_type = str
-else:
-    string_type = basestring
-    text_type = unicode
-
 MY_PATH = os.path.dirname(os.path.abspath(__file__))
 
 TEST_EMAIL_USER = 'koro'
 TEST_EMAIL_ADDRESS = '%s@testing.blender.org' % TEST_EMAIL_USER
-TEST_FULL_NAME = u'врач Сергей'
+TEST_FULL_NAME = 'врач Сергей'
 TEST_SUBCLIENT_TOKEN = 'my-subclient-token-for-pillar'
 BLENDER_ID_USER_RESPONSE = {'status': 'success',
                             'user': {'email': TEST_EMAIL_ADDRESS,
                                      'full_name': TEST_FULL_NAME,
                                      'id': ctd.BLENDER_ID_TEST_USERID},
-                            'token_expires': 'Mon, 1 Jan 2018 01:02:03 GMT'}
+                            'token_expires': 'Mon, 1 Jan 2218 01:02:03 GMT'}
 
 
 class PillarTestServer(pillar.PillarServer):
@@ -71,10 +61,65 @@ class PillarTestServer(pillar.PillarServer):
         logging.getLogger('werkzeug').setLevel(logging.DEBUG)
         logging.getLogger('eve').setLevel(logging.DEBUG)
 
+    def _config_celery(self):
+        """Disables Celery by entirely mocking it.
+
+        Without this, actual Celery tasks will be created while the tests are running.
+        """
+
+        from celery import Celery, Task
+
+        self.celery = unittest.mock.MagicMock(Celery)
+
+        def fake_task(*task_args, bind=False, **task_kwargs):
+            def decorator(f):
+                def delay(*args, **kwargs):
+                    if bind:
+                        return f(decorator.sender, *args, **kwargs)
+                    else:
+                        return f(*args, **kwargs)
+
+                f.delay = delay
+                f.si = unittest.mock.MagicMock()
+                f.s = unittest.mock.MagicMock()
+                return f
+
+            if bind:
+                decorator.sender = unittest.mock.MagicMock(Task)
+
+            return decorator
+
+        self.celery.task = fake_task
+
 
 class AbstractPillarTest(TestMinimal):
     pillar_server_class = PillarTestServer
 
+    @classmethod
+    def setUpClass(cls):
+        import tempfile
+
+        # Store the global temporary directory location, as Pillar itself will
+        # change this into the config['STORAGE_DIR'] directory. If we don't
+        # restore that, mkdtemp() will keep trying to create inside its previously
+        # created temporary storage directory.
+        cls._orig_tempdir = tempfile.gettempdir()
+
+        # Point the storage directory to something temporary.
+        try:
+            cls._pillar_storage_dir = tempfile.mkdtemp(prefix='test-pillar-storage-')
+        except FileNotFoundError as ex:
+            raise FileNotFoundError(f'Error creating temp dir: {ex}')
+        os.environ['PILLAR_STORAGE_DIR'] = cls._pillar_storage_dir
+
+    @classmethod
+    def tearDownClass(cls):
+        import tempfile
+        import shutil
+
+        tempfile.tempdir = cls._orig_tempdir
+        shutil.rmtree(cls._pillar_storage_dir)
+
     def setUp(self, **kwargs):
         eve_settings_file = os.path.join(MY_PATH, 'eve_test_settings.py')
         kwargs['settings_file'] = eve_settings_file
@@ -84,20 +129,43 @@ class AbstractPillarTest(TestMinimal):
         from eve.utils import config
         config.DEBUG = True
 
-        self.app = self.pillar_server_class(os.path.dirname(os.path.dirname(__file__)))
+        self.app = self.pillar_server_class(pathlib.Path(__file__).parents[2])
+        self.assertEqual(self.app.config['STORAGE_DIR'], self._pillar_storage_dir)
+
+        # Run self.enter_app_context() to create this context.
+        self._app_ctx: flask.ctx.AppContext = None
+
         self.app.process_extensions()
         assert self.app.config['MONGO_DBNAME'] == 'pillar_test'
 
-        self.client = self.app.test_client()
+        self.app.testing = True
+        self.client = self.app.test_client(use_cookies=False)
         assert isinstance(self.client, FlaskClient)
 
     def tearDown(self):
         super(AbstractPillarTest, self).tearDown()
 
+        if self._app_ctx is not None:
+            self._app_ctx.__exit__(*sys.exc_info())
+
         # Not only delete self.app (like the superclass does),
         # but also un-import the application.
         self.unload_modules('pillar')
 
+    def enter_app_context(self):
+        """Globally starts an app context.
+
+        The app context is automatically exited upon testcase teardown.
+        """
+
+        from flask import g
+
+        self._app_ctx: flask.ctx.AppContext = self.app.app_context()
+        self._app_ctx.__enter__()
+
+        if hasattr(g, 'current_user'):
+            g.current_user = None
+
     def unload_modules(self, module_name):
         """Uploads the named module, and all submodules."""
 
@@ -108,7 +176,10 @@ class AbstractPillarTest(TestMinimal):
         for modname in remove:
             del sys.modules[modname]
 
-    def ensure_file_exists(self, file_overrides=None):
+    def ensure_file_exists(self, file_overrides=None, *, example_file=None) -> (ObjectId, dict):
+        if example_file is None:
+            example_file = ctd.EXAMPLE_FILE
+
         if file_overrides and file_overrides.get('project'):
             self.ensure_project_exists({'_id': file_overrides['project']})
         else:
@@ -118,7 +189,7 @@ class AbstractPillarTest(TestMinimal):
             files_collection = self.app.data.driver.db['files']
             assert isinstance(files_collection, pymongo.collection.Collection)
 
-            file = copy.deepcopy(ctd.EXAMPLE_FILE)
+            file = copy.deepcopy(example_file)
             if file_overrides is not None:
                 file.update(file_overrides)
             if '_id' in file and file['_id'] is None:
@@ -132,7 +203,7 @@ class AbstractPillarTest(TestMinimal):
             from_db = files_collection.find_one(file_id)
             return file_id, from_db
 
-    def ensure_project_exists(self, project_overrides=None):
+    def ensure_project_exists(self, project_overrides=None) -> typing.Tuple[ObjectId, dict]:
         self.ensure_group_exists(ctd.EXAMPLE_ADMIN_GROUP_ID, 'project admin')
         self.ensure_group_exists(ctd.EXAMPLE_PROJECT_READONLY_GROUP_ID, 'r/o group')
         self.ensure_group_exists(ctd.EXAMPLE_PROJECT_READONLY_GROUP2_ID, 'r/o group 2')
@@ -185,39 +256,80 @@ class AbstractPillarTest(TestMinimal):
 
             found = groups_coll.find_one(group_id)
             if found:
-                return
+                return group_id
 
             result = groups_coll.insert_one({'_id': group_id, 'name': name})
             assert result.inserted_id
+            return result.inserted_id
 
     def create_user(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',),
-                    groups=None):
+                    groups=None, *, token: str = None, email: str = TEST_EMAIL_ADDRESS) -> ObjectId:
         from pillar.api.utils.authentication import make_unique_username
+        import uuid
 
         with self.app.test_request_context():
             users = self.app.data.driver.db['users']
             assert isinstance(users, pymongo.collection.Collection)
 
-            result = users.insert_one({
-                '_id': ObjectId(user_id),
+            user = {'_id': ObjectId(user_id),
                     '_updated': datetime.datetime(2016, 4, 15, 13, 15, 11, tzinfo=tz_util.utc),
                     '_created': datetime.datetime(2016, 4, 15, 13, 15, 11, tzinfo=tz_util.utc),
+                    '_etag': 'unittest-%s' % uuid.uuid4().hex,
                     'username': make_unique_username('tester'),
                     'groups': groups or [],
-                'roles': list(roles),
                     'settings': {'email_communications': 1},
                     'auth': [{'token': '',
-                          'user_id': unicode(ctd.BLENDER_ID_TEST_USERID),
+                              'user_id': str(ctd.BLENDER_ID_TEST_USERID),
                               'provider': 'blender-id'}],
-                'full_name': u'คนรักของผัดไทย',
-                'email': TEST_EMAIL_ADDRESS
-            })
+                    'full_name': 'คนรักของผัดไทย',
+                    'email': email}
+            if roles:
+                user['roles'] = list(roles)
 
-            return result.inserted_id
+            result = users.insert_one(user)
+            user_id = result.inserted_id
+
+        if token:
+            self.create_valid_auth_token(user_id, token)
+
+        return user_id
+
+    def create_user_object(self, user_id=ObjectId(), roles=frozenset(), group_ids=None):
+        """Creates a pillar.auth.UserClass object.
+
+        :rtype: pillar.auth.UserClass
+        """
+
+        from pillar.auth import UserClass
+
+        db_user = copy.deepcopy(ctd.EXAMPLE_USER)
+        db_user['_id'] = user_id
+        db_user['roles'] = list(roles) if roles is not None else None
+
+        if group_ids is not None:
+            db_user['groups'] = list(group_ids)
+
+        return UserClass.construct('', db_user)
+
+    def login_api_as(self, user_id=ObjectId(), roles=frozenset(), group_ids=None):
+        """Creates a pillar.auth.UserClass object and sets it as g.current_user
+
+        Requires an active request context!
+
+        :rtype: pillar.auth.UserClass
+        """
+
+        from flask import g
+
+        user = self.create_user_object(user_id, roles, group_ids)
+        g.current_user = user
+
+        return user
 
     def create_valid_auth_token(self, user_id, token='token'):
-        now = datetime.datetime.now(tz_util.utc)
-        future = now + datetime.timedelta(days=1)
+        from pillar.api.utils import utcnow
+
+        future = utcnow() + datetime.timedelta(days=1)
 
         with self.app.test_request_context():
             from pillar.api.utils import authentication as auth
@@ -226,13 +338,14 @@ class AbstractPillarTest(TestMinimal):
 
         return token_data
 
-    def create_project_with_admin(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',)):
+    def create_project_with_admin(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',),
+                                  project_overrides=None):
         """Creates a project and a user that's member of the project's admin group.
 
         :returns: (project_id, user_id)
         :rtype: tuple
         """
-        project_id, proj = self.ensure_project_exists()
+        project_id, proj = self.ensure_project_exists(project_overrides=project_overrides)
         user_id = self.create_project_admin(proj, user_id, roles)
 
         return project_id, user_id
@@ -305,12 +418,13 @@ class AbstractPillarTest(TestMinimal):
                       json=BLENDER_ID_USER_RESPONSE,
                       status=200)
 
-    def make_header(self, username, subclient_id=''):
+    def make_header(self, username: str, subclient_id: str = '') -> bytes:
         """Returns a Basic HTTP Authentication header value."""
 
-        return 'basic ' + base64.b64encode('%s:%s' % (username, subclient_id))
+        content = '%s:%s' % (username, subclient_id)
+        return b'basic ' + base64.b64encode(content.encode())
 
-    def create_standard_groups(self, additional_groups=()):
+    def create_standard_groups(self, additional_groups=()) -> dict:
         """Creates standard admin/demo/subscriber groups, plus any additional.
 
         :returns: mapping from group name to group ID
@@ -334,6 +448,11 @@ class AbstractPillarTest(TestMinimal):
             proj_coll = self.app.db()['projects']
             return proj_coll.find_one(project_id)
 
+    def fetch_user_from_db(self, user_id=ctd.EXAMPLE_USER['_id']):
+        with self.app.app_context():
+            users_coll = self.app.db('users')
+            return users_coll.find_one(user_id)
+
     @staticmethod
     def join_url_params(params):
         """Constructs a query string from a dictionary and appends it to a url.
@@ -351,27 +470,29 @@ class AbstractPillarTest(TestMinimal):
         if not isinstance(params, dict):
             return params
 
-        def convert_to_string(param):
+        def convert_to_bytes(param):
             if isinstance(param, dict):
                 return json.dumps(param, sort_keys=True)
-            if isinstance(param, text_type):
+            if isinstance(param, str):
                 return param.encode('utf-8')
             return param
 
         # Pass as (key, value) pairs, so that the sorted order is maintained.
         jsonified_params = [
-            (key, convert_to_string(params[key]))
+            (key, convert_to_bytes(params[key]))
             for key in sorted(params.keys())]
         return urlencode(jsonified_params)
 
     def client_request(self, method, path, qs=None, expected_status=200, auth_token=None, json=None,
-                       data=None, headers=None, files=None, content_type=None):
+                       data=None, headers=None, files=None, content_type=None, etag=None,
+                       environ_overrides=None):
         """Performs a HTTP request to the server."""
 
         from pillar.api.utils import dumps
         import json as mod_json
 
         headers = headers or {}
+        environ_overrides = environ_overrides or {}
         if auth_token is not None:
             headers['Authorization'] = self.make_header(auth_token)
 
@@ -379,6 +500,14 @@ class AbstractPillarTest(TestMinimal):
             data = dumps(json)
             headers['Content-Type'] = 'application/json'
 
+        if etag is not None:
+            if method in {'PUT', 'PATCH', 'DELETE'}:
+                headers['If-Match'] = etag
+            elif method == 'GET':
+                headers['If-None-Match'] = etag
+            else:
+                raise ValueError('Not sure what to do with etag and method %s' % method)
+
         if files:
             data = data or {}
             content_type = 'multipart/form-data'
@@ -386,18 +515,20 @@ class AbstractPillarTest(TestMinimal):
 
         resp = self.client.open(path=path, method=method, data=data, headers=headers,
                                 content_type=content_type,
-                                query_string=self.join_url_params(qs))
+                                query_string=self.join_url_params(qs),
+                                environ_overrides=environ_overrides)
         self.assertEqual(expected_status, resp.status_code,
                          'Expected status %i but got %i. Response: %s' % (
                              expected_status, resp.status_code, resp.data
                          ))
 
-        def json():
+        def get_json():
             if resp.mimetype != 'application/json':
                 raise TypeError('Unable to load JSON from mimetype %r' % resp.mimetype)
             return mod_json.loads(resp.data)
 
-        resp.json = json
+        resp.json = get_json
+        resp.get_json = get_json
 
         return resp
 
@@ -416,6 +547,25 @@ class AbstractPillarTest(TestMinimal):
     def patch(self, *args, **kwargs):
         return self.client_request('PATCH', *args, **kwargs)
 
+    def assertAllowsAccess(self,
+                           token: typing.Union[str, dict],
+                           expected_user_id: typing.Union[str, ObjectId] = None):
+        """Asserts that this authentication token allows access to /api/users/me."""
+
+        if isinstance(token, dict) and 'token' in token:
+            token = token['token']
+
+        if not isinstance(token, str):
+            raise TypeError(f'token should be a string, but is {token!r}')
+        if expected_user_id and not isinstance(expected_user_id, (str, ObjectId)):
+            raise TypeError('expected_user_id should be a string or ObjectId, '
+                            f'but is {expected_user_id!r}')
+
+        resp = self.get('/api/users/me', expected_status=200, auth_token=token).json()
+
+        if expected_user_id:
+            self.assertEqual(resp['_id'], str(expected_user_id))
+
 
 def mongo_to_sdk(data):
     """Transforms a MongoDB dict to a dict suitable to give to the PillarSDK.

pillar/tests/common_test_data.py

@@ -11,55 +11,55 @@ EXAMPLE_PROJECT_READONLY_GROUP2_ID = ObjectId('564733b56dcaf85da2faee8a')
 EXAMPLE_PROJECT_ID = ObjectId('5672beecc0261b2005ed1a33')
 EXAMPLE_PROJECT_OWNER_ID = ObjectId('552b066b41acdf5dec4436f2')
 
-EXAMPLE_FILE = {u'_id': ObjectId('5672e2c1c379cf0007b31995'),
-                u'_updated': datetime.datetime(2016, 3, 25, 10, 28, 24, tzinfo=tz_util.utc),
-                u'height': 2048,
-                u'name': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4.png', u'format': 'png',
-                u'variations': [
-                    {u'format': 'jpg', u'height': 160, u'width': 160, u'length': 8558,
-                     u'link': 'http://localhost:8002/file-variant-h', u'content_type': 'image/jpeg',
-                     u'md5': '--', u'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4-b.jpg',
-                     u'size': 'b'},
-                    {u'format': 'jpg', u'height': 2048, u'width': 2048, u'length': 819569,
-                     u'link': 'http://localhost:8002/file-variant-h', u'content_type': 'image/jpeg',
-                     u'md5': '--', u'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4-h.jpg',
-                     u'size': 'h'},
-                    {u'format': 'jpg', u'height': 64, u'width': 64, u'length': 8195,
-                     u'link': 'http://localhost:8002/file-variant-t', u'content_type': 'image/jpeg',
-                     u'md5': '--', u'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4-t.jpg',
-                     u'size': 't'},
+EXAMPLE_FILE = {'_id': ObjectId('5672e2c1c379cf0007b31995'),
+                '_updated': datetime.datetime(2016, 3, 25, 10, 28, 24, tzinfo=tz_util.utc),
+                'height': 2048,
+                'name': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4.png', 'format': 'png',
+                'variations': [
+                    {'format': 'jpg', 'height': 160, 'width': 160, 'length': 8558,
+                     'link': 'http://localhost:8002/file-variant-h', 'content_type': 'image/jpeg',
+                     'md5': '--', 'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4-b.jpg',
+                     'size': 'b'},
+                    {'format': 'jpg', 'height': 2048, 'width': 2048, 'length': 819569,
+                     'link': 'http://localhost:8002/file-variant-h', 'content_type': 'image/jpeg',
+                     'md5': '--', 'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4-h.jpg',
+                     'size': 'h'},
+                    {'format': 'jpg', 'height': 64, 'width': 64, 'length': 8195,
+                     'link': 'http://localhost:8002/file-variant-t', 'content_type': 'image/jpeg',
+                     'md5': '--', 'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4-t.jpg',
+                     'size': 't'},
                 ],
-                u'filename': 'brick_dutch_soft_bump.png',
-                u'project': EXAMPLE_PROJECT_ID,
-                u'width': 2048, u'length': 6227670,
-                u'user': ObjectId('56264fc4fa3a250344bd10c5'),
-                u'content_type': 'image/png',
-                u'_etag': '044ce3aede2e123e261c0d8bd77212f264d4f7b0',
-                u'_created': datetime.datetime(2015, 12, 17, 16, 28, 49, tzinfo=tz_util.utc),
-                u'md5': '',
-                u'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4.png',
-                u'backend': 'pillar',
-                u'link': 'http://localhost:8002/file',
-                u'link_expires': datetime.datetime(2016, 3, 22, 9, 28, 22, tzinfo=tz_util.utc)}
+                'filename': 'brick_dutch_soft_bump.png',
+                'project': EXAMPLE_PROJECT_ID,
+                'width': 2048, 'length': 6227670,
+                'user': ObjectId('56264fc4fa3a250344bd10c5'),
+                'content_type': 'image/png',
+                '_etag': '044ce3aede2e123e261c0d8bd77212f264d4f7b0',
+                '_created': datetime.datetime(2015, 12, 17, 16, 28, 49, tzinfo=tz_util.utc),
+                'md5': '',
+                'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4.png',
+                'backend': 'local',
+                'link': 'http://localhost:8002/file',
+                'link_expires': datetime.datetime(2016, 3, 22, 9, 28, 22, tzinfo=tz_util.utc)}
 
 EXAMPLE_PROJECT = {
-    u'_created': datetime.datetime(2015, 12, 17, 13, 22, 56, tzinfo=tz_util.utc),
-    u'_etag': u'cc4643e98d3606f87bbfaaa200bfbae941b642f3',
-    u'_id': EXAMPLE_PROJECT_ID,
-    u'_updated': datetime.datetime(2016, 1, 7, 18, 59, 4, tzinfo=tz_util.utc),
-    u'category': u'assets',
-    u'description': u'Welcome to this curated collection of Blender Institute textures and image '
-                    u'resources. This collection is an on-going project, as with each project we '
-                    u'create a number of textures based on our own resources (photographs, scans, '
-                    u'etc.) or made completely from scratch. At the moment you can find all the '
-                    u'textures from the past Open Projects that were deemed re-usable. \r\n\r\n'
-                    u'People who have contributed to these textures:\r\n\r\nAndrea Weikert, Andy '
-                    u'Goralczyk, Basse Salmela, Ben Dansie, Campbell Barton, Enrico Valenza, Ian '
-                    u'Hubert, Kjartan Tysdal, Manu J\xe4rvinen, Massimiliana Pulieso, Matt Ebb, '
-                    u'Pablo Vazquez, Rob Tuytel, Roland Hess, Sarah Feldlaufer, S\xf6nke M\xe4ter',
-    u'is_private': False,
-    u'name': u'Unittest project',
-    u'node_types': [
+    '_created': datetime.datetime(2015, 12, 17, 13, 22, 56, tzinfo=tz_util.utc),
+    '_etag': 'cc4643e98d3606f87bbfaaa200bfbae941b642f3',
+    '_id': EXAMPLE_PROJECT_ID,
+    '_updated': datetime.datetime(2016, 1, 7, 18, 59, 4, tzinfo=tz_util.utc),
+    'category': 'assets',
+    'description': 'Welcome to this curated collection of Blender Institute textures and image '
+                    'resources. This collection is an on-going project, as with each project we '
+                    'create a number of textures based on our own resources (photographs, scans, '
+                    'etc.) or made completely from scratch. At the moment you can find all the '
+                    'textures from the past Open Projects that were deemed re-usable. \r\n\r\n'
+                    'People who have contributed to these textures:\r\n\r\nAndrea Weikert, Andy '
+                    'Goralczyk, Basse Salmela, Ben Dansie, Campbell Barton, Enrico Valenza, Ian '
+                    'Hubert, Kjartan Tysdal, Manu J\xe4rvinen, Massimiliana Pulieso, Matt Ebb, '
+                    'Pablo Vazquez, Rob Tuytel, Roland Hess, Sarah Feldlaufer, S\xf6nke M\xe4ter',
+    'is_private': False,
+    'name': 'Unittest project',
+    'node_types': [
         PILLAR_NAMED_NODE_TYPES['group_texture'],
         PILLAR_NAMED_NODE_TYPES['group'],
         PILLAR_NAMED_NODE_TYPES['asset'],
@@ -69,36 +69,36 @@ EXAMPLE_PROJECT = {
         PILLAR_NAMED_NODE_TYPES['post'],
         PILLAR_NAMED_NODE_TYPES['texture'],
     ],
-    u'nodes_blog': [],
-    u'nodes_featured': [],
-    u'nodes_latest': [],
-    u'permissions': {u'groups': [{u'group': EXAMPLE_ADMIN_GROUP_ID,
-                                  u'methods': [u'GET', u'POST', u'PUT', u'DELETE']}],
-                     u'users': [],
-                     u'world': [u'GET']},
-    u'picture_header': ObjectId('5673f260c379cf0007b31bc4'),
-    u'picture_square': ObjectId('5673f256c379cf0007b31bc3'),
-    u'status': u'published',
-    u'summary': u'Texture collection from all Blender Institute open projects.',
-    u'url': u'textures',
-    u'user': EXAMPLE_PROJECT_OWNER_ID}
+    'nodes_blog': [],
+    'nodes_featured': [],
+    'nodes_latest': [],
+    'permissions': {'groups': [{'group': EXAMPLE_ADMIN_GROUP_ID,
+                                  'methods': ['GET', 'POST', 'PUT', 'DELETE']}],
+                     'users': [],
+                     'world': ['GET']},
+    'picture_header': ObjectId('5673f260c379cf0007b31bc4'),
+    'picture_square': ObjectId('5673f256c379cf0007b31bc3'),
+    'status': 'published',
+    'summary': 'Texture collection from all Blender Institute open projects.',
+    'url': 'default-project',
+    'user': EXAMPLE_PROJECT_OWNER_ID}
 
 EXAMPLE_NODE = {
-    u'_id': ObjectId('572761099837730efe8e120d'),
-    u'picture': ObjectId('572761f39837730efe8e1210'),
-    u'description': u'',
-    u'node_type': u'asset',
-    u'user': ObjectId('57164ca1983773118cbaf779'),
-    u'properties': {
-        u'status': u'published',
-        u'content_type': u'image',
-        u'file': ObjectId('572761129837730efe8e120e')
+    '_id': ObjectId('572761099837730efe8e120d'),
+    'picture': ObjectId('572761f39837730efe8e1210'),
+    'description': '',
+    'node_type': 'asset',
+    'user': ObjectId('57164ca1983773118cbaf779'),
+    'properties': {
+        'status': 'published',
+        'content_type': 'image',
+        'file': ObjectId('572761129837730efe8e120e')
     },
-    u'_updated': datetime.datetime(2016, 5, 2, 14, 19, 58, 0, tzinfo=tz_util.utc),
-    u'name': u'Image test',
-    u'project': EXAMPLE_PROJECT_ID,
-    u'_created': datetime.datetime(2016, 5, 2, 14, 19, 37, 0, tzinfo=tz_util.utc),
-    u'_etag': u'6b8589b42c880e3626f43f3e82a5c5b946742687'
+    '_updated': datetime.datetime(2016, 5, 2, 14, 19, 58, 0, tzinfo=tz_util.utc),
+    'name': 'Image test',
+    'project': EXAMPLE_PROJECT_ID,
+    '_created': datetime.datetime(2016, 5, 2, 14, 19, 37, 0, tzinfo=tz_util.utc),
+    '_etag': '6b8589b42c880e3626f43f3e82a5c5b946742687'
 }
 
 BLENDER_ID_TEST_USERID = 1533

pillar/tests/config_testing.py

@@ -1,6 +1,11 @@
 """Flask configuration file for unit testing."""
 
-BLENDER_ID_ENDPOINT = 'http://127.0.0.1:8001'  # nonexistant server, no trailing slash!
+BLENDER_ID_ENDPOINT = 'http://id.local:8001'  # Non existant server
+
+SERVER_NAME = 'localhost'
+PILLAR_SERVER_ENDPOINT = 'http://localhost/api/'
+
+MAIN_PROJECT_ID = '5672beecc0261b2005ed1a33'
 
 DEBUG = False
 TESTING = True
@@ -8,4 +13,34 @@ TESTING = True
 CDN_STORAGE_USER = 'u41508580125621'
 
 FILESIZE_LIMIT_BYTES_NONSUBS = 20 * 2 ** 10
-ROLES_FOR_UNLIMITED_UPLOADS = {u'subscriber', u'demo', u'admin'}
+ROLES_FOR_UNLIMITED_UPLOADS = {'subscriber', 'demo', 'admin'}
+
+GCLOUD_APP_CREDENTIALS = 'invalid-file-because-gcloud-storage-should-be-mocked-in-tests'
+STORAGE_BACKEND = 'local'
+
+EXTERNAL_SUBSCRIPTIONS_MANAGEMENT_SERVER = "http://store.localhost/api"
+
+SECRET_KEY = '12345'
+
+OAUTH_CREDENTIALS = {
+    'blender-id': {
+        'id': 'blender-id-app-id',
+        'secret': 'blender-id–secret',
+    },
+    'facebook': {
+        'id': 'fb-app-id',
+        'secret': 'facebook-secret'
+    },
+    'google': {
+        'id': 'google-app-id',
+        'secret': 'google-secret'
+    }
+}
+
+ELASTIC_INDICES = {
+    'NODE': 'test_nodes',
+    'USER': 'test_users',
+}
+
+# MUST be 8 characters long, see pillar.flask_extra.HashedPathConverter
+STATIC_FILE_HASH = 'abcd1234'