archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Merge branch 'master' of git.blender.org:pillar into elastic

Browse Source
This commit is contained in:
Stephan preeker
2017-12-29 12:19:47 +01:00
parent 10732f9a10 d73146ff62
commit d726e15ed8
28 changed files with 529 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
pillar/__init__.py
View File

@@ -21,7 +21,6 @@ from flask_babel import Babel, gettext as _
from flask.templating import TemplateNotFound
import pymongo.collection
import pymongo.database
from raven.contrib.flask import Sentry
from werkzeug.local import LocalProxy
@@ -42,6 +41,7 @@ import pillar.web.jinja
from . import api
from . import web
from . import auth
from . import sentry_extra
import pillar.api.organizations
empty_settings = {
@@ -106,7 +106,7 @@ class PillarServer(BlinkerCompatibleEve):
self._config_tempdirs()
self._config_git()
self.sentry: typing.Optional[Sentry] = None
self.sentry: typing.Optional[sentry_extra.PillarSentry] = None
self._config_sentry()
self._config_google_cloud_storage()
@@ -207,8 +207,9 @@ class PillarServer(BlinkerCompatibleEve):
self.sentry = None
return
self.sentry = Sentry(self, logging=True, level=logging.WARNING,
logging_exclusions=('werkzeug',))
self.sentry = sentry_extra.PillarSentry(
self, logging=True, level=logging.WARNING,
logging_exclusions=('werkzeug',))
# bugsnag.before_notify(bugsnag_extra.add_pillar_request_to_notification)
# got_request_exception.connect(self.__notify_bugsnag)
@@ -424,7 +425,7 @@ class PillarServer(BlinkerCompatibleEve):
custom_jinja_loader = jinja2.ChoiceLoader(paths_list)
self.jinja_loader = custom_jinja_loader
pillar.web.jinja.setup_jinja_env(self.jinja_env)
pillar.web.jinja.setup_jinja_env(self.jinja_env, self.config)
# Register context processors from extensions
for ext in self.pillar_extensions.values():
@@ -461,6 +462,7 @@ class PillarServer(BlinkerCompatibleEve):
'pillar.celery.tasks',
'pillar.celery.search_index_tasks',
'pillar.celery.file_link_tasks',
'pillar.celery.email_tasks',
]
# Allow Pillar extensions from defining their own Celery tasks.
@@ -754,7 +756,7 @@ class PillarServer(BlinkerCompatibleEve):
return 'basic ' + base64.b64encode('%s:%s' % (username, subclient_id))
def post_internal(self, resource, payl=None, skip_validation=False):
def post_internal(self, resource: str, payl=None, skip_validation=False):
"""Workaround for Eve issue https://github.com/nicolaiarocci/eve/issues/810"""
from eve.methods.post import post_internal
@@ -763,7 +765,7 @@ class PillarServer(BlinkerCompatibleEve):
with self.__fake_request_url_rule('POST', path):
return post_internal(resource, payl=payl, skip_validation=skip_validation)[:4]
def put_internal(self, resource, payload=None, concurrency_check=False,
def put_internal(self, resource: str, payload=None, concurrency_check=False,
skip_validation=False, **lookup):
"""Workaround for Eve issue https://github.com/nicolaiarocci/eve/issues/810"""
from eve.methods.put import put_internal
@@ -774,7 +776,7 @@ class PillarServer(BlinkerCompatibleEve):
return put_internal(resource, payload=payload, concurrency_check=concurrency_check,
skip_validation=skip_validation, **lookup)[:4]
def patch_internal(self, resource, payload=None, concurrency_check=False,
def patch_internal(self, resource: str, payload=None, concurrency_check=False,
skip_validation=False, **lookup):
"""Workaround for Eve issue https://github.com/nicolaiarocci/eve/issues/810"""
from eve.methods.patch import patch_internal

19
pillar/api/blender_cloud/subscription.py
View File

@@ -1,6 +1,7 @@
import logging
import typing
import blinker
from flask import Blueprint, Response
import requests
from requests.adapters import HTTPAdapter
@@ -21,6 +22,10 @@ ROLES_BID_TO_PILLAR = {
'cloud_has_subscription': 'has_subscription',
}
user_subscription_updated = blinker.NamedSignal(
'user_subscription_updated',
'The sender is a UserClass instance, kwargs includes "revoke_roles" and "grant_roles".')
@blueprint.route('/update-subscription')
@authorization.require_login()
@@ -31,7 +36,7 @@ def update_subscription() -> typing.Tuple[str, int]:
"""
my_log: logging.Logger = log.getChild('update_subscription')
current_user = auth.get_current_user()
real_current_user = auth.get_current_user() # multiple accesses, just get unproxied.
try:
bid_user = blender_id.fetch_blenderid_user()
@@ -41,10 +46,10 @@ def update_subscription() -> typing.Tuple[str, int]:
if not bid_user:
my_log.warning('Logged in user %s has no BlenderID account! '
'Unable to update subscription status.', current_user.user_id)
'Unable to update subscription status.', real_current_user.user_id)
return '', 204
do_update_subscription(current_user, bid_user)
do_update_subscription(real_current_user, bid_user)
return '', 204
@@ -157,6 +162,14 @@ def do_update_subscription(local_user: auth.UserClass, bid_user: dict):
user_id, email, ', '.join(sorted(revoke_roles)))
service.do_badger('revoke', roles=revoke_roles, user_id=user_id)
# Let the world know this user's subscription was updated.
final_roles = (plr_roles - revoke_roles).union(grant_roles)
local_user.roles = list(final_roles)
local_user.collect_capabilities()
user_subscription_updated.send(local_user,
grant_roles=grant_roles,
revoke_roles=revoke_roles)
# Re-index the user in the search database.
from pillar.api.users import hooks
hooks.push_updated_user_to_search({'_id': user_id}, {})

7
pillar/api/organizations/__init__.py
View File

@@ -376,6 +376,13 @@ class OrgManager:
return bool(org_count)
def user_is_unknown_member(self, member_email: str) -> bool:
"""Return True iff the email is an unknown member of some org."""
org_coll = current_app.db('organizations')
org_count = org_coll.count({'unknown_members': member_email})
return bool(org_count)
def setup_app(app):
from . import patch, hooks

2
pillar/api/projects/hooks.py
View File

@@ -2,7 +2,6 @@ import copy
import logging
from flask import request, abort, current_app
from gcloud import exceptions as gcs_exceptions
from pillar.api.node_types.asset import node_type_asset
from pillar.api.node_types.comment import node_type_comment
@@ -10,7 +9,6 @@ from pillar.api.node_types.group import node_type_group
from pillar.api.node_types.group_texture import node_type_group_texture
from pillar.api.node_types.texture import node_type_texture
from pillar.api.file_storage_backends import default_storage_backend
from pillar.api.file_storage_backends.gcs import GoogleCloudStorageBucket
from pillar.api.utils import authorization, authentication
from pillar.api.utils import remove_private_keys
from pillar.api.utils.authorization import user_has_role, check_permissions

5
pillar/api/users/hooks.py
View File

@@ -198,8 +198,9 @@ def after_inserting_users(user_docs):
user_email = user_doc.get('email')
if not user_id or not user_email:
log.warning('User created with _id=%r and email=%r, unable to check organizations',
user_id, user_email)
# Missing emails can happen when creating a service account, it's fine.
log.info('User created with _id=%r and email=%r, unable to check organizations',
user_id, user_email)
continue
om.make_member_known(user_id, user_email)

29
pillar/api/utils/authentication.py
View File

@@ -69,19 +69,19 @@ def find_user_in_db(user_info: dict, provider='blender-id'):
users = current_app.data.driver.db['users']
user_id = user_info['id']
query = {'$or': [
{'auth': {'$elemMatch': {
'user_id': str(user_info['id']),
'user_id': str(user_id),
'provider': provider}}},
{'email': user_info['email']},
]}
]}
log.debug('Querying: %s', query)
db_user = users.find_one(query)
if db_user:
log.debug('User with {provider} id {user_id} already in our database, '
'updating with info from {provider}.'.format(
provider=provider, user_id=user_info['id']))
log.debug('User with %s id %s already in our database, updating with info from %s',
provider, user_id, provider)
db_user['email'] = user_info['email']
# Find out if an auth entry for the current provider already exists
@@ -89,13 +89,13 @@ def find_user_in_db(user_info: dict, provider='blender-id'):
if not provider_entry:
db_user['auth'].append({
'provider': provider,
'user_id': str(user_info['id']),
'user_id': str(user_id),
'token': ''})
else:
log.debug('User %r not yet in our database, create a new one.', user_info['id'])
log.debug('User %r not yet in our database, create a new one.', user_id)
db_user = create_new_user_document(
email=user_info['email'],
user_id=user_info['id'],
user_id=user_id,
username=user_info['full_name'],
provider=provider)
db_user['username'] = make_unique_username(user_info['email'])
@@ -118,9 +118,13 @@ def validate_token():
from pillar.auth import AnonymousUser
auth_header = request.headers.get('Authorization') or ''
if request.authorization:
token = request.authorization.username
oauth_subclient = request.authorization.password
elif auth_header.startswith('Bearer '):
token = auth_header[7:].strip()
oauth_subclient = ''
else:
# Check the session, the user might be logged in through Flask-Login.
from pillar import auth
@@ -180,7 +184,6 @@ def validate_this_token(token, oauth_subclient=None):
def remove_token(token: str):
"""Removes the token from the database."""
tokens_coll = current_app.db('tokens')
token_hashed = hash_auth_token(token)
@@ -261,13 +264,13 @@ def create_new_user(email, username, user_id):
def create_new_user_document(email, user_id, username, provider='blender-id',
token=''):
token='', *, full_name=''):
"""Creates a new user document, without storing it in MongoDB. The token
parameter is a password in case provider is "local".
"""
user_data = {
'full_name': username,
'full_name': full_name or username,
'username': username,
'email': email,
'auth': [{
@@ -371,6 +374,10 @@ def upsert_user(db_user):
raise wz_exceptions.InternalServerError(
'Non-ObjectID string found in user.groups: %s' % db_user)
if not db_user['full_name']:
# Blender ID doesn't need a full name, but we do.
db_user['full_name'] = db_user['username']
r = {}
for retry in range(5):
if '_id' in db_user:

4
pillar/auth/__init__.py
View File

@@ -49,7 +49,7 @@ class UserClass(flask_login.UserMixin):
user = cls(token)
user.user_id = db_user['_id']
user.user_id = db_user.get('_id')
user.roles = db_user.get('roles') or []
user.group_ids = db_user.get('groups') or []
user.email = db_user.get('email') or ''
@@ -57,7 +57,7 @@ class UserClass(flask_login.UserMixin):
user.full_name = db_user.get('full_name') or ''
# Derived properties
user.objectid = str(db_user['_id'])
user.objectid = str(user.user_id or '')
user.gravatar = utils.gravatar(user.email)
user.groups = [str(g) for g in user.group_ids]
user.collect_capabilities()

50
pillar/celery/email_tasks.py Normal file
View File

@@ -0,0 +1,50 @@
"""Deferred email support.
Note that this module can only be imported when an application context is
active. Best to late-import this in the functions where it's needed.
"""
from email.message import EmailMessage
from email.headerregistry import Address
import logging
import smtplib
import celery
from pillar import current_app
log = logging.getLogger(__name__)
@current_app.celery.task(bind=True, ignore_result=True, acks_late=True)
def send_email(self: celery.Task, to_name: str, to_addr: str, subject: str, text: str, html: str):
"""Send an email to a single address."""
# WARNING: when changing the signature of this function, also change the
# self.retry() call below.
cfg = current_app.config
# Construct the message
msg = EmailMessage()
msg['Subject'] = subject
msg['From'] = Address(cfg['MAIL_DEFAULT_FROM_NAME'], addr_spec=cfg['MAIL_DEFAULT_FROM_ADDR'])
msg['To'] = (Address(to_name, addr_spec=to_addr),)
msg.set_content(text)
msg.add_alternative(html, subtype='html')
# Refuse to send mail when we're testing.
if cfg['TESTING']:
log.warning('not sending mail to %s <%s> because we are TESTING', to_name, to_addr)
return
log.info('sending email to %s <%s>', to_name, to_addr)
# Send the message via local SMTP server.
try:
with smtplib.SMTP(cfg['SMTP_HOST'], cfg['SMTP_PORT'], timeout=cfg['SMTP_TIMEOUT']) as smtp:
if cfg.get('SMTP_USERNAME') and cfg.get('SMTP_PASSWORD'):
smtp.login(cfg['SMTP_USERNAME'], cfg['SMTP_PASSWORD'])
smtp.send_message(msg)
except (IOError, OSError) as ex:
log.exception('error sending email to %s <%s>, will retry later: %s',
to_name, to_addr, ex)
self.retry((to_name, to_addr, subject, text, html), countdown=cfg['MAIL_RETRY'])
else:
log.info('mail to %s <%s> successfully sent', to_name, to_addr)

14
pillar/config.py
View File

@@ -28,7 +28,7 @@ SECRET_KEY = ''
AUTH_TOKEN_HMAC_KEY = b''
# Authentication settings
BLENDER_ID_ENDPOINT = 'http://blender_id:8000/'
BLENDER_ID_ENDPOINT = 'http://blender-id:8000/'
PILLAR_SERVER_ENDPOINT = 'http://pillar:5001/api/'
@@ -124,7 +124,7 @@ BLENDER_ID_USER_INFO_TOKEN = '-set-in-config-local-'
# 'blender-id': {
# 'id': 'CLOUD-OF-SNOWFLAKES-43',
# 'secret': 'thesecret',
# 'base_url': 'http://blender_id:8000/'
# 'base_url': 'http://blender-id:8000/'
# }
# }
# OAuth providers are defined in pillar.auth.oauth
@@ -238,3 +238,13 @@ DEFAULT_LOCALE = 'en_US'
# never show the site in English.
SUPPORT_ENGLISH = True
# Mail options, see pillar.celery.email_tasks.
SMTP_HOST = 'localhost'
SMTP_PORT = 2525
SMTP_USERNAME = ''
SMTP_PASSWORD = ''
SMTP_TIMEOUT = 30 # timeout in seconds, https://docs.python.org/3/library/smtplib.html#smtplib.SMTP
MAIL_RETRY = 180 # in seconds, delay until trying to send an email again.
MAIL_DEFAULT_FROM_NAME = 'Blender Cloud'
MAIL_DEFAULT_FROM_ADDR = 'cloudsupport@localhost'

42
pillar/sentry_extra.py Normal file
View File

@@ -0,0 +1,42 @@
from raven.contrib.flask import Sentry
from .auth import current_user
from . import current_app
class PillarSentry(Sentry):
"""Flask Sentry with Pillar support.
This is mostly for obtaining user information on API calls,
and for preventing the auth tokens to be logged as user ID.
"""
def get_user_info(self, request):
user_info = super().get_user_info(request)
# The auth token is stored as the user ID in the flask_login
# current_user object, so don't send that to Sentry.
user_info.pop('id', None)
if len(user_info) > 1:
# Sentry always includes the IP address, but when they find a
# logged-in user, they add more info. In that case we're done.
return user_info
# This is pretty much a copy-paste from Sentry, except that it uses
# pillar.auth.current_user instead.
try:
if not current_user.is_authenticated:
return user_info
except AttributeError:
# HACK: catch the attribute error thrown by flask-login is not attached
# > current_user = LocalProxy(lambda: _request_ctx_stack.top.user)
# E AttributeError: 'RequestContext' object has no attribute 'user'
return user_info
if 'SENTRY_USER_ATTRS' in current_app.config:
for attr in current_app.config['SENTRY_USER_ATTRS']:
if hasattr(current_user, attr):
user_info[attr] = getattr(current_user, attr)
return user_info

16
pillar/tests/__init__.py
View File

@@ -42,7 +42,7 @@ BLENDER_ID_USER_RESPONSE = {'status': 'success',
'user': {'email': TEST_EMAIL_ADDRESS,
'full_name': TEST_FULL_NAME,
'id': ctd.BLENDER_ID_TEST_USERID},
'token_expires': 'Mon, 1 Jan 2018 01:02:03 GMT'}
'token_expires': 'Mon, 1 Jan 2218 01:02:03 GMT'}
class PillarTestServer(pillar.PillarServer):
@@ -67,20 +67,26 @@ class PillarTestServer(pillar.PillarServer):
Without this, actual Celery tasks will be created while the tests are running.
"""
from celery import Celery
from celery import Celery, Task
self.celery = unittest.mock.MagicMock(Celery)
def fake_task(*task_args, **task_kwargs):
def fake_task(*task_args, bind=False, **task_kwargs):
def decorator(f):
def delay(*args, **kwargs):
return f(*args, **kwargs)
if bind:
return f(decorator.sender, *args, **kwargs)
else:
return f(*args, **kwargs)
f.delay = delay
f.si = unittest.mock.MagicMock()
f.s = unittest.mock.MagicMock()
return f
if bind:
decorator.sender = unittest.mock.MagicMock(Task)
return decorator
self.celery.task = fake_task
@@ -251,7 +257,7 @@ class AbstractPillarTest(TestMinimal):
return result.inserted_id
def create_user(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',),
groups=None, *, token: str = None, email: str = TEST_EMAIL_ADDRESS):
groups=None, *, token: str = None, email: str = TEST_EMAIL_ADDRESS) -> ObjectId:
from pillar.api.utils.authentication import make_unique_username
import uuid

2
pillar/tests/config_testing.py
View File

@@ -24,7 +24,7 @@ OAUTH_CREDENTIALS = {
'blender-id': {
'id': 'blender-id-app-id',
'secret': 'blender-id–secret',
'base_url': 'http://blender_id:8000/'
'base_url': 'http://blender-id:8000/'
},
'facebook': {
'id': 'fb-app-id',

6
pillar/web/jinja.py
View File

@@ -1,5 +1,6 @@
"""Our custom Jinja filters and other template stuff."""
import functools
import logging
import typing
@@ -146,7 +147,7 @@ def do_yesno(value, arg=None):
return no
def setup_jinja_env(jinja_env):
def setup_jinja_env(jinja_env, app_config: dict):
jinja_env.filters['pretty_date'] = format_pretty_date
jinja_env.filters['pretty_date_time'] = format_pretty_date_time
jinja_env.filters['undertitle'] = format_undertitle
@@ -157,5 +158,8 @@ def setup_jinja_env(jinja_env):
jinja_env.filters['yesno'] = do_yesno
jinja_env.filters['repr'] = repr
jinja_env.globals['url_for_node'] = do_url_for_node
jinja_env.globals['abs_url'] = functools.partial(flask.url_for,
_external=True,
_scheme=app_config['SCHEME'])
jinja_env.globals['session'] = flask.session
jinja_env.globals['current_user'] = flask_login.current_user

61
pillar/web/projects/routes.py
View File

@@ -1,3 +1,4 @@
import datetime
import json
import logging
import itertools
@@ -7,6 +8,7 @@ from pillarsdk import Node
from pillarsdk import Project
from pillarsdk.exceptions import ResourceNotFound
from pillarsdk.exceptions import ForbiddenAccess
import flask
from flask import Blueprint
from flask import render_template
from flask import request
@@ -78,6 +80,19 @@ def index():
'sort': '-_created'
}, api=api)
show_deleted_projects = request.args.get('deleted') is not None
if show_deleted_projects:
timeframe = utils.datetime_now() - datetime.timedelta(days=31)
projects_deleted = Project.all({
'where': {'user': current_user.objectid,
'category': {'$ne': 'home'},
'_deleted': True,
'_updated': {'$gt': timeframe}},
'sort': '-_created'
}, api=api)
else:
projects_deleted = {'_items': []}
projects_shared = Project.all({
'where': {'user': {'$ne': current_user.objectid},
'permissions.groups.group': {'$in': current_user.groups},
@@ -87,17 +102,17 @@ def index():
}, api=api)
# Attach project images
for project in projects_user['_items']:
utils.attach_project_pictures(project, api)
for project in projects_shared['_items']:
utils.attach_project_pictures(project, api)
for project_list in (projects_user, projects_deleted, projects_shared):
for project in project_list['_items']:
utils.attach_project_pictures(project, api)
return render_template(
'projects/index_dashboard.html',
gravatar=utils.gravatar(current_user.email, size=128),
projects_user=projects_user['_items'],
projects_deleted=projects_deleted['_items'],
projects_shared=projects_shared['_items'],
show_deleted_projects=show_deleted_projects,
api=api)
@@ -274,7 +289,7 @@ def view(project_url):
header_video_file = None
header_video_node = None
if project.header_node and project.header_node.node_type == 'asset' and \
project.header_node.properties.content_type == 'video':
project.header_node.properties.content_type == 'video':
header_video_node = project.header_node
header_video_file = utils.get_file(project.header_node.properties.file)
header_video_node.picture = utils.get_file(header_video_node.picture)
@@ -847,3 +862,37 @@ def edit_extension(project: Project, extension_name):
return ext.project_settings(project,
ext_pages=find_extension_pages())
@blueprint.route('/undelete', methods=['POST'])
@login_required
def undelete():
"""Undelete a deleted project.
Can only be done by the owner of the project or an admin.
"""
# This function takes an API-style approach, even though it's a web
# endpoint. Undeleting via a REST approach would mean GETting the
# deleted project, which now causes a 404 exception to bubble to the
# client.
from pillar.api.utils import mongo, remove_private_keys
from pillar.api.utils.authorization import check_permissions
project_id = request.form.get('project_id')
if not project_id:
raise wz_exceptions.BadRequest('missing project ID')
# Check that the user has PUT permissions on the project itself.
project = mongo.find_one_or_404('projects', project_id)
check_permissions('projects', project, 'PUT')
pid = project['_id']
log.info('Undeleting project %s on behalf of %s', pid, current_user.email)
r, _, _, status = current_app.put_internal('projects', remove_private_keys(project), _id=pid)
if status != 200:
log.warning('Error %d un-deleting project %s: %s', status, pid, r)
return 'Error un-deleting project', 500
resp = flask.Response('', status=204)
resp.location = flask.url_for('projects.view', project_url=project['url'])
return resp

7
pillar/web/users/routes.py
View File

@@ -5,6 +5,7 @@ from flask import abort, Blueprint, redirect, render_template, request, session,
from flask_login import login_required
from werkzeug import exceptions as wz_exceptions
from pillar import current_app
import pillar.api.blender_cloud.subscription
import pillar.auth
from pillar.api.blender_cloud.subscription import update_subscription
@@ -16,6 +17,7 @@ from pillar.auth.oauth import OAuthSignIn, ProviderConfigurationMissing, Provide
from pillar.web import system_util
from pillarsdk import exceptions as sdk_exceptions
from pillarsdk.users import User
from . import forms
log = logging.getLogger(__name__)
@@ -83,8 +85,7 @@ def oauth_callback(provider):
def login():
if request.args.get('force'):
log.debug('Forcing logout of user before rendering login page.')
logout_user()
session.clear()
pillar.auth.logout_user()
session['next_after_login'] = request.args.get('next') or request.referrer
return render_template('login.html')
@@ -121,9 +122,11 @@ def switch():
# Without this URL, the user will remain on the Blender ID site. We want them to come
# back to the Cloud after switching users.
scheme = current_app.config.get('PREFERRED_URL_SCHEME', 'https')
next_url_after_bid_login = url_for('users.login',
next=next_url_after_cloud_login,
force='yes',
_scheme=scheme,
_external=True)
return redirect(blender_id.switch_user_url(next_url=next_url_after_bid_login))

21
pillar/web/utils/jstree.py
View File

@@ -115,15 +115,17 @@ def jstree_build_from_node(node):
# Get the parent node
parent = None
parent_projection = {'projection': {
'name': 1,
'parent': 1,
'project': 1,
'node_type': 1,
'properties.content_type': 1,
}}
if node.parent:
try:
parent = Node.find(node.parent, {
'projection': {
'name': 1,
'node_type': 1,
'parent': 1,
'properties.content_type': 1,
}}, api=api)
parent = Node.find(node.parent, parent_projection, api=api)
# Define the child node of the tree (usually an asset)
except ResourceNotFound:
# If not found, we might be on the top level, in which case we skip the
@@ -147,10 +149,7 @@ def jstree_build_from_node(node):
# If we have a parent
if parent.parent:
try:
parent = Node.find(parent.parent, {
'projection': {
'name': 1, 'parent': 1, 'project': 1, 'node_type': 1},
}, api=api)
parent = Node.find(parent.parent, parent_projection, api=api)
except ResourceNotFound:
parent = None
else: