archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into elastic

Browse Source
This commit is contained in:
Sybren A. Stüvel
2017-12-08 12:55:57 +01:00
parent b6a93452cd 821f11393c
commit b7773e69c7
22 changed files with 289 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
pillar/__init__.py
View File

@@ -21,6 +21,7 @@ from flask_babel import Babel, gettext as _
from flask.templating import TemplateNotFound
import pymongo.collection
import pymongo.database
from raven.contrib.flask import Sentry
from werkzeug.local import LocalProxy
@@ -59,7 +60,17 @@ class ConfigurationMissingError(SystemExit):
"""
class PillarServer(Eve):
class BlinkerCompatibleEve(Eve):
"""Workaround for https://github.com/pyeve/eve/issues/1087"""
def __getattr__(self, name):
if name in {"im_self", "im_func"}:
raise AttributeError("type object '%s' has no attribute '%s'" %
(self.__class__.__name__, name))
return super().__getattr__(name)
class PillarServer(BlinkerCompatibleEve):
def __init__(self, app_root, **kwargs):
from .extension import PillarExtension
from celery import Celery
@@ -75,7 +86,7 @@ class PillarServer(Eve):
# The default roles Pillar uses. Will probably all move to extensions at some point.
self._user_roles: typing.Set[str] = {
'demo', 'admin', 'subscriber', 'homeproject',
'protected',
'protected', 'org-subscriber', 'video-encoder',
'service', 'badger', 'svner', 'urler',
}
self._user_roles_indexable: typing.Set[str] = {'demo', 'admin', 'subscriber'}
@@ -94,7 +105,9 @@ class PillarServer(Eve):
self._config_auth_token_hmac_key()
self._config_tempdirs()
self._config_git()
self._config_bugsnag()
self.sentry: typing.Optional[Sentry] = None
self._config_sentry()
self._config_google_cloud_storage()
self.algolia_index_users = None
@@ -187,39 +200,19 @@ class PillarServer(Eve):
self.config['GIT_REVISION'] = 'unknown'
self.log.info('Git revision %r', self.config['GIT_REVISION'])
def _config_bugsnag(self):
bugsnag_api_key = self.config.get('BUGSNAG_API_KEY')
if self.config.get('TESTING') or not bugsnag_api_key:
self.log.info('Bugsnag NOT configured.')
def _config_sentry(self):
sentry_dsn = self.config.get('SENTRY_CONFIG', {}).get('dsn')
if self.config.get('TESTING') or sentry_dsn in {'', '-set-in-config-local-'}:
self.log.warning('Sentry NOT configured.')
self.sentry = None
return
import bugsnag
from bugsnag.handlers import BugsnagHandler
self.sentry = Sentry(self, logging=True, level=logging.WARNING,
logging_exclusions=('werkzeug',))
release_stage = self.config.get('BUGSNAG_RELEASE_STAGE', 'unconfigured')
if self.config.get('DEBUG'):
release_stage += '-debug'
bugsnag.configure(
api_key=bugsnag_api_key,
project_root="/data/git/pillar/pillar",
release_stage=release_stage
)
bs_handler = BugsnagHandler()
bs_handler.setLevel(logging.ERROR)
self.log.addHandler(bs_handler)
# This is what bugsnag.flask.handle_exceptions also tries to do,
# but it passes the app to the connect() call, which causes an
# error. Since we only have one app, we can do without.
from flask import got_request_exception
from . import bugsnag_extra
bugsnag.before_notify(bugsnag_extra.add_pillar_request_to_notification)
got_request_exception.connect(self.__notify_bugsnag)
self.log.info('Bugsnag setup complete')
# bugsnag.before_notify(bugsnag_extra.add_pillar_request_to_notification)
# got_request_exception.connect(self.__notify_bugsnag)
self.log.info('Sentry setup complete')
def __notify_bugsnag(self, sender, exception, **extra):
import bugsnag

226
pillar/api/blender_cloud/subscription.py
View File

@@ -1,94 +1,37 @@
import logging
import typing
from flask import current_app, Blueprint
from flask import Blueprint, Response
import requests
from requests.adapters import HTTPAdapter
from pillar.api.utils import authorization
from pillar import auth, current_app
from pillar.api import blender_id
from pillar.api.utils import authorization, jsonify
from pillar.auth import current_user
log = logging.getLogger(__name__)
blueprint = Blueprint('blender_cloud.subscription', __name__)
def fetch_subscription_info(email: str) -> typing.Optional[dict]:
"""Returns the user info dict from the external subscriptions management server.
:returns: the store user info, or None if the user can't be found or there
was an error communicating. A dict like this is returned:
{
"shop_id": 700,
"cloud_access": 1,
"paid_balance": 314.75,
"balance_currency": "EUR",
"start_date": "2014-08-25 17:05:46",
"expiration_date": "2016-08-24 13:38:45",
"subscription_status": "wc-active",
"expiration_date_approximate": true
}
"""
import requests
from requests.adapters import HTTPAdapter
import requests.exceptions
external_subscriptions_server = current_app.config['EXTERNAL_SUBSCRIPTIONS_MANAGEMENT_SERVER']
if log.isEnabledFor(logging.DEBUG):
import urllib.parse
log_email = urllib.parse.quote(email)
log.debug('Connecting to store at %s?blenderid=%s',
external_subscriptions_server, log_email)
# Retry a few times when contacting the store.
s = requests.Session()
s.mount(external_subscriptions_server, HTTPAdapter(max_retries=5))
try:
r = s.get(external_subscriptions_server,
params={'blenderid': email},
verify=current_app.config['TLS_CERT_FILE'],
timeout=current_app.config.get('EXTERNAL_SUBSCRIPTIONS_TIMEOUT_SECS', 10))
except requests.exceptions.ConnectionError as ex:
log.error('Error connecting to %s: %s', external_subscriptions_server, ex)
return None
except requests.exceptions.Timeout as ex:
log.error('Timeout communicating with %s: %s', external_subscriptions_server, ex)
return None
except requests.exceptions.RequestException as ex:
log.error('Some error communicating with %s: %s', external_subscriptions_server, ex)
return None
if r.status_code != 200:
log.warning("Error communicating with %s, code=%i, unable to check "
"subscription status of user %s",
external_subscriptions_server, r.status_code, email)
return None
store_user = r.json()
if log.isEnabledFor(logging.DEBUG):
import json
log.debug('Received JSON from store API: %s',
json.dumps(store_user, sort_keys=False, indent=4))
return store_user
# Mapping from roles on Blender ID to roles here in Pillar.
# Roles not mentioned here will not be synced from Blender ID.
ROLES_BID_TO_PILLAR = {
'cloud_subscriber': 'subscriber',
'cloud_demo': 'demo',
'cloud_has_subscription': 'has_subscription',
}
@blueprint.route('/update-subscription')
@authorization.require_login()
def update_subscription():
def update_subscription() -> typing.Tuple[str, int]:
"""Updates the subscription status of the current user.
Returns an empty HTTP response.
"""
import pprint
from pillar import auth
from pillar.api import blender_id, service
from pillar.api.utils import authentication
my_log: logging.Logger = log.getChild('update_subscription')
user_id = authentication.current_user_id()
current_user = auth.get_current_user()
try:
bid_user = blender_id.fetch_blenderid_user()
@@ -98,46 +41,125 @@ def update_subscription():
if not bid_user:
my_log.warning('Logged in user %s has no BlenderID account! '
'Unable to update subscription status.', user_id)
'Unable to update subscription status.', current_user.user_id)
return '', 204
# Use the Blender ID email address to check with the store. At least that reduces the
# number of email addresses that could be out of sync to two (rather than three when we
# use the email address from our local database).
do_update_subscription(current_user, bid_user)
return '', 204
@blueprint.route('/update-subscription-for/<user_id>', methods=['POST'])
@authorization.require_login(require_cap='admin')
def update_subscription_for(user_id: str):
"""Updates the user based on their info at Blender ID."""
from urllib.parse import urljoin
from pillar.api.utils import str2id
my_log = log.getChild('update_subscription_for')
bid_session = requests.Session()
bid_session.mount('https://', HTTPAdapter(max_retries=5))
bid_session.mount('http://', HTTPAdapter(max_retries=5))
users_coll = current_app.db('users')
db_user = users_coll.find_one({'_id': str2id(user_id)})
if not db_user:
my_log.warning('User %s not found in database', user_id)
return Response(f'User {user_id} not found in our database', status=404)
log.info('Updating user %s from Blender ID on behalf of %s',
db_user['email'], current_user.email)
bid_user_id = blender_id.get_user_blenderid(db_user)
if not bid_user_id:
my_log.info('User %s has no Blender ID', user_id)
return Response('User has no Blender ID', status=404)
# Get the user info from Blender ID, and handle errors.
api_url = current_app.config['BLENDER_ID_USER_INFO_API']
api_token = current_app.config['BLENDER_ID_USER_INFO_TOKEN']
url = urljoin(api_url, bid_user_id)
resp = bid_session.get(url, headers={'Authorization': f'Bearer {api_token}'})
if resp.status_code == 404:
my_log.info('User %s has a Blender ID %s but Blender ID itself does not find it',
user_id, bid_user_id)
return Response(f'User {bid_user_id} does not exist at Blender ID', status=404)
if resp.status_code != 200:
my_log.info('Error code %s getting user %s from Blender ID (resp = %s)',
resp.status_code, user_id, resp.text)
return Response(f'Error code {resp.status_code} from Blender ID', status=resp.status_code)
# Update the user in our database.
local_user = auth.UserClass.construct('', db_user)
bid_user = resp.json()
do_update_subscription(local_user, bid_user)
return '', 204
def do_update_subscription(local_user: auth.UserClass, bid_user: dict):
"""Updates the subscription status of the user given the Blender ID user info.
Uses the badger service to update the user's roles from Blender ID.
bid_user should be a dict like:
{'id': 1234,
'full_name': 'मूंगफली मक्खन प्रेमी',
'email': 'here@example.com',
'roles': {'cloud_demo': True}}
The 'roles' key can also be an interable of role names instead of a dict.
"""
from pillar.api import service
my_log: logging.Logger = log.getChild('do_update_subscription')
try:
email = bid_user['email']
except KeyError:
my_log.error('Blender ID response did not include an email address, '
'unable to update subscription status: %s',
pprint.pformat(bid_user, compact=True))
return 'Internal error', 500
store_user = fetch_subscription_info(email) or {}
email = '-missing email-'
# Transform the BID roles from a dict to a set.
bidr = bid_user.get('roles', set())
if isinstance(bidr, dict):
bid_roles = {role
for role, has_role in bid_user.get('roles', {}).items()
if has_role}
else:
bid_roles = set(bidr)
# Handle the role changes via the badger service functionality.
grant_subscriber = store_user.get('cloud_access', 0) == 1
grant_demo = bid_user.get('roles', {}).get('cloud_demo', False)
plr_roles = set(local_user.roles)
is_subscriber = authorization.user_has_role('subscriber')
is_demo = authorization.user_has_role('demo')
grant_roles = set()
revoke_roles = set()
for bid_role, plr_role in ROLES_BID_TO_PILLAR.items():
if bid_role in bid_roles and plr_role not in plr_roles:
grant_roles.add(plr_role)
continue
if bid_role not in bid_roles and plr_role in plr_roles:
revoke_roles.add(plr_role)
if grant_subscriber != is_subscriber:
action = 'grant' if grant_subscriber else 'revoke'
my_log.info('%sing subscriber role to user %s (Blender ID email %s)',
action, user_id, email)
service.do_badger(action, role='subscriber', user_id=user_id)
else:
my_log.debug('Not changing subscriber role, grant=%r and is=%s',
grant_subscriber, is_subscriber)
user_id = local_user.user_id
if grant_demo != is_demo:
action = 'grant' if grant_demo else 'revoke'
my_log.info('%sing demo role to user %s (Blender ID email %s)', action, user_id, email)
service.do_badger(action, role='demo', user_id=user_id)
else:
my_log.debug('Not changing demo role, grant=%r and is=%s',
grant_demo, is_demo)
if grant_roles:
if my_log.isEnabledFor(logging.INFO):
my_log.info('granting roles to user %s (Blender ID %s): %s',
user_id, email, ', '.join(sorted(grant_roles)))
service.do_badger('grant', roles=grant_roles, user_id=user_id)
return '', 204
if revoke_roles:
if my_log.isEnabledFor(logging.INFO):
my_log.info('revoking roles to user %s (Blender ID %s): %s',
user_id, email, ', '.join(sorted(revoke_roles)))
service.do_badger('revoke', roles=revoke_roles, user_id=user_id)
# Re-index the user in the search database.
from pillar.api.users import hooks
hooks.push_updated_user_to_algolia({'_id': user_id}, {})
def setup_app(app, url_prefix):

28
pillar/api/blender_id.py
View File

@@ -168,6 +168,24 @@ def _compute_token_expiry(token_expires_string):
return min(blid_expiry, our_expiry)
def get_user_blenderid(db_user: dict) -> str:
"""Returns the Blender ID user ID for this Pillar user.
Takes the string from 'auth.*.user_id' for the '*' where 'provider'
is 'blender-id'.
:returns the user ID, or the empty string when the user has none.
"""
bid_user_ids = [auth['user_id']
for auth in db_user['auth']
if auth['provider'] == 'blender-id']
try:
return bid_user_ids[0]
except IndexError:
return ''
def fetch_blenderid_user() -> dict:
"""Returns the user info of the currently logged in user from BlenderID.
@@ -181,7 +199,8 @@ def fetch_blenderid_user() -> dict:
"roles": {
"admin": true,
"bfct_trainer": false,
"cloud_single_member": true,
"cloud_has_subscription": true,
"cloud_subscriber": true,
"conference_speaker": true,
"network_member": true
}
@@ -218,12 +237,13 @@ def fetch_blenderid_user() -> dict:
log.warning('Error %i from BlenderID %s: %s', bid_resp.status_code, bid_url, bid_resp.text)
return {}
if not bid_resp.json():
payload = bid_resp.json()
if not payload:
log.warning('Empty data returned from BlenderID %s', bid_url)
return {}
log.debug('BlenderID returned %s', bid_resp.json())
return bid_resp.json()
log.debug('BlenderID returned %s', payload)
return payload
def setup_app(app, url_prefix):

8
pillar/api/file_storage/__init__.py
View File

@@ -26,7 +26,7 @@ from flask import url_for, helpers
from pillar.api import utils
from pillar.api.file_storage_backends.gcs import GoogleCloudStorageBucket, \
GoogleCloudStorageBlob
from pillar.api.utils import remove_private_keys, authentication
from pillar.api.utils import remove_private_keys
from pillar.api.utils.authorization import require_login, user_has_role, \
user_matches_roles
from pillar.api.utils.cdn import hash_file_path
@@ -291,8 +291,8 @@ def process_file(bucket: Bucket,
# TODO: overrule the content type based on file extention & magic numbers.
mime_category, src_file['format'] = src_file['content_type'].split('/', 1)
# Prevent video handling for non-admins.
if not user_has_role('admin') and mime_category == 'video':
# Only allow video encoding when the user has the correct capability.
if not current_user.has_cap('encode-video') and mime_category == 'video':
if src_file['format'].startswith('x-'):
xified = src_file['format']
else:
@@ -300,7 +300,7 @@ def process_file(bucket: Bucket,
src_file['content_type'] = 'application/%s' % xified
mime_category = 'application'
log.info('Not processing video file %s for non-admin user', file_id)
log.info('Not processing video file %s for non-video-encoding user', file_id)
# Run the required processor, based on the MIME category.
processors: typing.Mapping[str, typing.Callable] = {

2
pillar/auth/__init__.py
View File

@@ -18,7 +18,7 @@ log = logging.getLogger(__name__)
CAPABILITIES = collections.defaultdict(**{
'subscriber': {'subscriber', 'home-project'},
'demo': {'subscriber', 'home-project'},
'admin': {'subscriber', 'home-project', 'video-encoding', 'admin',
'admin': {'video-encoding', 'admin',
'view-pending-nodes', 'edit-project-node-types'},
}, default_factory=frozenset)

21
pillar/config.py
View File

@@ -65,8 +65,13 @@ GOOGLE_SITE_VERIFICATION = ''
ADMIN_USER_GROUP = '5596e975ea893b269af85c0e'
SUBSCRIBER_USER_GROUP = '5596e975ea893b269af85c0f'
BUGSNAG_API_KEY = ''
BUGSNAG_RELEASE_STAGE = 'development'
SENTRY_CONFIG = {
'dsn': '-set-in-config-local-',
# 'release': raven.fetch_git_sha(os.path.dirname(__file__)),
}
# See https://docs.sentry.io/clients/python/integrations/flask/#settings
SENTRY_USER_ATTRS = ['username', 'full_name', 'email', 'objectid']
ALGOLIA_USER = '-SECRET-'
ALGOLIA_API_KEY = '-SECRET-'
@@ -106,6 +111,12 @@ FULL_FILE_ACCESS_ROLES = {'admin', 'subscriber', 'demo'}
BLENDER_ID_CLIENT_ID = 'SPECIAL-SNOWFLAKE-57'
BLENDER_ID_SUBCLIENT_ID = 'PILLAR'
# Blender ID user info API endpoint URL and auth token, used for
# reconciling subscribers and updating their info from /u/.
# The token requires the 'userinfo' scope.
BLENDER_ID_USER_INFO_API = 'http://blender-id:8000/api/user/'
BLENDER_ID_USER_INFO_TOKEN = '-set-in-config-local-'
# Collection of supported OAuth providers (Blender ID, Facebook and Google).
# Example entry:
# OAUTH_CREDENTIALS = {
@@ -180,9 +191,6 @@ URLER_SERVICE_AUTH_TOKEN = None
# front-end.
BLENDER_CLOUD_ADDON_VERSION = '1.4'
EXTERNAL_SUBSCRIPTIONS_MANAGEMENT_SERVER = 'https://store.blender.org/api/'
EXTERNAL_SUBSCRIPTIONS_TIMEOUT_SECS = 10
# Certificate file for communication with other systems.
TLS_CERT_FILE = requests.certs.where()
@@ -204,8 +212,9 @@ CELERY_BEAT_SCHEDULE = {
USER_CAPABILITIES = defaultdict(**{
'subscriber': {'subscriber', 'home-project'},
'demo': {'subscriber', 'home-project'},
'admin': {'subscriber', 'home-project', 'video-encoding', 'admin',
'admin': {'encode-video', 'admin',
'view-pending-nodes', 'edit-project-node-types', 'create-organization'},
'video-encoder': {'encode-video'},
'org-subscriber': {'subscriber', 'home-project'},
}, default_factory=frozenset)

2
pillar/web/nodes/custom/comments.py
View File

@@ -134,7 +134,7 @@ def comments_for_node(node_id):
project = Project({'_id': node.project})
can_post_comments = project.node_type_has_method('comment', 'POST', api=api)
can_comment_override = request.args.get('can_comment', 'True') == 'True'
can_post_comments = can_post_comments and can_comment_override
can_post_comments = can_post_comments and can_comment_override and current_user.has_cap('subscriber')
# Query for all children, i.e. comments on the node.
comments = Node.all({

5
pillar/web/settings/routes.py
View File

@@ -23,13 +23,10 @@ def profile():
api = system_util.pillar_api()
user = User.find(current_user.objectid, api=api)
form = forms.UserProfileForm(
full_name=user.full_name,
username=user.username)
form = forms.UserProfileForm(username=user.username)
if form.validate_on_submit():
try:
user.full_name = form.full_name.data
user.username = form.username.data
user.update(api=api)
flash("Profile updated", 'success')

11
pillar/web/users/forms.py
View File

@@ -24,17 +24,16 @@ class UserLoginForm(Form):
class UserProfileForm(Form):
full_name = StringField('Full Name', validators=[DataRequired(), Length(
min=3, max=128, message="Min. 3 and max. 128 chars please")])
username = StringField('Username', validators=[DataRequired(), Length(
min=3, max=128, message="Min. 3, max. 128 chars please"), Regexp(
r'^[\w.@+-]+$', message="Please do not use spaces")])
def __init__(self, csrf_enabled=False, *args, **kwargs):
super(UserProfileForm, self).__init__(csrf_enabled=False, *args, **kwargs)
super().__init__(csrf_enabled=csrf_enabled, *args, **kwargs)
self.user = None
def validate(self):
rv = Form.validate(self)
rv = super().validate()
if not rv:
return False
@@ -42,11 +41,11 @@ class UserProfileForm(Form):
user = User.find(current_user.objectid, api=api)
if user.username != self.username.data:
username = User.find_first(
{'where': '{"username": "%s"}' % self.username.data},
{'where': {"username": self.username.data}},
api=api)
if username:
self.username.errors.append('Sorry, username already exists!')
self.username.errors.append('Sorry, this username is already taken.')
return False
self.user = user

3
pillar/web/users/routes.py
View File

@@ -69,8 +69,7 @@ def oauth_callback(provider):
pillar.auth.login_user(token['token'], load_from_db=True)
if provider == 'blender-id' and current_user.is_authenticated:
# Check with the store for user roles. If the user has an active subscription, we apply
# the 'subscriber' role
# Check with Blender ID to update certain user roles.
update_subscription()
next_after_login = session.pop('next_after_login', None)