From c7b6e798c0ef3162344b400aefb5de0a04a7377b Mon Sep 17 00:00:00 2001 From: Francesco Siddi Date: Wed, 10 Feb 2016 16:13:07 +0100 Subject: [PATCH] Introducing Algolia search for users With the new dependency, algoliasearch, we can push and store on the Algolia service a limited set of properties from the Users collection. --- pillar/application/__init__.py | 16 ++++++++++++++++ pillar/application/utils/algolia.py | 20 ++++++++++++++++++++ pillar/manage.py | 9 +++++++++ pillar/settings.py | 2 +- requirements.txt | 13 +++++++------ 5 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 pillar/application/utils/algolia.py diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index 1b82f42a..3f880115 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -4,6 +4,7 @@ from bson import ObjectId from datetime import datetime import bugsnag from bugsnag.flask import handle_exceptions +from algoliasearch import algoliasearch from flask import g from flask import request from flask import url_for @@ -95,11 +96,21 @@ bugsnag.configure( ) handle_exceptions(app) +# Algolia search +if 'ALGOLIA_USER' in app.config: + client = algoliasearch.Client( + app.config['ALGOLIA_USER'], + app.config['ALGOLIA_API_KEY']) + algolia_index_users = client.init_index(app.config['ALGOLIA_INDEX_USERS']) +else: + algolia_index_users = None + from application.utils.authentication import validate_token from application.utils.authorization import check_permissions from application.utils.cdn import hash_file_path from application.utils.gcs import GoogleCloudStorageBucket from application.utils.gcs import update_file_name +from application.utils.algolia import algolia_index_user_save def before_returning_item_permissions(response): @@ -226,7 +237,12 @@ def post_GET_user(request, payload): # compute_permissions(json_data['_id'], app.data.driver) payload.data = json.dumps(json_data) +def after_replacing_user(item, original): + """Push an update to the Algolia index when a user item is updated""" + algolia_index_user_save(item) + app.on_post_GET_users += post_GET_user +app.on_replace_users += after_replacing_user from modules.file_storage import process_file from modules.file_storage import delete_file diff --git a/pillar/application/utils/algolia.py b/pillar/application/utils/algolia.py new file mode 100644 index 00000000..822acf78 --- /dev/null +++ b/pillar/application/utils/algolia.py @@ -0,0 +1,20 @@ +from application import algolia_index_users + +def algolia_index_user_save(user): + # Define accepted roles + accepted_roles = ['admin', 'subscriber', 'demo'] + # Strip unneeded roles + if 'roles' in user: + roles = [r for r in user['roles'] if r in accepted_roles] + else: + roles = None + if algolia_index_users: + # Create or update Algolia index for the user + algolia_index_users.save_object({ + 'objectID': user['_id'], + 'full_name': user['full_name'], + 'username': user['username'], + 'roles': roles, + 'groups': user['groups'], + 'email': user['email'] + }) diff --git a/pillar/manage.py b/pillar/manage.py index 87f7b23d..6864ffdf 100644 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -550,5 +550,14 @@ def test_post_internal(node_id): print post_internal('nodes', node) +@manager.command +def algolia_push_users(): + """Loop through all users and push them to Algolia""" + from application.utils.algolia import algolia_index_user_save + users_collection = app.data.driver.db['users'] + for user in users_collection.find(): + print "Pushing {0}".format(user['username']) + algolia_index_user_save(user) + if __name__ == '__main__': manager.run() diff --git a/pillar/settings.py b/pillar/settings.py index b737207e..5d9b0a18 100644 --- a/pillar/settings.py +++ b/pillar/settings.py @@ -42,7 +42,7 @@ users_schema = { }, 'roles': { 'type': 'list', - 'allowed': ["admin", "subscriber"], + 'allowed': ["admin", "subscriber", "demo"], }, 'groups': { 'type': 'list', diff --git a/requirements.txt b/requirements.txt index 90d292af..1e283414 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,14 @@ +algoliasearch==1.7.1 blinker==1.4 bugsnag==2.3.1 -Cerberus==0.8 +Cerberus==0.9.1 cffi==1.2.1 cryptography==1.0.2 enum34==1.0.4 Eve==0.5.3 Events==0.2.1 Flask==0.10.1 -Flask-PyMongo==0.3.0 +Flask-PyMongo==0.3.1 Flask-Script==2.0.5 flup==1.0.2 gcloud==0.7.1 @@ -16,7 +17,7 @@ httplib2==0.9.2 idna==2.0 ipaddress==1.0.14 itsdangerous==0.24 -Jinja2==2.7.3 +Jinja2==2.8 MarkupSafe==0.23 ndg-httpsclient==0.4.0 oauth2client==1.5.1 @@ -27,12 +28,12 @@ pyasn1==0.1.9 pyasn1-modules==0.0.8 pycparser==2.14 pycrypto==2.6.1 -pymongo==2.8 +pymongo==2.9.1 pyOpenSSL==0.15.1 requests==2.6.0 rsa==3.2 -simplejson==3.6.5 +simplejson==3.8.1 six==1.9.0 WebOb==1.5.0 -Werkzeug==0.10.1 +Werkzeug==0.10.4 wheel==0.24.0