archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

T53161 start working on elastic..

Browse Source
This commit is contained in:
Stephan preeker 2017-10-25 17:09:10 +02:00
parent 49a6a6a758
commit 2233d015f3
7 changed files with 98 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pillar/__init__.py
View File

@ -466,7 +466,7 @@ class PillarServer(Eve):
# Pillar-defined Celery task modules:
celery_task_modules = [
'pillar.celery.tasks',
'pillar.celery.algolia_tasks',
'pillar.celery.search_index_tasks',
'pillar.celery.file_link_tasks',
]

14
pillar/api/nodes/__init__.py
View File

@ -167,7 +167,8 @@ def create_short_code(node) -> str:
def short_link_info(short_code):
"""Returns the short link info in a dict."""
short_link = urllib.parse.urljoin(current_app.config['SHORT_LINK_BASE_URL'], short_code)
short_link = urllib.parse.urljoin(
current_app.config['SHORT_LINK_BASE_URL'], short_code)
return {
'short_code': short_code,
@ -185,7 +186,7 @@ def after_replacing_node(item, original):
project is private, prevent public indexing.
"""
from pillar.celery import algolia_tasks
from pillar.celery import search_index_tasks as index
projects_collection = current_app.data.driver.db['projects']
project = projects_collection.find_one({'_id': item['project']})
@ -195,10 +196,11 @@ def after_replacing_node(item, original):
status = item['properties'].get('status', 'unpublished')
node_id = str(item['_id'])
if status == 'published':
algolia_tasks.algolia_index_node_save.delay(node_id)
index.node_save.delay(node_id)
else:
algolia_tasks.algolia_index_node_delete.delay(node_id)
index.node_delete.delay(node_id)
def before_inserting_nodes(items):
@ -372,8 +374,8 @@ def before_deleting_node(node: dict):
def after_deleting_node(item):
from pillar.celery import algolia_tasks
algolia_tasks.algolia_index_node_delete.delay(str(item['_id']))
from pillar.celery import search_index_tasks as index
index.node_delete.delay(str(item['_id']))
only_for_comments = only_for_node_type_decorator('comment')

17
pillar/api/users/hooks.py
View File

@ -61,19 +61,26 @@ def before_replacing_user(request, lookup):
# Regular users should always have an email address
if 'service' not in put_data.get('roles', ()):
if not put_data.get('email'):
raise wz_exceptions.UnprocessableEntity('email field must be given')
raise wz_exceptions.UnprocessableEntity(
'email field must be given')
def push_updated_user_to_algolia(user, original):
"""Push an update to the Algolia index when a user item is updated"""
"""
Push an update to the Algolia index when a user
item is updated
"""
from pillar.celery import algolia_tasks
from pillar.celery import search_index_tasks as index
algolia_tasks.push_updated_user_to_algolia.delay(str(user['_id']))
index.updated_user.delay(str(user['_id']))
def send_blinker_signal_roles_changed(user, original):
"""Sends a Blinker signal that the user roles were changed, so others can respond."""
"""
Sends a Blinker signal that the user roles were
changed, so others can respond.
"""
current_roles = set(user.get('roles', []))
original_roles = set(original.get('roles', []))

54
pillar/api/utils/algolia.py
View File

@ -35,7 +35,31 @@ def algolia_index_user_save(user):
'email': user['email']
})
log.debug('Pushed user %r to Algolia index %r', user['_id'], index_users.index_name)
log.debug(
'Pushed user %r to Algolia index %r',
user['_id'], index_users.index_name)
def _handle_picture(node, doc):
"""
add picture fields to be indexed
"""
if 'picture' in node and node['picture']:
files_collection = current_app.data.driver.db['files']
lookup = {'_id': ObjectId(node['picture'])}
picture = files_collection.find_one(lookup)
img_variation_t = next(
(item for item in picture['variations']
if item['size'] == 't'), None)
if img_variation_t:
doc['picture'] = generate_link(
picture['backend'],
img_variation_t['file_path'],
project_id=str(picture['project']),
is_public=True)
@skip_when_testing
@ -54,7 +78,7 @@ def algolia_index_node_save(node):
users_collection = current_app.data.driver.db['users']
user = users_collection.find_one({'_id': ObjectId(node['user'])})
node_ob = {
doc = {
'objectID': node['_id'],
'name': node['name'],
'project': {
@ -69,35 +93,27 @@ def algolia_index_node_save(node):
'full_name': user['full_name']
},
}
if 'description' in node and node['description']:
node_ob['description'] = node['description']
if 'picture' in node and node['picture']:
files_collection = current_app.data.driver.db['files']
lookup = {'_id': ObjectId(node['picture'])}
picture = files_collection.find_one(lookup)
if picture['backend'] == 'gcs':
variation_t = next((item for item in picture['variations'] \
if item['size'] == 't'), None)
if variation_t:
node_ob['picture'] = generate_link(picture['backend'],
variation_t['file_path'],
project_id=str(picture['project']),
is_public=True)
doc['description'] = node['description']
_handle_picture(node, doc)
# If the node has world permissions, compute the Free permission
if 'permissions' in node and 'world' in node['permissions']:
if 'GET' in node['permissions']['world']:
node_ob['is_free'] = True
doc['is_free'] = True
# Append the media key if the node is of node_type 'asset'
if node['node_type'] == 'asset':
node_ob['media'] = node['properties']['content_type']
doc['media'] = node['properties']['content_type']
# Add extra properties
for prop in ('tags', 'license_notes'):
if prop in node['properties']:
node_ob[prop] = node['properties'][prop]
doc[prop] = node['properties'][prop]
current_app.algolia_index_nodes.save_object(node_ob)
current_app.algolia_index_nodes.save_object(doc)
@skip_when_testing

16
pillar/celery/algolia_tasks.py
View File

@ -8,7 +8,6 @@ from pillar import current_app
log = logging.getLogger(__name__)
@current_app.celery.task(ignore_result=True)
def push_updated_user_to_algolia(user_id: str):
"""Push an update to the Algolia index when a user item is updated"""
@ -25,12 +24,11 @@ def push_updated_user_to_algolia(user_id: str):
try:
algolia_index_user_save(user)
except AlgoliaException as ex:
log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s',
log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s', # noqa
user.get('username'), user_id, ex)
@current_app.celery.task(ignore_result=True)
def algolia_index_node_save(node_id: str):
def index_node_save(node_id: str):
from pillar.api.utils.algolia import algolia_index_node_save
node_oid = bson.ObjectId(node_id)
@ -46,17 +44,17 @@ def algolia_index_node_save(node_id: str):
try:
algolia_index_node_save(node)
except AlgoliaException as ex:
log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex)
log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex) # noqa
@current_app.celery.task(ignore_result=True)
def algolia_index_node_delete(node_id: str):
def index_node_delete(node_id: str):
from pillar.api.utils.algolia import algolia_index_node_delete
# Deleting a node takes nothing more than the ID anyway. No need to fetch anything from Mongo.
# Deleting a node takes nothing more than the ID anyway.
# No need to fetch anything from Mongo.
fake_node = {'_id': bson.ObjectId(node_id)}
try:
algolia_index_node_delete(fake_node)
except AlgoliaException as ex:
log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex)
log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex) # noqa

30
pillar/celery/search_index_tasks.py Normal file
View File

@ -0,0 +1,30 @@
import logging
from . import algolia_tasks
from pillar import current_app
log = logging.getLogger(__name__)
# TODO make index backend conditional on settings.
# now uses angolia, but should use elastic
@current_app.celery.task(ignore_result=True)
def updated_user(user_id: str):
"""Push an update to the index when a user item is updated"""
algolia_tasks.push_updated_user_to_algolia(user_id)
@current_app.celery.task(ignore_result=True)
def node_save(node_id: str):
algolia_tasks.index_node_save(node_id)
@current_app.celery.task(ignore_result=True)
def node_delete(node_id: str):
algolia_tasks.index_node_delete(node_id)

5
src/scripts/algolia_search.js
View File

@ -21,6 +21,7 @@ $(document).ready(function() {
var sliderTemplate = Hogan.compile($('#slider-template').text());
var paginationTemplate = Hogan.compile($('#pagination-template').text());
// replace with something elasticy!
// Client initialization
var algolia = algoliasearch(APPLICATION_ID, SEARCH_ONLY_API_KEY);
@ -36,6 +37,7 @@ $(document).ready(function() {
})
};
// replace with something elastici!
// Setup the search helper
var helper = algoliasearchHelper(algolia, INDEX_NAME, params);
@ -43,6 +45,7 @@ $(document).ready(function() {
var result = $.grep(FACET_CONFIG, function(e) {
return e.hidden && e.hidden == true;
});
for (var i = 0; i < result.length; i++) {
var f = result[i];
helper.addFacetRefinement(f.name, f.value);
@ -60,9 +63,11 @@ $(document).ready(function() {
helper.on('change', function(state) {
setURLParams(state);
});
helper.on('error', function(error) {
console.log(error);
});
helper.on('result', function(content, state) {
renderStats(content);
renderHits(content);