T53161 WIP create elasticsearch app / doc / stuff
This commit is contained in:
parent
b6af919fa9
commit
1fe88819f4
@ -241,7 +241,7 @@ class PillarServer(Eve):
|
||||
|
||||
def _config_algolia(self):
|
||||
# Algolia search
|
||||
if self.config['SEARCH_BACKEND'] != 'algolia':
|
||||
if 'algolia' not in self.config['SEARCH_BACKENDS']:
|
||||
return
|
||||
|
||||
from algoliasearch import algoliasearch
|
||||
|
||||
0
pillar/api/search/__init__.py
Normal file
0
pillar/api/search/__init__.py
Normal file
38
pillar/api/search/algolia_indexing.py
Normal file
38
pillar/api/search/algolia_indexing.py
Normal file
@ -0,0 +1,38 @@
|
||||
import logging
|
||||
|
||||
from algoliasearch.helpers import AlgoliaException
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def push_updated_user(user_to_index: dict):
|
||||
"""Push an update to the Algolia index when a user item is updated"""
|
||||
|
||||
from pillar.api.utils.algolia import index_user_save
|
||||
|
||||
try:
|
||||
index_user_save(user_to_index)
|
||||
except AlgoliaException as ex:
|
||||
log.warning(
|
||||
'Unable to push user info to Algolia for user "%s", id=%s; %s', # noqa
|
||||
user_to_index.get('username'),
|
||||
user_to_index.get('objectID'), ex)
|
||||
|
||||
|
||||
def index_node_save(node_to_index: dict):
|
||||
from pillar.api.utils import algolia
|
||||
|
||||
try:
|
||||
algolia.index_node_save(node_to_index)
|
||||
except AlgoliaException as ex:
|
||||
log.warning(
|
||||
'Unable to push node info to Algolia for node %s; %s', node_to_index, ex) # noqa
|
||||
|
||||
|
||||
def index_node_delete(delete_id: str):
|
||||
|
||||
from pillar.api.utils import algolia
|
||||
try:
|
||||
algolia.index_node_delete(delete_id)
|
||||
except AlgoliaException as ex:
|
||||
log.warning('Unable to delete node info to Algolia for node %s; %s', delete_id, ex) # noqa
|
||||
66
pillar/api/search/documents.py
Normal file
66
pillar/api/search/documents.py
Normal file
@ -0,0 +1,66 @@
|
||||
import logging
|
||||
|
||||
import elasticsearch_dsl as es
|
||||
from elasticsearch_dsl import analysis
|
||||
# from pillar import current_app
|
||||
|
||||
# define elasticsearch document mapping.
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
edge_ngram_filter = analysis.token_filter(
|
||||
'edge_ngram_filter',
|
||||
type='edge_ngram',
|
||||
min_gram=1,
|
||||
max_gram=15
|
||||
)
|
||||
|
||||
|
||||
autocomplete = es.analyzer(
|
||||
'autocomplete',
|
||||
tokenizer='standard',
|
||||
filter=['lowercase', edge_ngram_filter]
|
||||
)
|
||||
|
||||
|
||||
class User(es.DocType):
|
||||
"""
|
||||
Elastic document describing user
|
||||
"""
|
||||
|
||||
name = es.String(
|
||||
fielddata=True,
|
||||
analyzer=autocomplete,
|
||||
)
|
||||
|
||||
|
||||
class Node(es.DocType):
|
||||
"""
|
||||
Elastic document describing user
|
||||
"""
|
||||
|
||||
node_type = es.Keyword()
|
||||
|
||||
x_code = es.String(
|
||||
multi=True,
|
||||
fielddata=True,
|
||||
analyzer=autocomplete,
|
||||
)
|
||||
|
||||
|
||||
def create_doc_from_user_data(user_to_index):
|
||||
doc_id = user_to_index['objectID']
|
||||
doc = User(_id=doc_id)
|
||||
return doc
|
||||
|
||||
|
||||
def create_doc_from_node_data(node_to_index):
|
||||
|
||||
# node stuff
|
||||
doc_id = node_to_index['objectID']
|
||||
doc = Node(_id=doc_id)
|
||||
doc.node_type = node_to_index['node_type']
|
||||
|
||||
return doc
|
||||
25
pillar/api/search/elastic_indexing.py
Normal file
25
pillar/api/search/elastic_indexing.py
Normal file
@ -0,0 +1,25 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def push_updated_user(user_to_index: dict):
|
||||
"""Push an update to the Algolia index when a user item is updated"""
|
||||
|
||||
log.warning(
|
||||
'WIP USER ELK INDEXING %s %s',
|
||||
user_to_index.get('username'),
|
||||
user_to_index.get('objectID'))
|
||||
|
||||
|
||||
def index_node_save(node_to_index: dict):
|
||||
|
||||
log.warning(
|
||||
'WIP USER NODE INDEXING %s',
|
||||
node_to_index.get('objectID'))
|
||||
|
||||
|
||||
def index_node_delete(delete_id: str):
|
||||
|
||||
log.warning(
|
||||
'WIP NODE DELETE INDEXING %s', delete_id)
|
||||
@ -3,123 +3,32 @@ import logging
|
||||
from bson import ObjectId
|
||||
|
||||
from pillar import current_app
|
||||
from pillar.api.file_storage import generate_link
|
||||
from . import skip_when_testing
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
INDEX_ALLOWED_NODE_TYPES = {'asset', 'texture', 'group', 'hdri'}
|
||||
|
||||
|
||||
@skip_when_testing
|
||||
def algolia_index_user_save(user):
|
||||
def index_user_save(to_index_user: dict):
|
||||
index_users = current_app.algolia_index_users
|
||||
if not index_users:
|
||||
log.debug('No Algolia index defined, so nothing to do.')
|
||||
return
|
||||
|
||||
user_roles = set(user.get('roles', ()))
|
||||
if 'service' in user_roles:
|
||||
return
|
||||
|
||||
# Strip unneeded roles
|
||||
index_roles = user_roles.intersection(current_app.user_roles_indexable)
|
||||
|
||||
# Create or update Algolia index for the user
|
||||
index_users.save_object({
|
||||
'objectID': user['_id'],
|
||||
'full_name': user['full_name'],
|
||||
'username': user['username'],
|
||||
'roles': list(index_roles),
|
||||
'groups': user['groups'],
|
||||
'email': user['email']
|
||||
})
|
||||
|
||||
log.debug(
|
||||
'Pushed user %r to Algolia index %r',
|
||||
user['_id'], index_users.index_name)
|
||||
|
||||
|
||||
def _handle_picture(node: dict, to_index: dict):
|
||||
"""
|
||||
add picture fields to be indexed
|
||||
"""
|
||||
|
||||
if not node.get('picture'):
|
||||
return
|
||||
|
||||
files_collection = current_app.data.driver.db['files']
|
||||
lookup = {'_id': ObjectId(node['picture'])}
|
||||
picture = files_collection.find_one(lookup)
|
||||
|
||||
img_variation_t = next(
|
||||
(item for item in picture['variations']
|
||||
if item['size'] == 't'), None)
|
||||
|
||||
if img_variation_t:
|
||||
to_index['picture'] = generate_link(
|
||||
picture['backend'],
|
||||
img_variation_t['file_path'],
|
||||
project_id=str(picture['project']),
|
||||
is_public=True)
|
||||
index_users.save_object()
|
||||
|
||||
|
||||
@skip_when_testing
|
||||
def algolia_index_node_save(node):
|
||||
def index_node_save(node_to_index):
|
||||
|
||||
if not current_app.algolia_index_nodes:
|
||||
return
|
||||
if node['node_type'] not in INDEX_ALLOWED_NODE_TYPES:
|
||||
return
|
||||
# If a nodes does not have status published, do not index
|
||||
if node['properties'].get('status') != 'published':
|
||||
return
|
||||
|
||||
projects_collection = current_app.data.driver.db['projects']
|
||||
project = projects_collection.find_one({'_id': ObjectId(node['project'])})
|
||||
|
||||
users_collection = current_app.data.driver.db['users']
|
||||
user = users_collection.find_one({'_id': ObjectId(node['user'])})
|
||||
|
||||
to_index = {
|
||||
'objectID': node['_id'],
|
||||
'name': node['name'],
|
||||
'project': {
|
||||
'_id': project['_id'],
|
||||
'name': project['name']
|
||||
},
|
||||
'created': node['_created'],
|
||||
'updated': node['_updated'],
|
||||
'node_type': node['node_type'],
|
||||
'user': {
|
||||
'_id': user['_id'],
|
||||
'full_name': user['full_name']
|
||||
},
|
||||
}
|
||||
|
||||
if 'description' in node and node['description']:
|
||||
to_index['description'] = node['description']
|
||||
|
||||
_handle_picture(node, to_index)
|
||||
|
||||
# If the node has world permissions, compute the Free permission
|
||||
if 'world' in node.get('permissions', {}):
|
||||
if 'GET' in node['permissions']['world']:
|
||||
to_index['is_free'] = True
|
||||
|
||||
# Append the media key if the node is of node_type 'asset'
|
||||
if node['node_type'] == 'asset':
|
||||
to_index['media'] = node['properties']['content_type']
|
||||
|
||||
# Add extra properties
|
||||
for prop in ('tags', 'license_notes'):
|
||||
if prop in node['properties']:
|
||||
to_index[prop] = node['properties'][prop]
|
||||
|
||||
current_app.algolia_index_nodes.save_object(to_index)
|
||||
current_app.algolia_index_nodes.save_object(node_to_index)
|
||||
|
||||
|
||||
@skip_when_testing
|
||||
def algolia_index_node_delete(node):
|
||||
def index_node_delete(delete_id):
|
||||
if current_app.algolia_index_nodes is None:
|
||||
return
|
||||
current_app.algolia_index_nodes.delete_object(node['_id'])
|
||||
current_app.algolia_index_nodes.delete_object(delete_id)
|
||||
|
||||
@ -1,61 +1,38 @@
|
||||
import logging
|
||||
|
||||
from algoliasearch.helpers import AlgoliaException
|
||||
import bson
|
||||
|
||||
from pillar import current_app
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def push_updated_user(user_id: str):
|
||||
def push_updated_user(user_to_index: dict):
|
||||
"""Push an update to the Algolia index when a user item is updated"""
|
||||
|
||||
from pillar.api.utils.algolia import algolia_index_user_save
|
||||
|
||||
user_oid = bson.ObjectId(user_id)
|
||||
log.info('Retrieving user %s', user_oid)
|
||||
users_coll = current_app.db('users')
|
||||
user = users_coll.find_one({'_id': user_oid})
|
||||
if user is None:
|
||||
log.warning('Unable to find user %s, not updating Algolia.', user_oid)
|
||||
return
|
||||
from pillar.api.utils.algolia import index_user_save
|
||||
|
||||
try:
|
||||
algolia_index_user_save(user)
|
||||
index_user_save(user_to_index)
|
||||
except AlgoliaException as ex:
|
||||
log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s', # noqa
|
||||
user.get('username'), user_id, ex)
|
||||
log.warning(
|
||||
'Unable to push user info to Algolia for user "%s", id=%s; %s', # noqa
|
||||
user_to_index.get('username'),
|
||||
user_to_index.get('objectID'), ex)
|
||||
|
||||
|
||||
def index_node_save(node_id: str):
|
||||
from pillar.api.utils.algolia import algolia_index_node_save
|
||||
|
||||
node_oid = bson.ObjectId(node_id)
|
||||
log.info('Retrieving node %s', node_oid)
|
||||
|
||||
nodes_coll = current_app.db('nodes')
|
||||
node = nodes_coll.find_one({'_id': node_oid})
|
||||
|
||||
if node is None:
|
||||
log.warning('Unable to find node %s, not updating Algolia.', node_id)
|
||||
return
|
||||
def index_node_save(node_to_index: dict):
|
||||
from pillar.api.utils import algolia
|
||||
|
||||
try:
|
||||
algolia_index_node_save(node)
|
||||
algolia.index_node_save(node_to_index)
|
||||
except AlgoliaException as ex:
|
||||
log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex) # noqa
|
||||
log.warning(
|
||||
'Unable to push node info to Algolia for node %s; %s', node_to_index, ex) # noqa
|
||||
|
||||
|
||||
def index_node_delete(node_id: str):
|
||||
|
||||
from pillar.api.utils.algolia import algolia_index_node_delete
|
||||
|
||||
# Deleting a node takes nothing more than the ID anyway.
|
||||
# No need to fetch anything from Mongo.
|
||||
fake_node = {'_id': bson.ObjectId(node_id)}
|
||||
def index_node_delete(delete_id: str):
|
||||
|
||||
from pillar.api.utils import algolia
|
||||
try:
|
||||
algolia_index_node_delete(fake_node)
|
||||
algolia.index_node_delete(delete_id)
|
||||
except AlgoliaException as ex:
|
||||
log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex) # noqa
|
||||
log.warning('Unable to delete node info to Algolia for node %s; %s', delete_id, ex) # noqa
|
||||
|
||||
@ -1,45 +1,179 @@
|
||||
import logging
|
||||
from bson import ObjectId
|
||||
from pillar import current_app
|
||||
from pillar.api.file_storage import generate_link
|
||||
|
||||
from . import algolia_indexing
|
||||
# from . import elastic_indexing
|
||||
# TODO WIP (stephan) make index backend conditional on settings.
|
||||
from pillar.api.search import elastic_indexing
|
||||
|
||||
from pillar.api.search import algolia_indexing
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# TODO(stephan) make index backend conditional on settings.
|
||||
|
||||
INDEX_ALLOWED_NODE_TYPES = {'asset', 'texture', 'group', 'hdri'}
|
||||
|
||||
|
||||
SEARCH_BACKENDS = {
|
||||
'algolia': algolia_indexing,
|
||||
'elastic': None, # elastic_indexing
|
||||
'elastic': elastic_indexing
|
||||
}
|
||||
|
||||
|
||||
def _get_node_from_id(node_id: str):
|
||||
"""
|
||||
"""
|
||||
node_oid = ObjectId(node_id)
|
||||
log.info('Retrieving node %s', node_oid)
|
||||
|
||||
nodes_coll = current_app.db('nodes')
|
||||
node = nodes_coll.find_one({'_id': node_oid})
|
||||
|
||||
return node
|
||||
|
||||
|
||||
def _handle_picture(node: dict, to_index: dict):
|
||||
"""
|
||||
add picture fields to be indexed
|
||||
"""
|
||||
|
||||
if not node.get('picture'):
|
||||
return
|
||||
|
||||
files_collection = current_app.data.driver.db['files']
|
||||
lookup = {'_id': ObjectId(node['picture'])}
|
||||
picture = files_collection.find_one(lookup)
|
||||
|
||||
img_variation_t = next(
|
||||
(item for item in picture['variations']
|
||||
if item['size'] == 't'), None)
|
||||
|
||||
if img_variation_t:
|
||||
to_index['picture'] = generate_link(
|
||||
picture['backend'],
|
||||
img_variation_t['file_path'],
|
||||
project_id=str(picture['project']),
|
||||
is_public=True)
|
||||
|
||||
|
||||
def prepare_node_data(node_id: str):
|
||||
"""
|
||||
Given node build data object with fields to index
|
||||
"""
|
||||
node = _get_node_from_id(node_id)
|
||||
|
||||
if node is None:
|
||||
log.warning('Unable to find node %s, not updating Algolia.', node_id)
|
||||
return
|
||||
|
||||
if node['node_type'] not in INDEX_ALLOWED_NODE_TYPES:
|
||||
return
|
||||
# If a nodes does not have status published, do not index
|
||||
if node['properties'].get('status') != 'published':
|
||||
return
|
||||
|
||||
projects_collection = current_app.data.driver.db['projects']
|
||||
project = projects_collection.find_one({'_id': ObjectId(node['project'])})
|
||||
|
||||
users_collection = current_app.data.driver.db['users']
|
||||
user = users_collection.find_one({'_id': ObjectId(node['user'])})
|
||||
|
||||
to_index = {
|
||||
'objectID': node['_id'],
|
||||
'name': node['name'],
|
||||
'project': {
|
||||
'_id': project['_id'],
|
||||
'name': project['name']
|
||||
},
|
||||
'created': node['_created'],
|
||||
'updated': node['_updated'],
|
||||
'node_type': node['node_type'],
|
||||
'user': {
|
||||
'_id': user['_id'],
|
||||
'full_name': user['full_name']
|
||||
},
|
||||
}
|
||||
|
||||
if 'description' in node and node['description']:
|
||||
to_index['description'] = node['description']
|
||||
|
||||
_handle_picture(node, to_index)
|
||||
|
||||
# If the node has world permissions, compute the Free permission
|
||||
if 'world' in node.get('permissions', {}):
|
||||
if 'GET' in node['permissions']['world']:
|
||||
to_index['is_free'] = True
|
||||
|
||||
# Append the media key if the node is of node_type 'asset'
|
||||
if node['node_type'] == 'asset':
|
||||
to_index['media'] = node['properties']['content_type']
|
||||
|
||||
# Add extra properties
|
||||
for prop in ('tags', 'license_notes'):
|
||||
if prop in node['properties']:
|
||||
to_index[prop] = node['properties'][prop]
|
||||
|
||||
return to_index
|
||||
|
||||
|
||||
def prepare_user_data(user_id: str):
|
||||
"""
|
||||
Prepare data to index for user node
|
||||
"""
|
||||
|
||||
user_oid = ObjectId(user_id)
|
||||
log.info('Retrieving user %s', user_oid)
|
||||
users_coll = current_app.db('users')
|
||||
user = users_coll.find_one({'_id': user_oid})
|
||||
if user is None:
|
||||
log.warning('Unable to find user %s, not updating Algolia.', user_oid)
|
||||
return
|
||||
|
||||
user_roles = set(user.get('roles', ()))
|
||||
if 'service' in user_roles:
|
||||
return
|
||||
|
||||
# Strip unneeded roles
|
||||
index_roles = user_roles.intersection(current_app.user_roles_indexable)
|
||||
|
||||
log.debug('Pushed user %r to Search index', user['_id'])
|
||||
|
||||
user_to_index = {
|
||||
'objectID': user['_id'],
|
||||
'full_name': user['full_name'],
|
||||
'username': user['username'],
|
||||
'roles': list(index_roles),
|
||||
'groups': user['groups'],
|
||||
'email': user['email']
|
||||
}
|
||||
|
||||
return user_to_index
|
||||
|
||||
|
||||
@current_app.celery.task(ignore_result=True)
|
||||
def updated_user(user_id: str):
|
||||
"""Push an update to the index when a user item is updated"""
|
||||
|
||||
algolia_indexing.push_updated_user(user_id)
|
||||
user_to_index = prepare_user_data(user_id)
|
||||
|
||||
for searchoption in current_app.config['SEARCH_BACKENDS']:
|
||||
for searchmodule in SEARCH_BACKENDS[searchoption]:
|
||||
searchmodule.push_updated_user(user_to_index)
|
||||
|
||||
|
||||
@current_app.celery.task(ignore_result=True)
|
||||
def node_save(node_id: str):
|
||||
|
||||
algolia_indexing.index_node_save(node_id)
|
||||
to_index = prepare_node_data(node_id)
|
||||
|
||||
algolia_indexing.index_node_save(to_index)
|
||||
|
||||
|
||||
@current_app.celery.task(ignore_result=True)
|
||||
def node_delete(node_id: str):
|
||||
|
||||
algolia_indexing.index_node_delete(node_id)
|
||||
|
||||
|
||||
|
||||
def build_doc_to_index_from(node: dict):
|
||||
"""
|
||||
Given node build an to_index document
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
|
||||
# Deleting a node takes nothing more than the ID anyway.
|
||||
# No need to fetch anything from Mongo.
|
||||
delete_id = ObjectId(node_id)
|
||||
algolia_indexing.index_node_delete(delete_id)
|
||||
|
||||
@ -73,7 +73,7 @@ ALGOLIA_API_KEY = '-SECRET-'
|
||||
ALGOLIA_INDEX_USERS = 'dev_Users'
|
||||
ALGOLIA_INDEX_NODES = 'dev_Nodes'
|
||||
|
||||
SEARCH_BACKEND = 'algolia' # algolia, elastic
|
||||
SEARCH_BACKENDS = ['algolia', 'elastic']
|
||||
|
||||
|
||||
ZENCODER_API_KEY = '-SECRET-'
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user