From 9cd3d97c75351e5166327a6f5edc3a7360584965 Mon Sep 17 00:00:00 2001 From: Stephan Preeker Date: Fri, 24 Nov 2017 17:47:38 +0100 Subject: [PATCH] T53161 javascript search stuff almost complete. --- pillar/api/search/queries.py | 34 +++- pillar/api/search/routes.py | 20 +- src/scripts/algolia_search.js | 226 +++++++++------------ src/scripts/tutti/4_search.js | 105 +++++++++- src/scripts/tutti/7_user_search.js | 2 +- src/templates/organizations/view_embed.pug | 6 - src/templates/projects/sharing.pug | 7 +- 7 files changed, 245 insertions(+), 155 deletions(-) diff --git a/pillar/api/search/queries.py b/pillar/api/search/queries.py index 2fe038be..1a94a955 100644 --- a/pillar/api/search/queries.py +++ b/pillar/api/search/queries.py @@ -18,7 +18,20 @@ client = Elasticsearch() log = logging.getLogger(__name__) -def do_search(query: str) -> dict: + +def add_aggs_to_search(search): + """ + """ + + agg_terms = ['node_type', 'media', 'tags', 'is_free'] + + for term in agg_terms: + search.aggs.bucket(term, 'terms', field=term) + + #search.aggs.bucket('project', 'terms', field='project.name') + + +def do_search(query: str, terms: dict) -> dict: """ Given user input search for node/stuff """ @@ -32,15 +45,26 @@ def do_search(query: str) -> dict: Q('term', media=query), Q('term', tags=query), ] + + #must = [] + + #for field, value in terms.items(): + # must.append( + bool_query = Q('bool', should=should) search = Search(using=client) search.query = bool_query + add_aggs_to_search(search) + if current_app.config['DEBUG']: - log.debug(json.dumps(search.to_dict(), indent=4)) + print(json.dumps(search.to_dict(), indent=4)) response = search.execute() + if current_app.config['DEBUG']: + print(json.dumps(response.to_dict(), indent=4)) + return response.to_dict() @@ -61,6 +85,9 @@ def do_user_search(query: str) -> dict: response = search.execute() + if current_app.config['DEBUG']: + log.debug('%s', json.dumps(response.to_dict(), indent=4)) + return response.to_dict() @@ -82,4 +109,7 @@ def do_user_search_admin(query: str) -> dict: response = search.execute() + if current_app.config['DEBUG']: + log.debug(json.dumps(response.to_dict(), indent=4)) + return response.to_dict() diff --git a/pillar/api/search/routes.py b/pillar/api/search/routes.py index ad637d09..378a4fa4 100644 --- a/pillar/api/search/routes.py +++ b/pillar/api/search/routes.py @@ -30,10 +30,28 @@ def _valid_search() -> str: return searchword +def _term_filters() -> dict: + """ + Check if frontent want to filter stuff + """ + + terms = [ + 'node_type', 'media', + 'tags', 'is_free', 'projectname'] + + parsed_terms = {} + + for term in terms: + parsed_terms[term] = request.args.get(term, '') + + return parsed_terms + + @blueprint_search.route('/') def search_nodes(): searchword = _valid_search() - data = queries.do_search(searchword) + terms = _term_filters() + data = queries.do_search(searchword, terms) return jsonify(data) diff --git a/src/scripts/algolia_search.js b/src/scripts/algolia_search.js index be2982c1..b537d901 100644 --- a/src/scripts/algolia_search.js +++ b/src/scripts/algolia_search.js @@ -1,11 +1,5 @@ $(document).ready(function() { - /******************** - * INITIALIZATION - * - * TODO (stephan) - * *******************/ - var HITS_PER_PAGE = 25; var MAX_VALUES_PER_FACET = 30; @@ -23,65 +17,44 @@ $(document).ready(function() { var sliderTemplate = Hogan.compile($('#slider-template').text()); var paginationTemplate = Hogan.compile($('#pagination-template').text()); - // replace with something elasticy! - // Client initialization - var algolia = algoliasearch(APPLICATION_ID, SEARCH_ONLY_API_KEY); - - // Helper initialization - var params = { - hitsPerPage: HITS_PER_PAGE, - maxValuesPerFacet: MAX_VALUES_PER_FACET, - facets: $.map(FACET_CONFIG, function(facet) { - return !facet.disjunctive ? facet.name : null; - }), - disjunctiveFacets: $.map(FACET_CONFIG, function(facet) { - return facet.disjunctive ? facet.name : null; - }) - }; - - // replace with something elastici! - // Setup the search helper - var helper = algoliasearchHelper(algolia, INDEX_NAME, params); - - // Check if we passed hidden facets in the FACET_CONFIG - var result = $.grep(FACET_CONFIG, function(e) { - return e.hidden && e.hidden == true; - }); - - for (var i = 0; i < result.length; i++) { - var f = result[i]; - helper.addFacetRefinement(f.name, f.value); - } - + // something elasticy! + var search = elasticSearcher; + // facets: $.map(FACET_CONFIG, function(facet) { + // return !facet.disjunctive ? facet.name : null; + // }), + // disjunctiveFacets: $.map(FACET_CONFIG, function(facet) { + // return facet.disjunctive ? facet.name : null; + // }) + //}; // Input binding $inputField.on('keyup change', function() { var query = $inputField.val(); + if(query === undefined) { return; } toggleIconEmptyInput(!query.trim()); - helper.setQuery(query).search(); + search.setQuery(query); + //setURLParams(search); + search.execute(); }).focus(); // AlgoliaHelper events - helper.on('change', function(state) { - setURLParams(state); - }); + //helper.on('change', function(state) { + //setURLParams(search); + //}); - helper.on('error', function(error) { - console.log(error); - }); - - helper.on('result', function(content, state) { + search.on('results', function(content){ renderStats(content); renderHits(content); - renderFacets(content, state); + renderFacets(content); renderPagination(content); bindSearchObjects(); - renderFirstHit($(hits).children('.search-hit:first')); }); - /************ - * SEARCH + //}); + + /*************** + * SEARCH RENDERING * ***********/ function renderFirstHit(firstHit) { @@ -113,11 +86,11 @@ $(document).ready(function() { $('#search-error').show().html('Houston!\n\n' + data.status + ' ' + data.statusText); }); }, 1000); - }; + } // Initial search initWithUrlParams(); - helper.search(); + //helper.search(); function convertTimestamp(timestamp) { var d = new Date(timestamp * 1000), // Convert the passed timestamp to milliseconds @@ -134,8 +107,8 @@ $(document).ready(function() { function renderStats(content) { var stats = { - nbHits: numberWithDelimiter(content.nbHits), - processingTimeMS: content.processingTimeMS, + nbHits: numberWithDelimiter(content.count), + processingTimeMS: content.took, nbHits_plural: content.nbHits !== 1 }; $stats.html(statsTemplate.render(stats)); @@ -145,13 +118,13 @@ $(document).ready(function() { var hitsHtml = ''; for (var i = 0; i < content.hits.length; ++i) { // console.log(content.hits[i]); - var created = content.hits[i]['created']; + var created = content.hits[i].created; if (created) { - content.hits[i]['created'] = convertTimestamp(created); + content.hits[i].created = convertTimestamp(created); } - var updated = content.hits[i]['updated']; + var updated = content.hits[i].updated; if (updated) { - content.hits[i]['updated'] = convertTimestamp(updated); + content.hits[i].updated = convertTimestamp(updated); } hitsHtml += hitTemplate.render(content.hits[i]); } @@ -159,87 +132,67 @@ $(document).ready(function() { $hits.html(hitsHtml); } - function renderFacets(content, state) { + function renderFacets(content) { + // If no results if (content.hits.length === 0) { $facets.empty(); return; } - // Process facets - var facets = []; - for (var facetIndex = 0; facetIndex < FACET_CONFIG.length; ++facetIndex) { - var facetParams = FACET_CONFIG[facetIndex]; - if (facetParams.hidden) { - continue - } - var facetResult = content.getFacetByName(facetParams.name); - if (facetResult) { - var facetContent = {}; - facetContent.facet = facetParams.name; - facetContent.title = facetParams.title; - facetContent.type = facetParams.type; + var storeValue = function (values, label){ - if (facetParams.type === 'slider') { - // if the facet is a slider - facetContent.min = facetResult.stats.min; - facetContent.max = facetResult.stats.max; - var valueMin = state.getNumericRefinement(facetParams.name, '>=') || facetResult.stats.min; - var valueMax = state.getNumericRefinement(facetParams.name, '<=') || facetResult.stats.max; - valueMin = Math.min(facetContent.max, Math.max(facetContent.min, valueMin)); - valueMax = Math.min(facetContent.max, Math.max(facetContent.min, valueMax)); - facetContent.values = [valueMin, valueMax]; - } else { - // format and sort the facet values - var values = []; - for (var v in facetResult.data) { - var label = ''; - if (v === 'true') { - label = 'Yes'; - } else if (v === 'false') { - label = 'No'; - } - // Remove any underscore from the value - else { - label = v.replace(/_/g, " "); - } - values.push({ - label: label, - value: v, - count: facetResult.data[v], - refined: helper.isRefined(facetParams.name, v) - }); - } - var sortFunction = facetParams.sortFunction || sortByCountDesc; - if (facetParams.topListIfRefined) sortFunction = sortByRefined(sortFunction); - values.sort(sortFunction); + return function(item){ + values.push({ + facet: label, + label: item.key, + value: item.key, + count: item.doc_count, + }); + }; + }; + + console.log('FACETS'); + var facets =[]; + var aggs = content.aggs; + + for (var label in aggs) { + + let values = []; + + let buckets = aggs[label].buckets; + + if (buckets.length === 0) { continue; } + + buckets.forEach(storeValue(values, label)); + + facets.push({ + title: label, + values: values.slice(0), + }); + } - facetContent.values = values.slice(0, 10); - facetContent.has_other_values = values.length > 10; - facetContent.other_values = values.slice(10); - facetContent.disjunctive = facetParams.disjunctive; - } - facets.push(facetContent); - } - } // Display facets var facetsHtml = ''; + for (var indexFacet = 0; indexFacet < facets.length; ++indexFacet) { var facet = facets[indexFacet]; - if (facet.type && facet.type === 'slider') facetsHtml += sliderTemplate.render(facet); - else facetsHtml += facetTemplate.render(facet); + //title, values[facet, value] + facetsHtml += facetTemplate.render(facet); } + $facets.html(facetsHtml); } function renderPagination(content) { // If no results - if (content.hits.length === 0) { + if (content.count === 0) { $pagination.empty(); return; } var maxPages = 2; + var nbPages = content.count / HITS_PER_PAGE; // Process pagination var pages = []; @@ -252,7 +205,7 @@ $(document).ready(function() { // pages.push({ current: false, number: '...', disabled: true }); } for (var p = content.page - maxPages; p < content.page + maxPages; ++p) { - if (p < 0 || p >= content.nbPages) { + if (p < 0 || p >= nbPages) { continue; } pages.push({ @@ -260,18 +213,18 @@ $(document).ready(function() { number: (p + 1) }); } - if (content.page + maxPages < content.nbPages) { + if (content.page + maxPages < nbPages) { // They don't really add much... // pages.push({ current: false, number: '...', disabled: true }); pages.push({ current: false, - number: content.nbPages + number: nbPages }); } var pagination = { pages: pages, prev_page: (content.page > 0 ? content.page : false), - next_page: (content.page + 1 < content.nbPages ? content.page + 2 : false) + next_page: (content.page + 1 < nbPages ? content.page + 2 : false) }; // Display pagination $pagination.html(paginationTemplate.render(pagination)); @@ -297,12 +250,15 @@ $(document).ready(function() { $(this).closest('ul').find('.show-less').toggle(); return false; }); + $(document).on('click', '.toggleRefine', function() { - helper.toggleRefine($(this).data('facet'), $(this).data('value')).search(); + search.addTerm($(this).data('facet'), $(this).data('value')); + search.execute(); return false; }); + $(document).on('click', '.gotoPage', function() { - helper.setCurrentPage(+$(this).data('page') - 1).search(); + //helper.setCurrentPage(+$(this).data('page') - 1).search(); $("html, body").animate({ scrollTop: 0 }, '500', 'swing'); @@ -310,7 +266,7 @@ $(document).ready(function() { }); $(document).on('click', '.sortBy', function() { $(this).closest('.btn-group').find('.sort-by').text($(this).text()); - helper.setIndex(INDEX_NAME + $(this).data('index-suffix')).search(); + //helper.setIndex(INDEX_NAME + $(this).data('index-suffix')).search(); return false; }); $(document).on('click', '#input-loop', function() { @@ -374,33 +330,35 @@ $(document).ready(function() { } var query = decodeURIComponent(sURLVariables[0].split('=')[1]); $inputField.val(query); - helper.setQuery(query); + search.setQuery(query); + for (var i = 2; i < sURLVariables.length; i++) { var sParameterName = sURLVariables[i].split('='); var facet = decodeURIComponent(sParameterName[0]); var value = decodeURIComponent(sParameterName[1]); - helper.toggleRefine(facet, value, false); + //helper.toggleRefine(facet, value, false); } // Page has to be set in the end to avoid being overwritten var page = decodeURIComponent(sURLVariables[1].split('=')[1]) - 1; - helper.setCurrentPage(page); + search.setCurrentPage(page); } function setURLParams(state) { - var urlParams = '#'; + var urlParams = '?'; var currentQuery = state.query; urlParams += 'q=' + encodeURIComponent(currentQuery); var currentPage = state.page + 1; urlParams += '&page=' + currentPage; - for (var facetRefine in state.facetsRefinements) { - urlParams += '&' + encodeURIComponent(facetRefine) + '=' + encodeURIComponent(state.facetsRefinements[facetRefine]); - } - for (var disjunctiveFacetrefine in state.disjunctiveFacetsRefinements) { - for (var value in state.disjunctiveFacetsRefinements[disjunctiveFacetrefine]) { - urlParams += '&' + encodeURIComponent(disjunctiveFacetrefine) + '=' + encodeURIComponent(state.disjunctiveFacetsRefinements[disjunctiveFacetrefine][value]); - } - } + + //for (var facetRefine in state.facetsRefinements) { + // urlParams += '&' + encodeURIComponent(facetRefine) + '=' + encodeURIComponent(state.facetsRefinements[facetRefine]); + //} + //for (var disjunctiveFacetrefine in state.disjunctiveFacetsRefinements) { + // for (var value in state.disjunctiveFacetsRefinements[disjunctiveFacetrefine]) { + // urlParams += '&' + encodeURIComponent(disjunctiveFacetrefine) + '=' + encodeURIComponent(state.disjunctiveFacetsRefinements[disjunctiveFacetrefine][value]); + // } + //} location.replace(urlParams); } diff --git a/src/scripts/tutti/4_search.js b/src/scripts/tutti/4_search.js index ce1fcb53..79cb2ab9 100644 --- a/src/scripts/tutti/4_search.js +++ b/src/scripts/tutti/4_search.js @@ -3,15 +3,108 @@ * index and algolia settings are defined in layout.pug */ +var elasticSearcher = (function() { + + var deze = { + + query:"", + url:"", + newhits: [], + terms: {}, + page: 0, + + setQuery: (function(q, _url){ + console.log('setQuery!: ' + q); + deze.query=q; + if (_url !== undefined) { + deze.url=_url; + } + }), + + setCurrentPage: (function(page){ + if(page === undefined){ + return; + } + deze.page = page; + }), + + //result callback + results: (function(content){}), + + //error callback + error: (function(message){ + console.log(message); + }), + + on: (function(type, callback){ + deze[type] = callback; + }), + + //parse the agg stuff + aggs: (function(data){ + return deze.newhits.aggregations; + }), + + addTerm: (function(term, value){ + deze.terms[term] = value; + }), + + //get response from elastic and rebuild json + //so we can be a drop in of angolia + execute: (function(){ + params = { + q: deze.query, + page: deze.page, + }; + //add term filters + Object.assign(params, deze.terms); + + var pstr = jQuery.param( params ); + + var jqxhr = $.getJSON("/api/newsearch" + deze.url + "?"+ pstr, function( data ) { + let hits = data.hits.hits; + var newhits = hits.map(function(hit){ + return hit._source; + }); + + deze.newhits = newhits.slice(0); + //cb(newhits.slice(0)); + deze.results({ + 'count': data.hits.total, + 'hits': newhits.slice(0), + 'took': data.took, + 'page': deze.page, + 'aggs': data.aggregations, + }); + }); + + }) + + }; + + return { + execute: deze.execute, + on: deze.on, + setQuery: deze.setQuery, + setCurrentPage: deze.setCurrentPage, + query: deze.query, + page: deze.page, + addTerm: deze.addTerm, + }; + +})(); + + var elasticSearch = (function($, url) { console.log(url); - return function findMatches(q, cb, async){ - if (!cb) { return; } - $.fn.getSearch(q, cb, async, url); - }; + return function findMatches(q, cb, async){ + if (!cb) { return; } + $.fn.getSearch(q, cb, async, url); + }; }); + (function( $ ){ $.fn.getSearch = function(q, cb, async, url){ @@ -20,7 +113,9 @@ var elasticSearch = (function($, url) { if(url === undefined){ url = ''; } - console.log('searching! '+ url + q); + + console.log('searching! '+ url + q); + $.getJSON("/api/newsearch" + url + "?q=" + q, function( data ) { let hits = data.hits.hits; newhits = hits.map(function(hit){ diff --git a/src/scripts/tutti/7_user_search.js b/src/scripts/tutti/7_user_search.js index db1bba34..466d3fc0 100644 --- a/src/scripts/tutti/7_user_search.js +++ b/src/scripts/tutti/7_user_search.js @@ -1,6 +1,6 @@ (function ( $ ) { // See organizations/view_embed.pug for example use. - $.fn.userSearch = function(algolia_application_id, algolia_public_key, algolia_index_users, on_selected) { + $.fn.userSearch = function(on_selected) { var target = this; this.autocomplete({hint: false}, [ diff --git a/src/templates/organizations/view_embed.pug b/src/templates/organizations/view_embed.pug index d04dccfe..759a444f 100644 --- a/src/templates/organizations/view_embed.pug +++ b/src/templates/organizations/view_embed.pug @@ -203,9 +203,6 @@ h4 Organization members script. $(document).ready(function() { $('#user-select').userSearch( - '{{config.ALGOLIA_USER}}', - '{{config.ALGOLIA_PUBLIC_KEY}}', - '{{config.ALGOLIA_INDEX_USERS}}', function (event, hit, dataset) { var $existing = $('li.sharing-users-item[data-user-id="' + hit.objectID + '"]'); if ($existing.length) { @@ -224,9 +221,6 @@ script. } ); $('#admin-select').userSearch( - '{{config.ALGOLIA_USER}}', - '{{config.ALGOLIA_PUBLIC_KEY}}', - '{{config.ALGOLIA_INDEX_USERS}}', function (event, hit, dataset) { setAdmin(hit.objectID, hit.full_name); } diff --git a/src/templates/projects/sharing.pug b/src/templates/projects/sharing.pug index 3a9087f7..81517316 100644 --- a/src/templates/projects/sharing.pug +++ b/src/templates/projects/sharing.pug @@ -86,12 +86,7 @@ script. addUser(hit.objectID); } - $('#user-select').userSearch( - '{{config.ALGOLIA_USER}}', - '{{config.ALGOLIA_PUBLIC_KEY}}', - '{{config.ALGOLIA_INDEX_USERS}}', - shareWithUser - ); + $('#user-select').userSearch(shareWithUser); }); function addUser(userId){ if (!userId || userId.length == 0) {