From 4484d51bc520e57d42265a95ef9db19356161353 Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Fri, 21 Jun 2024 17:19:49 +0200 Subject: [PATCH] feat: personal sort with demo (#174) A sort option for scripts + a demo in off.html Part of: https://github.com/openfoodfacts/search-a-licious/issues/172 --- README.md | 47 +++++++++- app/_types.py | 3 +- app/config.py | 11 ++- app/es_scripts.py | 23 ++++- app/query.py | 9 +- data/config/openfoodfacts.yml | 21 ++++- frontend/README.md | 4 + frontend/public/off.html | 33 +++---- frontend/src/mixins/history.ts | 6 +- frontend/src/mixins/search-ctl.ts | 106 +++++++++++++++-------- frontend/src/search-a-licious.ts | 1 + frontend/src/search-sort-script.ts | 55 ++++++++++++ frontend/src/search-sort.ts | 24 +++-- frontend/src/test/search-bar_test.ts | 12 ++- frontend/src/utils/constants.ts | 5 +- tests/unit/data/openfoodfacts_config.yml | 21 ++++- 16 files changed, 300 insertions(+), 81 deletions(-) create mode 100644 frontend/src/search-sort-script.ts diff --git a/README.md b/README.md index 5a01576b..8f83a2bc 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ To run tests without committing: pre-commit run ``` -#### Debugging the backend app +#### Debugging the backend app To debug the backend app: * stop API instance: `docker compose stop api` * add a pdb.set_trace() at the point you want, @@ -147,6 +147,51 @@ You should also import taxonomies: `make import-taxonomies` +### Using sort script + +In your index configuration, you can add scripts, used for personalized sorting. + +For example: +```yaml + scripts: + personal_score: + # see https://www.elastic.co/guide/en/elasticsearch/painless/8.14/index.html + lang: painless + # the script source, here a trivial example + source: |- + doc[params["preferred_field"]].size > 0 ? doc[params["preferred_field"]].value : (doc[params["secondary_field"]].size > 0 ? doc[params["secondary_field"]].value : 0) + # gives an example of parameters + params: + preferred_field: "field1" + secondary_field: "field2" + # more non editable parameters, can be easier than to declare constants in the script + static_params: + param1 : "foo" +``` + +You then have to import this script in your elasticsearch instance, by running: + +```bash +docker compose run --rm api python -m app sync-scripts +``` + +You can now use it with the POST API: +```bash +curl -X POST http://127.0.0.1:8000/search \ + -H "Content-type: application/json" \ + -d '{"q": "", "sort_by": "personal_score", "sort_params": {"preferred_field": "nova_group", "secondary_field": "last_modified_t"}} +``` + +Or you can now use it inside a the sort web-component: +```html + + + Personal preferences + + +``` +even better the parameters might be retrieved for local storage. + ## Thank you to our sponsors ! diff --git a/app/_types.py b/app/_types.py index 8195883d..6a6f1909 100644 --- a/app/_types.py +++ b/app/_types.py @@ -289,7 +289,8 @@ def sort_by_scripts_needs_params(self): raise ValueError("`sort_params` must be a dict") # verifies keys are those expected request_keys = set(self.sort_params.keys()) - expected_keys = set(self.index_config.scripts[self.sort_by].params.keys()) + sort_sign, sort_by = self.sign_sort_by + expected_keys = set(self.index_config.scripts[sort_by].params.keys()) if request_keys != expected_keys: missing = expected_keys - request_keys missing_str = ("missing keys: " + ", ".join(missing)) if missing else "" diff --git a/app/config.py b/app/config.py index 9049c22f..5318bc9b 100644 --- a/app/config.py +++ b/app/config.py @@ -295,7 +295,6 @@ class ScriptConfig(BaseModel): ] params: ( Annotated[ - # FIXME: not sure of the type here dict[str, Any], Field( description="Params for the scripts. We need this to retrieve and validate parameters" @@ -303,7 +302,15 @@ class ScriptConfig(BaseModel): ] | None ) - # TODO: do we want to add a list of mandatory parameters ? + static_params: ( + Annotated[ + dict[str, Any], + Field( + description="Additional params for the scripts that can't be supplied by the API (constants)" + ), + ] + | None + ) # Or some type checking/transformation ? diff --git a/app/es_scripts.py b/app/es_scripts.py index 6780894c..722c33ee 100644 --- a/app/es_scripts.py +++ b/app/es_scripts.py @@ -1,8 +1,12 @@ """Module to manage ES scripts that can be used for personalized sorting """ +import elasticsearch + from app import config -from app.utils import connection +from app.utils import connection, get_logger + +logger = get_logger(__name__) def get_script_prefix(index_id: str): @@ -35,10 +39,11 @@ def _store_script( script_id: str, script: config.ScriptConfig, index_config: config.IndexConfig ): """Store a script in Elasticsearch.""" + params = dict((script.params or {}), **(script.static_params or {})) payload = { "lang": script.lang.value, "source": script.source, - "params": script.params, + "params": params, } # hardcode context to scoring for the moment context = "score" @@ -53,7 +58,17 @@ def sync_scripts(index_id: str, index_config: config.IndexConfig) -> dict[str, i # remove them _remove_scripts(current_ids, index_config) # store scripts + stored_scripts = 0 if index_config.scripts: for script_id, script in index_config.scripts.items(): - _store_script(get_script_id(index_id, script_id), script, index_config) - return {"removed": len(current_ids), "added": len(index_config.scripts or [])} + try: + _store_script(get_script_id(index_id, script_id), script, index_config) + stored_scripts += 1 + except elasticsearch.ApiError as e: + logger.error( + "Unable to store script %s, got exception %s: %s", + script_id, + e, + e.body, + ) + return {"removed": len(current_ids), "added": stored_scripts} diff --git a/app/query.py b/app/query.py index 97c67f63..eaec2d43 100644 --- a/app/query.py +++ b/app/query.py @@ -1,4 +1,5 @@ import elastic_transport +import elasticsearch import luqum.exceptions from elasticsearch_dsl import A, Q, Search from elasticsearch_dsl.aggs import Agg @@ -265,12 +266,14 @@ def parse_sort_by_script( if script is None: raise ValueError(f"Unknown script '{sort_by}'") script_id = get_script_id(index_id, sort_by) + # join params and static params + script_params = dict((params or {}), **(script.static_params or {})) return { "_script": { "type": "number", "script": { "id": script_id, - "params": params, + "params": script_params, }, "order": "desc" if operator == "-" else "asc", } @@ -402,6 +405,10 @@ def execute_query( debug = SearchResponseDebug(query=query.to_dict()) try: results = query.execute() + except elasticsearch.ApiError as e: + logger.error("Error while running query: %s %s", str(e), str(e.body)) + errors.append(SearchResponseError(title="es_api_error", description=str(e))) + return ErrorSearchResponse(debug=debug, errors=errors) except elastic_transport.ConnectionError as e: errors.append( SearchResponseError(title="es_connection_error", description=str(e)) diff --git a/data/config/openfoodfacts.yml b/data/config/openfoodfacts.yml index c22741af..6981acb1 100644 --- a/data/config/openfoodfacts.yml +++ b/data/config/openfoodfacts.yml @@ -8,11 +8,24 @@ indices: number_of_shards: 4 scripts: personal_score: - lang: expression - source: | - 1 + # see https://www.elastic.co/guide/en/elasticsearch/painless/8.14/index.html + lang: painless + source: |- + String nova_index = (doc['nova_group'].size() != 0) ? doc['nova_group'].value.toString() : "unknown"; + String nutri_index = (doc['nutriscore_grade'].size() != 0) ? doc['nutriscore_grade'].value : 'e'; + String eco_index = (doc['ecoscore_grade'].size() != 0) ? doc['ecoscore_grade'].value : 'e'; + return ( + params['nova_to_score'].getOrDefault(nova_index, 0) * params['nova_group'] + + params['grades_to_score'].getOrDefault(nutri_index, 0) * params['nutri_score'] + + params['grades_to_score'].getOrDefault(eco_index, 0) * params['eco_score'] + ); params: - preferences: 2 + eco_score: 1 + nutri_score: 1 + nova_group: 1 + static_params: + nova_to_score: {"1": 100, "2": 100, "3": 75, "4": 0, "unknown": 0} + grades_to_score: {"a": 100, "b": 75, "c": 50, "d": 25, "e": 0, "unknown": 0, "not-applicable": 0} fields: code: required: true diff --git a/frontend/README.md b/frontend/README.md index e3b87d4f..9243c154 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -28,6 +28,10 @@ The project is currently composed of several widgets. * you must add searchalicious-sort-field elements inside to add sort options * with a field= to indicate the field * the label is the text inside the element + * or a searchalicious-sort-script + * with a script= to indicate a script + * and a params= which is a either a json encoded object, + or a key in localStorage prefixed with "local:" * you can add element to slot `label` to change the label **IMPORTANT:** diff --git a/frontend/public/off.html b/frontend/public/off.html index 8c41aa4a..14792410 100644 --- a/frontend/public/off.html +++ b/frontend/public/off.html @@ -158,14 +158,14 @@ * As document loads, load preferences from local storage, * and update preferences form **/ - document.onload = function() { + document.addEventListener('DOMContentLoaded', function() { try { loadPreferences(); updateForm(preferences); } catch (e) { console.log(e); } - } + }); /** * load preferences from local storage @@ -184,12 +184,14 @@ const element = preferences_form_elements[i]; if (element.nodeName === "SELECT") { const targetValue = preferences[element.name] ?? element.value; - const options = Array.from(element.getElementsByTagName("option")); - options.forEach(item => { - if (item.value === targetValue) { - item.selected = true; - } - }); + if (targetValue != null) { + const options = Array.from(element.getElementsByTagName("option")); + options.forEach(item => { + if (item.value.toString() === targetValue.toString()) { + item.selected = true; + } + }); + } } } } @@ -221,7 +223,7 @@ for (let i = 0; i < preferences_form_elements.length; i++) { const preference = preferences_form_elements[i]; if (preference.nodeName === "SELECT") { - preferences[preference.name] = preference.value; + preferences[preference.name] = parseInt(preference.value); } } localStorage.setItem("off_preferences", JSON.stringify(preferences)); @@ -301,6 +303,7 @@ Products with the best Nutri-Score Recently added products Recently modified products + Personal preferences @@ -316,20 +319,20 @@