feat: personal sort with demo (#174)

A sort option for scripts + a demo in off.html

Part of: #172

README.md

@@ -125,7 +125,7 @@ To run tests without committing:
 pre-commit run
 ```
 
-#### Debugging the backend app 
+#### Debugging the backend app
 To debug the backend app:
 * stop API instance: `docker compose stop api`
 * add a pdb.set_trace() at the point you want,
@@ -147,6 +147,51 @@ You should also import taxonomies:
 
 `make import-taxonomies`
 
+### Using sort script
+
+In your index configuration, you can add scripts, used for personalized sorting.
+
+For example:
+```yaml
+    scripts:
+      personal_score:
+        # see https://www.elastic.co/guide/en/elasticsearch/painless/8.14/index.html
+        lang: painless
+        # the script source, here a trivial example
+        source: |-
+          doc[params["preferred_field"]].size > 0 ? doc[params["preferred_field"]].value : (doc[params["secondary_field"]].size > 0 ? doc[params["secondary_field"]].value : 0)
+        # gives an example of parameters
+        params:
+          preferred_field: "field1"
+          secondary_field: "field2"
+        # more non editable parameters, can be easier than to declare constants in the script
+        static_params:
+          param1 : "foo"
+```
+
+You then have to import this script in your elasticsearch instance, by running:
+
+```bash
+docker compose run --rm api python -m app sync-scripts
+```
+
+You can now use it with the POST API:
+```bash
+curl -X POST http://127.0.0.1:8000/search \
+  -H "Content-type: application/json" \
+  -d '{"q": "", "sort_by": "personal_score", "sort_params": {"preferred_field": "nova_group", "secondary_field": "last_modified_t"}}
+```
+
+Or you can now use it inside a the sort web-component:
+```html
+  <searchalicious-sort auto-refresh>
+    <searchalicious-sort-script script="personal_score" parameters='{"preferred_field": "nova_group", "secondary_field": "last_modified_t"}}'>
+      Personal preferences
+    </searchalicious-sort-script>
+  </searchalicious-sort>
+```
+even better the parameters might be retrieved for local storage.
+
 
 ## Thank you to our sponsors !
 

app/_types.py

@@ -289,7 +289,8 @@ def sort_by_scripts_needs_params(self):
                 raise ValueError("`sort_params` must be a dict")
             # verifies keys are those expected
             request_keys = set(self.sort_params.keys())
-            expected_keys = set(self.index_config.scripts[self.sort_by].params.keys())
+            sort_sign, sort_by = self.sign_sort_by
+            expected_keys = set(self.index_config.scripts[sort_by].params.keys())
             if request_keys != expected_keys:
                 missing = expected_keys - request_keys
                 missing_str = ("missing keys: " + ", ".join(missing)) if missing else ""

app/config.py

@@ -295,15 +295,22 @@ class ScriptConfig(BaseModel):
     ]
     params: (
         Annotated[
-            # FIXME: not sure of the type here
             dict[str, Any],
             Field(
                 description="Params for the scripts. We need this to retrieve and validate parameters"
             ),
         ]
         | None
     )
-    # TODO: do we want to add a list of mandatory parameters ?
+    static_params: (
+        Annotated[
+            dict[str, Any],
+            Field(
+                description="Additional params for the scripts that can't be supplied by the API (constants)"
+            ),
+        ]
+        | None
+    )
     # Or some type checking/transformation ?
 
 

app/es_scripts.py

@@ -1,8 +1,12 @@
 """Module to manage ES scripts that can be used for personalized sorting
 """
 
+import elasticsearch
+
 from app import config
-from app.utils import connection
+from app.utils import connection, get_logger
+
+logger = get_logger(__name__)
 
 
 def get_script_prefix(index_id: str):
@@ -35,10 +39,11 @@ def _store_script(
     script_id: str, script: config.ScriptConfig, index_config: config.IndexConfig
 ):
     """Store a script in Elasticsearch."""
+    params = dict((script.params or {}), **(script.static_params or {}))
     payload = {
         "lang": script.lang.value,
         "source": script.source,
-        "params": script.params,
+        "params": params,
     }
     # hardcode context to scoring for the moment
     context = "score"
@@ -53,7 +58,17 @@ def sync_scripts(index_id: str, index_config: config.IndexConfig) -> dict[str, i
     # remove them
     _remove_scripts(current_ids, index_config)
     # store scripts
+    stored_scripts = 0
     if index_config.scripts:
         for script_id, script in index_config.scripts.items():
-            _store_script(get_script_id(index_id, script_id), script, index_config)
-    return {"removed": len(current_ids), "added": len(index_config.scripts or [])}
+            try:
+                _store_script(get_script_id(index_id, script_id), script, index_config)
+                stored_scripts += 1
+            except elasticsearch.ApiError as e:
+                logger.error(
+                    "Unable to store script %s, got exception %s: %s",
+                    script_id,
+                    e,
+                    e.body,
+                )
+    return {"removed": len(current_ids), "added": stored_scripts}

app/query.py

@@ -1,4 +1,5 @@
 import elastic_transport
+import elasticsearch
 import luqum.exceptions
 from elasticsearch_dsl import A, Q, Search
 from elasticsearch_dsl.aggs import Agg
@@ -265,12 +266,14 @@ def parse_sort_by_script(
     if script is None:
         raise ValueError(f"Unknown script '{sort_by}'")
     script_id = get_script_id(index_id, sort_by)
+    # join params and static params
+    script_params = dict((params or {}), **(script.static_params or {}))
     return {
         "_script": {
             "type": "number",
             "script": {
                 "id": script_id,
-                "params": params,
+                "params": script_params,
             },
             "order": "desc" if operator == "-" else "asc",
         }
@@ -402,6 +405,10 @@ def execute_query(
     debug = SearchResponseDebug(query=query.to_dict())
     try:
         results = query.execute()
+    except elasticsearch.ApiError as e:
+        logger.error("Error while running query: %s %s", str(e), str(e.body))
+        errors.append(SearchResponseError(title="es_api_error", description=str(e)))
+        return ErrorSearchResponse(debug=debug, errors=errors)
     except elastic_transport.ConnectionError as e:
         errors.append(
             SearchResponseError(title="es_connection_error", description=str(e))

data/config/openfoodfacts.yml

@@ -8,11 +8,24 @@ indices:
       number_of_shards: 4
     scripts:
       personal_score:
-        lang: expression
-        source: |
-          1
+        # see https://www.elastic.co/guide/en/elasticsearch/painless/8.14/index.html
+        lang: painless
+        source: |-
+          String nova_index = (doc['nova_group'].size() != 0) ? doc['nova_group'].value.toString() : "unknown";
+          String nutri_index = (doc['nutriscore_grade'].size() != 0) ? doc['nutriscore_grade'].value : 'e';
+          String eco_index = (doc['ecoscore_grade'].size() != 0) ? doc['ecoscore_grade'].value : 'e';
+          return (
+            params['nova_to_score'].getOrDefault(nova_index, 0) * params['nova_group']
+            + params['grades_to_score'].getOrDefault(nutri_index, 0) * params['nutri_score']
+            + params['grades_to_score'].getOrDefault(eco_index, 0) * params['eco_score']
+          );
         params:
-          preferences: 2
+          eco_score: 1
+          nutri_score: 1
+          nova_group: 1
+        static_params:
+          nova_to_score: {"1": 100, "2": 100, "3": 75, "4": 0, "unknown": 0}
+          grades_to_score: {"a": 100, "b": 75, "c": 50, "d": 25, "e": 0, "unknown": 0, "not-applicable": 0}
     fields:
       code:
         required: true

frontend/README.md

@@ -28,6 +28,10 @@ The project is currently composed of several widgets.
   * you must add searchalicious-sort-field elements inside to add sort options
     * with a field= to indicate the field
     * the label is the text inside the element
+  * or a searchalicious-sort-script
+    * with a script= to indicate a script
+    * and a params= which is a either a json encoded object, 
+      or a key in localStorage prefixed with "local:"
   * you can add element to slot `label` to change the label
 
 **IMPORTANT:**

frontend/public/off.html

@@ -158,14 +158,14 @@
          * As document loads, load preferences from local storage,
          * and update preferences form
          **/
-        document.onload = function() {
+        document.addEventListener('DOMContentLoaded', function() {
           try {
             loadPreferences();
             updateForm(preferences);
           } catch (e) {
             console.log(e);
           }
-        }
+        });
 
         /**
          * load preferences from local storage
@@ -184,12 +184,14 @@
             const element = preferences_form_elements[i];
             if (element.nodeName === "SELECT") {
               const targetValue = preferences[element.name] ?? element.value;
-              const options = Array.from(element.getElementsByTagName("option"));
-              options.forEach(item => {
-                if (item.value === targetValue) {
-                  item.selected = true;
-                }
-              });
+              if (targetValue != null) {
+                const options = Array.from(element.getElementsByTagName("option"));
+                options.forEach(item => {
+                  if (item.value.toString() === targetValue.toString()) {
+                    item.selected = true;
+                  }
+                });
+              }
             }
           }
         }
@@ -221,7 +223,7 @@
             for (let i = 0; i < preferences_form_elements.length; i++) {
               const preference = preferences_form_elements[i];
               if (preference.nodeName === "SELECT") {
-                preferences[preference.name] = preference.value;
+                preferences[preference.name] = parseInt(preference.value);
               }
             }
             localStorage.setItem("off_preferences", JSON.stringify(preferences));
@@ -301,6 +303,7 @@
                             <searchalicious-sort-field field="nutriscore_grade">Products with the best Nutri-Score</searchalicious-sort-field>
                             <searchalicious-sort-field field="-created_t">Recently added products</searchalicious-sort-field>
                             <searchalicious-sort-field field="-last_modified_t">Recently modified products</searchalicious-sort-field>
+                            <searchalicious-sort-script script="-personal_score" parameters="local:off_preferences">Personal preferences</searchalicious-sort-script>
                           </searchalicious-sort>  
                         </div>
                       </div>
@@ -316,20 +319,20 @@
                     <div id="preferences_config" style="display:none">
                       <form id="preferences_form" onSubmit="return storePreferences()">
                         <div><a onclick="return togglePreferences(false)">Select your Food preferences :</a></div>
-                        <label for="nutriscore">Nutriscore</label>
-                        <select name="nutriscore">
+                        <label for="nutri_score">Nutriscore</label>
+                        <select name="nutri_score">
                           <option value="0">not important</option>
                           <option value="1">important</option>
                           <option value="2">very important</option>
                         </select>
-                        <label for="nova">Nova</label>
-                        <select name="nova">
+                        <label for="nova_group">Nova</label>
+                        <select name="nova_group">
                           <option value="0">not important</option>
                           <option value="1">important</option>
                           <option value="2">very important</option>
                         </select>
-                        <label for="ecoscore">Environmental impact</label>
-                        <select name="ecoscore">
+                        <label for="eco_score">Environmental impact</label>
+                        <select name="eco_score">
                           <option value="0">not important</option>
                           <option value="1">important</option>
                           <option value="2">very important</option>

frontend/src/mixins/history.ts

@@ -5,7 +5,7 @@ import {
   removeParenthesis,
 } from '../utils/url';
 import {isNullOrUndefined} from '../utils';
-import {BuildParamsOutput} from './search-ctl';
+import {SearchParameters} from './search-ctl';
 import {property} from 'lit/decorators.js';
 import {QueryOperator} from '../utils/enums';
 import {SearchaliciousSort} from '../search-sort';
@@ -21,7 +21,7 @@ export type SearchaliciousHistoryInterface = {
   _sortElement: () => SearchaliciousSort | null;
   convertHistoryParamsToValues: (params: URLSearchParams) => HistoryOutput;
   setValuesFromHistory: (values: HistoryOutput) => void;
-  buildHistoryParams: (params: BuildParamsOutput) => HistoryParams;
+  buildHistoryParams: (params: SearchParameters) => HistoryParams;
   setParamFromUrl: () => {launchSearch: boolean; values: HistoryOutput};
 };
 
@@ -169,7 +169,7 @@ export const SearchaliciousHistoryMixin = <T extends Constructor<LitElement>>(
      * It will be used to update the URL when searching
      * @param params
      */
-    buildHistoryParams = (params: BuildParamsOutput) => {
+    buildHistoryParams = (params: SearchParameters) => {
       const urlParams: Record<string, string | undefined | null> = {
         [HistorySearchParams.QUERY]: this.query,
         [HistorySearchParams.SORT_BY]: this._sortElement()?.getSortOptionId(),