Add no-index option to avoid indexing a field

arlas/cli/index.py

@@ -119,6 +119,7 @@ def mapping(
     nb_lines: int = typer.Option(default=2, help="Number of line to consider for generating the mapping. Avoid going over 10."),
     field_mapping: list[str] = typer.Option(default=[], help="Override the mapping with the provided field path/type. Example: fragment.location:geo_point. Important: the full field path must be provided."),
     no_fulltext: list[str] = typer.Option(default=[], help="List of keyword or text fields that should not be in the fulltext search. Important: the field name only must be provided."),
+    no_index: list[str] = typer.Option(default=[], help="List of fields that should not be indexed."),
     push_on: str = typer.Option(default=None, help="Push the generated mapping for the provided index name"),
 ):
     config = variables["arlas"]
@@ -138,7 +139,7 @@ def mapping(
             else:
                 print(f"Error: invalid field_mapping \"{fm}\". The format is \"field:type\" like \"fragment.location:geo_point\"", file=sys.stderr)
                 exit(1)
-    mapping = make_mapping(file=file, nb_lines=nb_lines, types=types, no_fulltext=no_fulltext)
+    mapping = make_mapping(file=file, nb_lines=nb_lines, types=types, no_fulltext=no_fulltext, no_index=no_index)
     if push_on and config:
         Service.create_index(
             config,

arlas/cli/model_infering.py

@@ -141,29 +141,34 @@ def __type_node__(n, name: str = None) -> str:
     return "UNDEFINED"
 
 # from the typed tree, generate the mapping.
-def __generate_mapping__(tree, mapping, no_fulltext: list[str]):
+def __generate_mapping__(tree, mapping, no_fulltext: list[str], no_index: list[str]):
     if type(tree) is dict:
-        for (k, v) in tree.items():
-            if k not in ["__type__", "__values__"]:
-                t: str = v.get("__type__")
-                if t == "object":
-                    mapping[k] = {"properties": {}}
-                    __generate_mapping__(v, mapping[k]["properties"], no_fulltext)
+        for (field_name, v) in tree.items():
+            if field_name not in ["__type__", "__values__"]:
+                field_type: str = v.get("__type__")
+                if field_type == "object":
+                    mapping[field_name] = {"properties": {}}
+                    __generate_mapping__(tree=v, mapping=mapping[field_name]["properties"], no_fulltext=no_fulltext,
+                                         no_index=no_index)
                 else:
-                    if t.startswith("date-"):
+                    if field_type.startswith("date-"):
                         # Dates can have format patterns containing '-'
-                        mapping[k] = {"type": "date", "format": t.split("-", 1)[1]}
+                        mapping[field_name] = {"type": "date", "format": field_type.split("-", 1)[1]}
                     else:
-                        mapping[k] = {"type": t}
-                        if t in ["keyword", "text"]:
-                            print("-->{}".format(k))
-                            if k not in no_fulltext:
-                                mapping[k]["copy_to"] = ["internal.fulltext", "internal.autocomplete"]
+                        mapping[field_name] = {"type": field_type}
+                        if field_type in ["keyword", "text"]:
+                            if field_name not in no_fulltext:
+                                mapping[field_name]["copy_to"] = ["internal.fulltext", "internal.autocomplete"]
+                    # Avoid indexing field if field in --no-index
+                    if field_name in no_index:
+                        mapping[field_name]["index"] = "false"
+                    print(f"-->{field_name}: {mapping[field_name]['type']}")
     else:
         raise Exception("Unexpected state")
 
 
-def make_mapping(file: str, nb_lines: int = 2, types: dict[str, str] = {}, no_fulltext: list[str] = []):
+def make_mapping(file: str, nb_lines: int = 2, types: dict[str, str] = {}, no_fulltext: list[str] = [],
+                 no_index: list[str] = []):
     tree = {}
     mapping = {}
     with open(file) as f:
@@ -176,7 +181,7 @@ def make_mapping(file: str, nb_lines: int = 2, types: dict[str, str] = {}, no_fu
                 hit = json.loads(line)
                 __build_tree__(tree, hit)
         __type_tree__("", tree, types)
-        __generate_mapping__(tree, mapping, no_fulltext)
+        __generate_mapping__(tree, mapping, no_fulltext, no_index)
     mapping["internal"] = {
         "properties": {
             "autocomplete": {

docs/docs/indices.md

@@ -85,15 +85,6 @@ The values of the first lines of the files are used to infer the mapping for eac
 
     Make sure to take enough rows to get all the fields with the option `--nb_lines`
 
-!!! tip
-    The data can be split in different NDJSON files in a folder:
-    ```
-    part-00000-[...].json
-    part-00001-[...].json
-    ...
-    ```
-    In practice, the `file` Argument can be filed with a **pattern** such as `path/to/data.json/part-0000*.json` to reference all the different files.
-
 
 ### Type identification
 
@@ -135,7 +126,7 @@ A **date** is identified as such if
 
 By default, the keywords and text fields are searchable as fulltext to be accessible in the search bar.
 
-!!! note "--no-fulltext "
+!!! note "--no-fulltext"
 
     If searching through a field value is not needed, it can be deactivated.
     That would result in better performances for the fulltext search.
@@ -144,6 +135,17 @@ By default, the keywords and text fields are searchable as fulltext to be access
 
     - `--no-fulltext field_keyword`
 
+!!! note "--no-index"
+    If a field doesn't need to be explored in the dashboard, it should be removed before indexing the data.
+
+    Alternatively, you can explicitly exclude the field from being indexed using the `--no-index` option.
+
+    Example:
+
+    - `--no-index unused_field`
+
+    The field will remain in the data but will not be indexed.
+
 ### Created mapping
 
 By default, the `arlas_cli indices mapping` directly returns the mapping in the command line.
@@ -269,6 +271,15 @@ Example:
    data {index_name} {path/to/data.json}
 ```
 
+!!! tip
+    The data can be split in different NDJSON files in a folder:
+    ```
+    part-00000-[...].json
+    part-00001-[...].json
+    ...
+    ```
+    In practice, the `files` argument can be filed with a **pattern** such as `path/to/data.json/part-0000*.json` to reference all the different files.
+
 !!! warning
     If the index already contains data, the data is added to the index.