From 8c96e838d3df02bde9a65daa02989e939a2bc8d0 Mon Sep 17 00:00:00 2001 From: Willi Date: Fri, 4 Oct 2024 18:32:25 +0200 Subject: [PATCH] WIP --- arlas/cli/index.py | 6 +- docs/docs/configuration.md | 127 +++++++++++++ docs/docs/confs.md | 1 + docs/docs/iam.md | 2 + docs/docs/indices.md | 335 +++++++++++++++++++++++++++++++-- docs/docs/install.md | 30 ++- docs/docs/persist.md | 77 ++++++++ docs/docs/started.md | 375 ++++++++++++++++++------------------- docs/mkdocs.yml | 15 +- 9 files changed, 748 insertions(+), 220 deletions(-) create mode 100644 docs/docs/configuration.md diff --git a/arlas/cli/index.py b/arlas/cli/index.py index e0dc652..5fbe572 100644 --- a/arlas/cli/index.py +++ b/arlas/cli/index.py @@ -73,7 +73,7 @@ def create( @indices.command(help="Index data") def data( index: str = typer.Argument(help="index's name"), - files: list[str] = typer.Argument(help="List of pathes to the file conaining the data. Format: NDJSON"), + files: list[str] = typer.Argument(help="List of paths to the file(s) containing the data. Format: NDJSON"), bulk: int = typer.Option(default=100, help="Bulk size for indexing data") ): config = variables["arlas"] @@ -89,9 +89,9 @@ def data( @indices.command(help="Generate the mapping based on the data") def mapping( - file: str = typer.Argument(help="Path to the file conaining the data. Format: NDJSON"), + file: str = typer.Argument(help="Path to the file containing the data. Format: NDJSON"), nb_lines: int = typer.Option(default=2, help="Number of line to consider for generating the mapping. Avoid going over 10."), - field_mapping: list[str] = typer.Option(default=[], help="Overide the mapping with the provided field path/type. Example: fragment.location:geo_point. Important: the full field path must be provided."), + field_mapping: list[str] = typer.Option(default=[], help="Override the mapping with the provided field path/type. Example: fragment.location:geo_point. Important: the full field path must be provided."), no_fulltext: list[str] = typer.Option(default=[], help="List of keyword or text fields that should not be in the fulltext search. Important: the field name only must be provided."), push_on: str = typer.Option(default=None, help="Push the generated mapping for the provided index name"), ): diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md new file mode 100644 index 0000000..de06588 --- /dev/null +++ b/docs/docs/configuration.md @@ -0,0 +1,127 @@ +# Configuration + + +## Initial configuration +`arlas_cli` uses a yaml file for storing various ARLAS and elasticsearch configurations. + +[One is automatically created for your convenience at the first launch](https://raw.githubusercontent.com/gisaia/arlas-cli/master/configuration.yaml). + + +```shell +> arlas_cli --version +X.X.X +Warning : no configuration file found, we created an empty one for +you (~/.arlas/cli/configuration.yaml). +``` + +By default, the file is located in `~/.arlas/cli/configuration.yaml`. +It contains the ARLAS demo endpoint and the local ARLAS and elasticsearch endpoints. + +The configuration can also contain references to collection models for creating collections. A default one is provided for ARLAS EO. A reference can be an http url or a path to a local file. +It can also contain references to index mappings for creating indices. A default one is provided for ARLAS EO. + +## Configurations + +The command line uses the `${HOME}/.arlas/cli/configuration.yaml` configuration file: + +```yaml +arlas: + local: + allow_delete: true + elastic: + headers: + Content-Type: application/json + location: http://localhost:9200 + server: + headers: + Content-Type: application/json + location: http://localhost:9999/arlas +mappings: + arlas_eo: + headers: null + location: https://raw.githubusercontent.com/gisaia/ARLAS-EO/master/mapping.json +models: + arlas_eo: + headers: null + location: https://raw.githubusercontent.com/gisaia/ARLAS-EO/master/collection.json +``` + +The `arlas` section contains the different deployment configurations. The mapping section lists the mapping template that you can use. + Finally, the models are the templates for the collections. A [detailed description](model/README.md) of the configuration structure is provided. + +### Create, describe and delete a configuration for `arlas_cli` + +You can edit directly the `${HOME}/.arlas/cli/configuration.yaml` configuration file to update your configurations. You can also use the command line itself. + +To list the configurations: + +```shell +> arlas_cli confs list ++-----------+-----------------------------+ +| name | url | ++-----------+-----------------------------+ +| local | http://localhost:9999/arlas | +| test_conf | http://localhost:9999 | ++-----------+-----------------------------+ +``` + +To describe a configuration: + + +```shell +arlas_cli confs describe local +allow_delete: true +authorization: null +elastic: + headers: + Content-Type: application/json + location: http://localhost:9200 +server: + headers: + Content-Type: application/json + location: http://localhost:9999/arlas +``` + +To create a simple configuration: + + +```shell +> arlas_cli confs create dev_conf \ + --server http://localhost:9999 \ + --headers "Content-Type:application/json" \ + --elastic http://localhost:9200 \ + --elastic-headers "Content-Type:application/json" \ + --no-allow-delete +``` + +For an arlas configuration with authentication: + + +```shell +> arlas_cli --config-file /tmp/configuration.yaml confs \ + create myarlas_as_user \ + --server http://myserver/arlas \ + --headers "arlas-org-filter:my_org_name" \ + --headers "Content-Type:application/json" \ + --no-allow-delete \ + --auth-token-url http://myserver/arlas_iam_server/session \ + --auth-login user \ + --auth-password my_password \ + --auth-headers "Content-Type:application/json;charset=utf-8"\ + --auth-arlas-iam +``` + +To delete the configuration: + + +```shell +> arlas_cli confs delete dev_conf +``` + +Also, it is possible to use a different configuration file than the one placed in your home directory (`$HOME/.arlas/cli/configuration.yaml`): + + +```shell +> arlas_cli --config-file /tmp/config.yaml +Warning : no configuration file found, we created an empty one for you (/tmp/config.yaml). +``` diff --git a/docs/docs/confs.md b/docs/docs/confs.md index 97f4b46..55f23af 100644 --- a/docs/docs/confs.md +++ b/docs/docs/confs.md @@ -1,3 +1,4 @@ +# Configurations !!! tip At its first launch, `arlas_cli` will create a first configuration file for you (`$HOME/.arlas/cli/configuration.yaml`), with two ARLAS configurations, one pointing at a local deployment, one on ARLAS demo (without elasticsearch). diff --git a/docs/docs/iam.md b/docs/docs/iam.md index 9f97e9a..675bf46 100644 --- a/docs/docs/iam.md +++ b/docs/docs/iam.md @@ -1,3 +1,5 @@ +# Identity & Access Management + ## Manage organisations, users, permissions and groups diff --git a/docs/docs/indices.md b/docs/docs/indices.md index 18708d1..f8cd9a4 100644 --- a/docs/docs/indices.md +++ b/docs/docs/indices.md @@ -1,4 +1,14 @@ -## List index management commands +# Indices + +## Elasticsearch index + +ARLAS rely on [elasticsearch](https://www.elastic.co/fr/elasticsearch), an open-source search engine to perform quick aggregation requests over data. + +To be explored in ARLAS dashboards, the data has to be indexed in an ES index. An index contains the data and a mapping to describe how fields have to be interpreted (types). + +`arlas_cli` provide tools to infer mapping from data and manage the ES indices with the `indices` subcommand. + +**List index management commands** ```shell @@ -9,7 +19,7 @@ ╭─ Options ──────────────────────────────────────────────────────────╮ │ * --config TEXT Name of the ARLAS configuration to use │ │ from your configuration file │ -│ (/Users/gaudan/.arlas/cli/configuration.… │ +│ (~/.arlas/cli/configuration.yaml │ │ [default: None] │ │ [required] │ │ --help Show this message and exit. │ @@ -26,10 +36,9 @@ ``` -## Generate mapping from a data file +## mapping - -To generate a mapping, you need to provide a NDJSON file (New line delimiter JSON). The first N lines are used for infering the mapping. If the mapping is wrong, you can overwrite the typing with the `--field-mapping` option. Once you're happy with the mapping, you can directly push it on elasticsearch (`--push-on`). +arlas_cli provide tools to infer the ES mapping directly from a data file. ```shell @@ -40,7 +49,7 @@ To generate a mapping, you need to provide a NDJSON file (New line delimiter JSO Generate the mapping based on the data ╭─ Arguments ────────────────────────────────────────────────────────╮ -│ * file TEXT Path to the file conaining the data. Format: │ +│ * file TEXT Path to the file containing the data. Format: │ │ NDJSON │ │ [default: None] │ │ [required] │ @@ -64,10 +73,95 @@ To generate a mapping, you need to provide a NDJSON file (New line delimiter JSO │ [default: None] │ │ --help Show this message and exit. │ ╰────────────────────────────────────────────────────────────────────╯ - ``` -## Create an index with its mapping +### Data file + +To generate a mapping, you need to provide a NDJSON `file` (New line delimiter JSON). + +The values of the first lines of the files are used for inferring the mapping for each field of the data. + +!!! note "--nb-lines" + + The default number of lines is 2, and going over 10 should be avoided + + +### Type identification + +A **geometry** is identified as such if + +- it is a geojson +- it is a WKT string +- the field name contains `geohash` +- it is a string containing two float separated by a comma + +A **date** is identified as such if + +- its name is one of `timestamp`, `date`, `start` or `end` and that it can be parsed as a date +- its name contains `timestamp`, `date`, `start` or `end` and its values are number within [631152000, 4102444800] or [631152000000, 4102444800000] (year 1990 to 2100) + +!!! note "--field-mapping" + If the mapping is wrong, you can overwrite the typing with the `--field-mapping` option. + + It has the structure **field_name:field_type** + + Examples: + + - `--field-mapping field_point:geo_point` + - `--field-mapping field_time:date-epoch_second` + +By default, the keywords and text fields are searchable as fulltext to be accessible in the searchbar. + +!!! note "--no-fulltext " + + If it is not required for a given field, it is more optimal to deactivate fulltext search. + + Example: + + - `--no-fulltext field_keyword` + +### Created mapping + +By default, the `arlas_cli indices mapping` directly returns the mapping in the command line. + +Once you're happy with the mapping, you can either store it in a file or directly push it on elasticsearch. + +!!! tip "Store mapping in a file" + To store the created mapping in a `mapping.json` file, simply use `>` as the end of your command. + + Example: + + ```shell + > arlas_cli indices \ + --config local \ + mapping data_sample.json \ + --field-mapping timestamps.start:date-epoch_second \ + --field-mapping timestamps.end:date-epoch_second \ + > path/to/mapping.json + ``` + +!!! note "--push-on" + + To push the inferred mapping directly in an elasticsearch index, use the `--push-on` option with the target index name. + + Example: + + ```shell + > arlas_cli indices \ + --config local \ + mapping data_sample.json \ + --field-mapping timestamps.start:date-epoch_second \ + --field-mapping timestamps.end:date-epoch_second \ + --push-on index_name + ``` + + The index is then created and the [index creation command](#create) can be skipped. + +## create + +Before putting the data in an elasticsearch index, the index has to be initialised with the [correct mapping](#mapping). + +The `indices create` sub-function create the index from a mapping json file. ```shell @@ -89,12 +183,30 @@ To generate a mapping, you need to provide a NDJSON file (New line delimiter JSO │ [default: 1] │ │ --help Show this message and exit. │ ╰────────────────────────────────────────────────────────────────────╯ +``` + +### Create an ES index with its mapping + +The index name and the path to the mapping json file have to be used to create the ES index. +The `indices create` sub-function create the index from a mapping json file. + +Example: + +```shell +> arlas_cli indices \ + --config local \ + create index_name \ + --mapping path/to/mapping.json ``` -## Index data +Once the index is created, it is ready to [ingest data](#data). + +## data -For indexing data, you'll need to provide one ore several NDJSON file(s). Indexing uses bulks for optimal performances. +To explore data in ARLAS, it has to be indexed in the created ES index. + +The `indices data` sub-function ingest the data in a given index. ```shell @@ -106,7 +218,7 @@ For indexing data, you'll need to provide one ore several NDJSON file(s). Indexi ╭─ Arguments ────────────────────────────────────────────────────────╮ │ * index TEXT index's name [default: None] [required] │ -│ * files FILES... List of pathes to the file conaining the │ +│ * files FILES... List of paths to the file containing the │ │ data. Format: NDJSON │ │ [default: None] │ │ [required] │ @@ -117,3 +229,204 @@ For indexing data, you'll need to provide one ore several NDJSON file(s). Indexi ╰────────────────────────────────────────────────────────────────────╯ ``` +### Ingest data + +For indexing data, you'll need to provide one or several NDJSON (New line delimiter JSON) file(s). +Indexing uses bulks for optimal performances. + +Example: +```shell +> arlas_cli indices \ + --config local \ + data index_name path/to/data.json +``` + +!!! warning + If the index already contains data, the data is added to the index. + + To reindex the same data, delete the index, and do not forget to recreate it with the correct mapping before ingesting the data + +!!! note "--bulk" + Indexing uses bulks for optimal performances. + + The default size of bulk is 100 and can be changed with the `--bulk` option + +## list + +To list the available ES indices, simply use the `indices list` sub-function. No arguments are required. + + +```shell +> arlas_cli indices --config local list --help + + Usage: python -m arlas.cli.cli indices help [OPTIONS] + + List indices + +╭─ Options ──────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰────────────────────────────────────────────────────────────────────╯ + +``` + +### List available ES indices + +It displays for each ES index its status, the number of elements it contains and the size of the index. + +Example: + +```shell +> arlas_cli indices --config local list ++--------------+--------+-------+--------+ +| name | status | count | size | ++--------------+--------+-------+--------+ +| .arlas | open | 4 | 11.9kb | +| index_name | open | 100 | 1mb | ++--------------+--------+-------+--------+ +``` + +## describe + +Once the index is created, the description of the fields it contains (corresponding to the mapping) can be displayed with the `indices describe` sub function: + + +```shell +> arlas_cli indices --config local data --help + + Usage: python -m arlas.cli.cli indices describe [OPTIONS] INDEX + + Describe an index + +╭─ Arguments ────────────────────────────────────────────────────────╮ +│ * index TEXT index's name [default: None] [required] │ +╰────────────────────────────────────────────────────────────────────╯ +╭─ Options ──────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰────────────────────────────────────────────────────────────────────╯ + +``` + +### Describe the index mapping + +For a given index, the description of its fields and their type can be displayed. + +For example: + +```shell +> arlas_cli indices --config local describe index_name ++------------------+-----------+ +| field name | type | ++------------------+-----------+ +| field_keyword | keyword | +| field_point | geo_point | +| field_long | long | +| field_shape | geo_shape | +| field_double | double | +| field_date | date | +| field_text | text | +| field_object | object | +| field_boolean | boolean | ++------------------+-----------+ + +``` + +## sample + +The first rows of the data contained in an index can be displayed with the `indices sample` sub function. + + +```shell +> arlas_cli indices --config local delete --help + + Usage: python -m arlas.cli.cli indices sample [OPTIONS] INDEX + + Display a sample of an index + +╭─ Arguments ────────────────────────────────────────────────────────╮ +│ * index TEXT index's name [default: None] [required] │ +╰────────────────────────────────────────────────────────────────────╯ +╭─ Options ──────────────────────────────────────────────────────────╮ +│ --pretty / --no-pretty [default: pretty] │ +│ --size INTEGER [default: 100] │ +│ --help Show this message and exit. │ +╰────────────────────────────────────────────────────────────────────╯ + +``` + +### Visualize few rows of your dataset + +For a given index `index_name`, the first rows of data can be displayed as a valid json dictionary. + +!!! note --size + The number of rows to display (default 100) can be changed + + Example: + + ```shell + > arlas_cli indices --config local sample index_name --size 10 + ``` + +By default, the json representation of the data is pretty printed (clear indentation and one line per field) + +!!! note --no-pretty + The pretty printing can be deactivated and data is displayed in a compact way + + Example: + + ```shell + > arlas_cli indices --config local sample index_name --no-pretty + ``` + +## delete + +The ES index can be deleted with `indices delete` sub command to free space on ES cluster. + + +```shell +> arlas_cli indices --config local delete --help + + Usage: python -m arlas.cli.cli indices delete [OPTIONS] INDEX + + Delete an index + +╭─ Arguments ────────────────────────────────────────────────────────╮ +│ * index TEXT index's name [default: None] [required] │ +╰────────────────────────────────────────────────────────────────────╯ +╭─ Options ──────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰────────────────────────────────────────────────────────────────────╯ + +``` + +### Delete an ES index + +To delete an ES index `index_name` on `local` configuration, run the following command: + + +```shell +> arlas_cli indices --config local delete index_name +``` +!!! warning + Delete an index is not reversible. Make sure you don't lost any data. + +!!! Note Delete is not always allowed + By default, it is not allowed to delete an index for a given configuration. + + To allow deleting, [edit the configuration file](configuration.md) and set `allow_delete` to `True`. + +!!! tip "Good practice: Set admin confs" + For a given ARLAS deployment, it is advised to set two configurations, with only the admin one with `allow_delete` at `True`. + + For example: + ``` + +----------------------+------------------------------------------+ + | name | url | + +----------------------+------------------------------------------+ + | cloud.arlas.io | https://cloud.arlas.io/arlas/server | + | cloud.arlas.io-admin | https://cloud.arlas.io/arlas/server | + | local | http://localhost/arlas | + +----------------------+------------------------------------------+ + ``` + + Here the configuration `--config cloud.arlas.io-admin` has to be used to delete any index. + \ No newline at end of file diff --git a/docs/docs/install.md b/docs/docs/install.md index 8c1065d..744425d 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -1,24 +1,42 @@ +# Installation ## Prerequisite - python 3.10 - pip -If you manage your own ARLAS stack, you will also need: -- [ARLAS](https://github.com/gisaia/ARLAS-Exploration-stack) - ## Install +Install `arlas_cli` with pip: + ```shell > pip install arlas_cli ``` -In a new terminal, you should be able to run it: +!!! success + + In a new terminal, you should be able to run it: + + + ```shell + > arlas_cli --version + X.X.X + ``` + + The version of `arlas_cli` is `xxx.yyy` where `xxx` is ARLAS Stack main version and `yyy` is the increment of arlas_cli version. +TODO: Check version explanation + +## Run ARLAS Locally + +arlas_cli is meant to interact with a deployed ARLAS stack. + +To run ARLAS and elasticsearch on the local machine, clone the [ARLAS Stack Exploration](https://github.com/gisaia/ARLAS-Exploration-stack) project and run the stack: ```shell -> arlas_cli --version -X.X.X +> git clone https://github.com/gisaia/ARLAS-Exploration-stack.git +> cd ARLAS-Exploration-stack +> ./start.sh ``` diff --git a/docs/docs/persist.md b/docs/docs/persist.md index aac0200..175c883 100644 --- a/docs/docs/persist.md +++ b/docs/docs/persist.md @@ -26,3 +26,80 @@ ARLAS Persistence allows you to place files within zones. A zone is visible by g ╰────────────────────────────────────────────────────────────────────╯ ``` + +## ARLAS Persistence + +### Add an entry + + +```shell +> arlas_cli persist --config local add ../arlas-stacks4tests/conf/config.json config.json --name courses_dashboard +32d2624b-d7cd-11ee-9a91-0242ac130004 +``` + +### Describe an entry + + +```shell +> arlas_cli persist --config local describe 32d2624b-d7cd-11ee-9a91-0242ac130004 ++------------------+--------------------------------------+ +| metadata | value | ++------------------+--------------------------------------+ +| ID | 6a415cec-d7cd-11ee-9a91-0242ac130004 | +| ispublic | None | +| last_update_date | 1709298774600 | +| name | courses_dashboard | +| organization | None | +| owner | anonymous | +| updatable | True | +| zone | config.json | ++------------------+--------------------------------------+ +``` + +### Get an entry value + + +```shell +> arlas_cli persist --config local get 32d2624b-d7cd-11ee-9a91-0242ac130004 +{ + "arlas": { + "web": { + "contributors": [ +... +} +``` + +### List entries within a zone + + +```shell +> arlas_cli persist --config local zone config.json ++--------------------------------------+-------------------+-------------+------------------+-----------+ +| id | name | zone | last_update_date | owner | ++--------------------------------------+-------------------+-------------+------------------+-----------+ +| 66984014-d0a1-11ee-b41c-0242ac190004 | Courses | config.json | 1708510231303 | anonymous | +... ++--------------------------------------+-------------------+-------------+------------------+-----------+ +``` + + +### List groups accessing a zone + + +```shell +> arlas_cli persist --config local groups config.json ++--------------+ +| group | ++--------------+ +| group/public | ++--------------+ +``` + + +### Delete an entry + + +```shell +> arlas_cli persist --config local delete 32d2624b-d7cd-11ee-9a91-0242ac130004 +Resource 32d2624b-d7cd-11ee-9a91-0242ac130004 deleted. +``` diff --git a/docs/docs/started.md b/docs/docs/started.md index b29bd21..927e2a7 100644 --- a/docs/docs/started.md +++ b/docs/docs/started.md @@ -1,62 +1,91 @@ -Install `arlas_cli` ([prerequisite](install.md#Prerequisite)) - +# Getting started with ARLAS CLI + +## Install arlas_cli + +To install `arlas_cli`, run the following command from the command line: + ```shell -> pip install arlas_cli +pip install arlas_cli ``` -The version of `arlas_cli` is `xxx.yyy` where `xxx` is ARLAS Stack main version and `yyy` is the increment of arlas_cli version. +For more details, see the [Installation Guide](install.md#installation). -!!! warning "Prerequisite" - For running the various examples bellow, ARLAS and elasticsearch must be running on the local machine: clone the [ARLAS Stack Exploration](https://github.com/gisaia/ARLAS-Exploration-stack) project and run `./start.sh` . -## Initial configuration -`arlas_cli` uses a yaml file for storing various ARLAS and elasticsearch configurations. By default, the file is located in `~/.arlas/cli/configuration.yaml`. [One is automatically created for your convenience at the first launch](https://raw.githubusercontent.com/gisaia/arlas-cli/master/configuration.yaml). It contains the ARLAS demo endpoint and the local ARLAS and elasticsearch endpoints. +## Configuration + +!!! warning "Prerequisite" + For running the various examples bellow, ARLAS and elasticsearch must be running on the local machine: + + Clone the [ARLAS Stack Exploration](https://github.com/gisaia/ARLAS-Exploration-stack) project and run `./start.sh` at project root. -The configuration can also contain references to collection models for creating collections. A default one is provided for ARLAS EO. A reference can be an http url or a path to a local file. -It can also contain references to index mappings for creating indices. A default one is provided for ARLAS EO. +By default, the local configuration is created during the installation. -## Running +To list the available configurations, run the following command from the command line: ```shell -> arlas_cli --version -X.X.X -Warning : no configuration file found, we created an empty one for -you (~/.arlas/cli/configuration.yaml). +> arlas_cli confs list ++----------------------+------------------------------------------+ +| name | url | ++----------------------+------------------------------------------+ +| local | http://localhost/arlas | ++----------------------+------------------------------------------+ ``` -## Examples +!!! success + At least the `local` configuration has to be here -In the following examples, you will see how to: +It will be used in the tutorial as `--config local` -- generate an elasticsearch mapping based on json objects -- add the mapping in elasticsearch -- list the elasticsearch indices -- add (index) data in the elasticsearch index -- get the structure of the mapping -- add a collection in ARLAS -- list the ARLAS collections -- get the structure of a ARLAS collection -- delete a collection from ARLAS -- delete a mapping from elasticsearch -- add, describe, get, list and delete an entry from ARLAS Persistence -- list, create, describe and delete a configuration for `arlas_cli` +For more details, see the [Configuration Guide](configuration.md#configuration). + +## Tutorial + +In the following tutorial, you will see how to: + +- Generate an elasticsearch mapping based on json objects +- Add the mapping in elasticsearch +- List the elasticsearch indices +- Add (index) data in the elasticsearch index +- Get the structure of the mapping +- Add a collection in ARLAS +- List the ARLAS collections +- Get the structure of a ARLAS collection +- Delete a collection from ARLAS +- Delete a mapping from elasticsearch +- Add, describe, get, list and delete an entry from ARLAS Persistence +- List, create, describe and delete a configuration for `arlas_cli` ... with the `arlas_cli` command line only! -We suppose you have an elasticsearch and arlas server running. +!!! warning "Prerequisite" + First, let's get a sample data file: + + ```shell + > curl -X GET https://raw.githubusercontent.com/gisaia/arlas-cli/master/tests/sample.json -o sample.json + ``` + The downloaded `sample.json` contains a sample of processed AIS data. + ### Generate the elasticsearch mapping -Writing the elasticsearch mapping for an index can be laborious. `arlas_cli` does it for you. `arlas_cli` can inspect a NDJSON file (one json object per line) and generate the corresponding elasticsearch mapping file. +Writing the elasticsearch mapping for an index can be laborious. `arlas_cli` does it for you. +`arlas_cli` can inspect a NDJSON file (one json object per line) and generate the corresponding elasticsearch mapping file. -First, let's get a sample data file: +To generate the mapping file based on that sample, run the following command: ```shell -> curl -X GET https://raw.githubusercontent.com/gisaia/arlas-cli/master/tests/sample.json -o sample.json +> arlas_cli indices \ + --config local \ + mapping sample.json ``` -Now we can generate the mapping file based on that sample: +By inspecting the mapping, we notice that the three timestamps are not identified as datetimes by `arlas_cli`. + +!!! info "--field-mapping" + The `--field-mapping` option allows to overwrite the detected type. + +To generate the mapping with forced types, run the following command: ```shell > arlas_cli indices \ @@ -64,64 +93,60 @@ Now we can generate the mapping file based on that sample: mapping sample.json \ --field-mapping track.timestamps.center:date-epoch_second \ --field-mapping track.timestamps.start:date-epoch_second \ - --field-mapping track.timestamps.end:date-epoch_second \ - --push-on courses + --field-mapping track.timestamps.end:date-epoch_second ``` -The `--push-on` option registers the mapping in the specified index. +The three timestamps are now well identified as datetime. -Note that the three timestamps are not identified as datetimes by `arlas_cli`. The `--field-mapping` allows to overwrite the detected type. +For more details, see the [Indices Mapping Guide](indices.md#generate-mapping-from-a-data-file). -#### Type identification -A geometry is identified as such if +### Create an empty index from inferred mapping -- it is a geojson -- it is a WKT string -- the field name contains `geohash` -- it is a string containing two float seperated by a comma +Once the inferred mapping is fine, an elasticsearch index based on this mapping has to be created. -A date is identified as such if +!!! info "--push-on" + In the previous `arlas_cli indices mapping` command, the `--push-on` option registers the mapping in the specified index. -- its name is one of `timestamp`, `date`, `start` or `end` and that it can be parsed as a date -- its name contains `timestamp`, `date`, `start` or `end` and its values are number within [631152000, 4102444800] or [631152000000, 4102444800000] (year 1990 to 2100) - - -### Generate the elasticsearch mapping - -To add a specific mapping, it is possible to use the `create`` command: + An empty index is created with the inferred mapping +To create the associated empty index `ais_courses` with the `--push-on` option, run the following command: ```shell -> arlas_cli indices \ +> arlas_cli indices \ --config local \ - create courses \ - --mapping mapping.json + mapping sample.json \ + --field-mapping track.timestamps.center:date-epoch_second \ + --field-mapping track.timestamps.start:date-epoch_second \ + --field-mapping track.timestamps.end:date-epoch_second \ + --push-on ais_courses ``` -### List indices +For more details, see the [Indices Creation Guide](indices.md#create-an-index-with-its-mapping). + +### Inspect the created indices + +To check that the index has been created, list the existing index: + ```shell > arlas_cli indices --config local list -+----------+--------+-------+--------+ -| name | status | count | size | -+----------+--------+-------+--------+ -| .arlas | open | 4 | 11.9kb | -| courses | open | 0 | 249b | -+----------+--------+-------+--------+ ++--------------+--------+-------+--------+ +| name | status | count | size | ++--------------+--------+-------+--------+ +| .arlas | open | 4 | 11.9kb | +| ais_courses | open | 0 | 249b | ++--------------+--------+-------+--------+ ``` -### Add data - -```shell -> arlas_cli indices --config local data courses sample.json -``` +!!! success + The `ais_courses` index exists and contains 0 elements -### Describe an index +To describe the fields of the index, use the `arlas_cli indices describe` command: ```shell -> arlas_cli indices --config local describe courses +> arlas_cli indices --config local describe ais_courses +----------------------------------------------------+-----------+ | field name | type | +----------------------------------------------------+-----------+ @@ -136,36 +161,87 @@ To add a specific mapping, it is possible to use the `create`` command: +----------------------------------------------------+-----------+ ``` -### Add a collection +It corresponds to the inferred mapping. + +### Add data to index + +To add data to the created `ais_courses` index, use the `arlas_cli indices data` command with the data file `sample.json` + +```shell +> arlas_cli indices --config local data ais_courses sample.json +``` + +To check that data has been correctly indexed, inspect the indices with: + + +```shell +> arlas_cli indices --config local list ++--------------+--------+-------+--------+ +| name | status | count | size | ++--------------+--------+-------+--------+ +| .arlas | open | 4 | 11.9kb | +| ais_courses | open | 100 | 1mb | ++--------------+--------+-------+--------+ +``` + +!!! success + The `ais_courses` index now contains 100 elements + +### Create a collection + +To explore the data in ARLAS, a collection has to be defined on top on an index. + +To create a `ais_courses` collection based on the `ais_courses` index, run the following command: + ```shell > arlas_cli collections \ - --config local \ - create courses \ - --index courses --display-name courses \ + --config local \ + create ais_courses \ + --index ais_courses \ + --display-name "AIS Courses" \ --id-path track.id \ --centroid-path track.location \ --geometry-path track.trail \ - --date-path track.timestamps.center + --date-path track.timestamps.center ``` +The `--index` option define the index to use and the `--display-name` define a pretty name used for collection in ARLAS. + +Several options define the data structure: + +- `--id-path`: The data field used as unique id of each element + +- `--centroid-path`: The data field containing a point geometry summarizing the location of each element (used for aggregation) + +- `--geometry-path`: The data field containing a geometry to represent the element (can be point, linestring, polygon) + +- `--date-path`: The data field containing the date associated to each element (used for timeline) + +For more details, see the [Collection Creation Guide](collections.md#create-a-collection). + +### Inspect the created collections + +To list the available collections, run the following command: -### List collections ```shell > arlas_cli collections --config local list -+---------+---------+ -| name | index | -+---------+---------+ -| courses | courses | -+---------+---------+ ++-------------+-------------+ +| name | index | ++-------------+-------------+ +| ais_courses | ais_courses | ++-------------+-------------+ ``` -### Describe a collection +!!! success + The `ais_courses` collection is now created + +To describe the fields of the collection, use the `arlas_cli collections describe` command: ```shell -> arlas_cli collections --config local describe courses +> arlas_cli collections --config local describe ais_courses +----------------------------------------------------+-----------+ | field name | type | +----------------------------------------------------+-----------+ @@ -181,21 +257,39 @@ To add a specific mapping, it is possible to use the `create`` command: +----------------------------------------------------+-----------+ ``` -### Delete a collection +It corresponds to the mapping of the data within the collection + +### Delete an index + +To remove the indexed data from the local elasticsearch instance, remove the index with following command: ```shell -> arlas_cli collections --config local delete courses +> arlas_cli indices --config local delete ais_courses ``` -### Delete an index +Check that `ais_courses` index no longer exists: ```shell -> arlas_cli indices --config local delete courses +> arlas_cli indices --config local list ++--------------+--------+-------+--------+ +| name | status | count | size | ++--------------+--------+-------+--------+ +| .arlas | open | 4 | 11.9kb | ++--------------+--------+-------+--------+ ``` -!!! Note Delete is not always allowed - By default, it is not allowed to delete an index for a given configuration. To allow deleting, edit the configuration file and set `allow_delete` to `True`. +!!! tip + Before reindexing data, do not forget to create [recreate the empty index from inferred mapping](#create-an-empty-index-from-inferred-mapping) + Collection do not need to be declared again + +### Delete a collection + +To delete the `ais_courses` course collection + +```shell +> arlas_cli collections --config local delete courses +``` ## ARLAS Persistence @@ -276,108 +370,3 @@ Resource 32d2624b-d7cd-11ee-9a91-0242ac130004 deleted. -## Configurations - -The command line uses the `${HOME}/.arlas/cli/configuration.yaml` configuration file: - -```yaml -arlas: - local: - allow_delete: true - elastic: - headers: - Content-Type: application/json - location: http://localhost:9200 - server: - headers: - Content-Type: application/json - location: http://localhost:9999/arlas -mappings: - arlas_eo: - headers: null - location: https://raw.githubusercontent.com/gisaia/ARLAS-EO/master/mapping.json -models: - arlas_eo: - headers: null - location: https://raw.githubusercontent.com/gisaia/ARLAS-EO/master/collection.json -``` - -The `arlas` section contains the different deployment configurations. The mapping section lists the mapping template that you can use. - Finally, the models are the templates for the collections. A [detailed description](model/README.md) of the configuration structure is provided. - -### Create, describe and delete a configuration for `arlas_cli` - -You can edit directly the `${HOME}/.arlas/cli/configuration.yaml` configuration file to update your configurations. You can also use the command line itself. - -To list the configurations: - -```shell -> arlas_cli confs list -+-----------+-----------------------------+ -| name | url | -+-----------+-----------------------------+ -| local | http://localhost:9999/arlas | -| test_conf | http://localhost:9999 | -+-----------+-----------------------------+ -``` - -To describe a configuration: - - -```shell -arlas_cli confs describe local -allow_delete: true -authorization: null -elastic: - headers: - Content-Type: application/json - location: http://localhost:9200 -server: - headers: - Content-Type: application/json - location: http://localhost:9999/arlas -``` - -To create a simple configuration: - - -```shell -> arlas_cli confs create dev_conf \ - --server http://localhost:9999 \ - --headers "Content-Type:application/json" \ - --elastic http://localhost:9200 \ - --elastic-headers "Content-Type:application/json" \ - --no-allow-delete -``` - -For an arlas configuration with authentication: - - -```shell -> arlas_cli --config-file /tmp/configuration.yaml confs \ - create myarlas_as_user \ - --server http://myserver/arlas \ - --headers "arlas-org-filter:my_org_name" \ - --headers "Content-Type:application/json" \ - --no-allow-delete \ - --auth-token-url http://myserver/arlas_iam_server/session \ - --auth-login user \ - --auth-password my_password \ - --auth-headers "Content-Type:application/json;charset=utf-8"\ - --auth-arlas-iam -``` - -To delete the configuration: - - -```shell -> arlas_cli confs delete dev_conf -``` - -Also, it is possible to use a different configuration file than the one placed in your home directory (`$HOME/.arlas/cli/configuration.yaml`): - - -```shell -> arlas_cli --config-file /tmp/config.yaml -Warning : no configuration file found, we created an empty one for you (/tmp/config.yaml). -``` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index c02f5d2..1bc1625 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -27,19 +27,20 @@ theme: accent: 'light-blue' nav: -- arlas_cli version: version.md - About arlas_cli: index.md -- Installation: install.md - Getting started: started.md +- Installation: install.md +- Configuration: configuration.md - Commands: - Getting help: help.md - - Indices: indices.md - - Collections: collections.md - - Persistence: persist.md - - Identity & Access: iam.md - - Configurations: confs.md + - confs: confs.md + - indices: indices.md + - collections: collections.md + - persist: persist.md + - iam: iam.md - Configuration data model: model/model.md - Configuring ARLAS Cloud: arlas_cloud.md +- arlas_cli version: version.md plugins: - termynal: