From 19924bbed37c2503b92f0de64b9aed6cc36d2713 Mon Sep 17 00:00:00 2001 From: Sylvain Gaudan Date: Thu, 24 Oct 2024 16:09:47 +0200 Subject: [PATCH 1/5] copy an index or migrate between clusters --- arlas/cli/index.py | 25 +++++++++++++++++++++ arlas/cli/service.py | 52 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/arlas/cli/index.py b/arlas/cli/index.py index e0dc652..49f5023 100644 --- a/arlas/cli/index.py +++ b/arlas/cli/index.py @@ -37,6 +37,31 @@ def describe( print(tab) +@indices.command(help="Clone an index and set its name") +def clone( + index: str = typer.Argument(help="index's name"), + name: str = typer.Argument(help="Cloned index's name") +): + config = variables["arlas"] + indices = Service.clone_index(config, index, name) + tab = PrettyTable(indices[0], sortby="name", align="l") + tab.add_rows(indices[1:]) + print(tab) + + +@indices.command(help="Migrate an index on another arlas configuration, and set the target index name") +def migrate( + index: str = typer.Argument(help="index's name"), + arlas_target: str = typer.Argument(help="ARLAS Configuration name"), + name: str = typer.Argument(help="Migrated index's name") +): + config = variables["arlas"] + indices = Service.migrate_index(config, index, arlas_target, name) + tab = PrettyTable(indices[0], sortby="name", align="l") + tab.add_rows(indices[1:]) + print(tab) + + @indices.command(help="Display a sample of an index") def sample( index: str = typer.Argument(help="index's name"), diff --git a/arlas/cli/service.py b/arlas/cli/service.py index 98eeff2..ee8922a 100644 --- a/arlas/cli/service.py +++ b/arlas/cli/service.py @@ -135,16 +135,17 @@ def list_collections(arlas: str) -> list[list[str]]: ]) return table - def list_indices(arlas: str) -> list[list[str]]: + def list_indices(arlas: str, keeponly: None) -> list[list[str]]: data = json.loads(Service.__es__(arlas, "_cat/indices?format=json")) table = [["name", "status", "count", "size"]] for index in data: - table.append([ - index.get("index"), - index.get("status"), - index.get("docs.count"), - index.get("store.size") - ]) + if keeponly is None or keeponly == index.get("index"): + table.append([ + index.get("index"), + index.get("status"), + index.get("docs.count"), + index.get("store.size") + ]) return table def set_collection_visibility(arlas: str, collection: str, public: bool): @@ -227,6 +228,35 @@ def describe_index(arlas: str, index: str) -> list[list[str]]: table.extend(Service.__get_fields__([], description.get(index, {}).get("mappings", {}).get("properties", {}))) return table + def clone_index(arlas: str, index: str, name: str) -> list[list[str]]: + Service.__es__(arlas, "/".join([index, "_block", "write"]), put="") + Service.__es__(arlas, "/".join([index, "_clone", name]), put="") + Service.__es__(arlas, "/".join([index, "_settings"]), put='{"index.blocks.write": false}') + return Service.list_indices(arlas, keeponly=name) + + def migrate_index(arlas: str, index: str, target_arlas: str, target_name: str) -> list[list[str]]: + source = Configuration.settings.arlas.get(arlas) + migration = { + "source": { + "remote": { + "host": source.elastic.location, + "username": source.elastic.login, + "password": source.elastic.password, + }, + "index": index, + "query": {"match_all": {}}, + }, + "dest": {"index": target_name}, + } + print("1/3: fetch mapping ...") + mapping = Service.__es__(arlas, "/".join([index, "_mapping"])) + mapping = json.dumps(json.loads(mapping).get(index)) + print("2/3: copy mapping ...") + Service.__es__(target_arlas, "/".join([target_name]), put=mapping) + print("3/3: copy data ...") + print(Service.__es__(target_arlas, "/".join(["_reindex"]), post=json.dumps(migration))) + return Service.list_indices(target_arlas, keeponly=target_name) + def sample_collection(arlas: str, collection: str, pretty: bool, size: int) -> dict: sample = Service.__arlas__(arlas, "/".join(["explore", collection, "_search"]) + "?size={}".format(size)) return sample @@ -432,6 +462,7 @@ def __arlas__(arlas: str, suffix, post=None, put=None, patch=None, delete=None, else: print("Error: request {} failed with status {}: {}".format(method, str(r.status_code), str(r.reason)), file=sys.stderr) print(" url: {}".format(url), file=sys.stderr) + print(r.content) exit(1) except Exception as e: print("Error: request {} failed on {}".format(method, e), file=sys.stderr) @@ -452,13 +483,13 @@ def __es__(arlas: str, suffix, post=None, put=None, delete=None, exit_on_failure auth = (endpoint.elastic.login, endpoint.elastic.password) if endpoint.elastic.login else None method = "GET" data = None - if post: + if post is not None: data = post method = "POST" - if put: + if put is not None: data = put method = "PUT" - if delete: + if delete is not None: method = "DELETE" r: requests.Response = Service.__request__(url, method, data, __headers, auth) if r.ok: @@ -466,6 +497,7 @@ def __es__(arlas: str, suffix, post=None, put=None, delete=None, exit_on_failure elif exit_on_failure: print("Error: request {} failed with status {}: {}".format(method, str(r.status_code), str(r.reason)), file=sys.stderr) print(" url: {}".format(url), file=sys.stderr) + print(r.content) exit(1) else: raise RequestException(r.status_code, r.content) From 4d4780b7be611611363f0ef62a0a9174e26975ea Mon Sep 17 00:00:00 2001 From: Sylvain Gaudan Date: Thu, 24 Oct 2024 16:18:25 +0200 Subject: [PATCH 2/5] add test for clone --- arlas/cli/index.py | 1 + arlas/cli/service.py | 2 +- scripts/tests.sh | 9 +++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/arlas/cli/index.py b/arlas/cli/index.py index 49f5023..1a937d9 100644 --- a/arlas/cli/index.py +++ b/arlas/cli/index.py @@ -112,6 +112,7 @@ def data( Service.index_hits(config, index=index, file_path=file, bulk_size=bulk, count=count) i = i + 1 + @indices.command(help="Generate the mapping based on the data") def mapping( file: str = typer.Argument(help="Path to the file conaining the data. Format: NDJSON"), diff --git a/arlas/cli/service.py b/arlas/cli/service.py index ee8922a..85922d5 100644 --- a/arlas/cli/service.py +++ b/arlas/cli/service.py @@ -135,7 +135,7 @@ def list_collections(arlas: str) -> list[list[str]]: ]) return table - def list_indices(arlas: str, keeponly: None) -> list[list[str]]: + def list_indices(arlas: str, keeponly: str = None) -> list[list[str]]: data = json.loads(Service.__es__(arlas, "_cat/indices?format=json")) table = [["name", "status", "count", "size"]] for index in data: diff --git a/scripts/tests.sh b/scripts/tests.sh index bd96a96..5f9734e 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -96,6 +96,15 @@ else exit 1 fi +# ---------------------------------------------------------- +echo "TEST clone index" +if python3.10 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml indices --config tests clone courses courses2 | grep courses | grep " 200 "; then + echo "OK: hundred hits found" +else + echo "ERROR: hits not found" + exit 1 +fi + # ---------------------------------------------------------- echo "TEST add collection" From a6aea5ae6a5c02342ecb9081e6200ee444b3ff28 Mon Sep 17 00:00:00 2001 From: Sylvain Gaudan Date: Thu, 24 Oct 2024 16:25:35 +0200 Subject: [PATCH 3/5] fix test for clone --- scripts/tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests.sh b/scripts/tests.sh index 5f9734e..2822f9d 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -98,7 +98,7 @@ fi # ---------------------------------------------------------- echo "TEST clone index" -if python3.10 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml indices --config tests clone courses courses2 | grep courses | grep " 200 "; then +if python3.10 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml indices --config tests clone courses courses2 | grep courses2 | grep " 200 "; then echo "OK: hundred hits found" else echo "ERROR: hits not found" From abceffe1908a3c30dd9db8c656ef2d3499482204 Mon Sep 17 00:00:00 2001 From: Sylvain Gaudan Date: Thu, 24 Oct 2024 16:38:33 +0200 Subject: [PATCH 4/5] fix test clone --- scripts/tests.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/tests.sh b/scripts/tests.sh index 2822f9d..fc1b2b7 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -98,10 +98,11 @@ fi # ---------------------------------------------------------- echo "TEST clone index" -if python3.10 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml indices --config tests clone courses courses2 | grep courses2 | grep " 200 "; then - echo "OK: hundred hits found" +if python3.10 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml indices --config tests clone courses courses2 | grep courses2 ; then + echo "OK: course2 found" + yes | python3.10 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml indices --config tests delete courses2 else - echo "ERROR: hits not found" + echo "ERROR: course2 not found" exit 1 fi From 855d8a3dc465f5862130d6ceedf962c29d7b3f87 Mon Sep 17 00:00:00 2001 From: Sylvain Gaudan Date: Thu, 24 Oct 2024 17:51:14 +0200 Subject: [PATCH 5/5] fix with PR comments --- arlas/cli/index.py | 10 +++++----- arlas/cli/service.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arlas/cli/index.py b/arlas/cli/index.py index 1a937d9..396d302 100644 --- a/arlas/cli/index.py +++ b/arlas/cli/index.py @@ -39,8 +39,8 @@ def describe( @indices.command(help="Clone an index and set its name") def clone( - index: str = typer.Argument(help="index's name"), - name: str = typer.Argument(help="Cloned index's name") + index: str = typer.Argument(help="Source index name"), + name: str = typer.Argument(help="Target cloned index name") ): config = variables["arlas"] indices = Service.clone_index(config, index, name) @@ -51,9 +51,9 @@ def clone( @indices.command(help="Migrate an index on another arlas configuration, and set the target index name") def migrate( - index: str = typer.Argument(help="index's name"), - arlas_target: str = typer.Argument(help="ARLAS Configuration name"), - name: str = typer.Argument(help="Migrated index's name") + index: str = typer.Argument(help="Source index name"), + arlas_target: str = typer.Argument(help="Target ARLAS Configuration name"), + name: str = typer.Argument(help="Target migrated index name") ): config = variables["arlas"] indices = Service.migrate_index(config, index, arlas_target, name) diff --git a/arlas/cli/service.py b/arlas/cli/service.py index 85922d5..bdef180 100644 --- a/arlas/cli/service.py +++ b/arlas/cli/service.py @@ -135,11 +135,11 @@ def list_collections(arlas: str) -> list[list[str]]: ]) return table - def list_indices(arlas: str, keeponly: str = None) -> list[list[str]]: + def list_indices(arlas: str, keep_only: str = None) -> list[list[str]]: data = json.loads(Service.__es__(arlas, "_cat/indices?format=json")) table = [["name", "status", "count", "size"]] for index in data: - if keeponly is None or keeponly == index.get("index"): + if keep_only is None or keep_only == index.get("index"): table.append([ index.get("index"), index.get("status"), @@ -232,7 +232,7 @@ def clone_index(arlas: str, index: str, name: str) -> list[list[str]]: Service.__es__(arlas, "/".join([index, "_block", "write"]), put="") Service.__es__(arlas, "/".join([index, "_clone", name]), put="") Service.__es__(arlas, "/".join([index, "_settings"]), put='{"index.blocks.write": false}') - return Service.list_indices(arlas, keeponly=name) + return Service.list_indices(arlas, keep_only=name) def migrate_index(arlas: str, index: str, target_arlas: str, target_name: str) -> list[list[str]]: source = Configuration.settings.arlas.get(arlas) @@ -255,7 +255,7 @@ def migrate_index(arlas: str, index: str, target_arlas: str, target_name: str) - Service.__es__(target_arlas, "/".join([target_name]), put=mapping) print("3/3: copy data ...") print(Service.__es__(target_arlas, "/".join(["_reindex"]), post=json.dumps(migration))) - return Service.list_indices(target_arlas, keeponly=target_name) + return Service.list_indices(target_arlas, keep_only=target_name) def sample_collection(arlas: str, collection: str, pretty: bool, size: int) -> dict: sample = Service.__arlas__(arlas, "/".join(["explore", collection, "_search"]) + "?size={}".format(size))