From 19924bbed37c2503b92f0de64b9aed6cc36d2713 Mon Sep 17 00:00:00 2001 From: Sylvain Gaudan Date: Thu, 24 Oct 2024 16:09:47 +0200 Subject: [PATCH] copy an index or migrate between clusters --- arlas/cli/index.py | 25 +++++++++++++++++++++ arlas/cli/service.py | 52 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/arlas/cli/index.py b/arlas/cli/index.py index e0dc652..49f5023 100644 --- a/arlas/cli/index.py +++ b/arlas/cli/index.py @@ -37,6 +37,31 @@ def describe( print(tab) +@indices.command(help="Clone an index and set its name") +def clone( + index: str = typer.Argument(help="index's name"), + name: str = typer.Argument(help="Cloned index's name") +): + config = variables["arlas"] + indices = Service.clone_index(config, index, name) + tab = PrettyTable(indices[0], sortby="name", align="l") + tab.add_rows(indices[1:]) + print(tab) + + +@indices.command(help="Migrate an index on another arlas configuration, and set the target index name") +def migrate( + index: str = typer.Argument(help="index's name"), + arlas_target: str = typer.Argument(help="ARLAS Configuration name"), + name: str = typer.Argument(help="Migrated index's name") +): + config = variables["arlas"] + indices = Service.migrate_index(config, index, arlas_target, name) + tab = PrettyTable(indices[0], sortby="name", align="l") + tab.add_rows(indices[1:]) + print(tab) + + @indices.command(help="Display a sample of an index") def sample( index: str = typer.Argument(help="index's name"), diff --git a/arlas/cli/service.py b/arlas/cli/service.py index 98eeff2..ee8922a 100644 --- a/arlas/cli/service.py +++ b/arlas/cli/service.py @@ -135,16 +135,17 @@ def list_collections(arlas: str) -> list[list[str]]: ]) return table - def list_indices(arlas: str) -> list[list[str]]: + def list_indices(arlas: str, keeponly: None) -> list[list[str]]: data = json.loads(Service.__es__(arlas, "_cat/indices?format=json")) table = [["name", "status", "count", "size"]] for index in data: - table.append([ - index.get("index"), - index.get("status"), - index.get("docs.count"), - index.get("store.size") - ]) + if keeponly is None or keeponly == index.get("index"): + table.append([ + index.get("index"), + index.get("status"), + index.get("docs.count"), + index.get("store.size") + ]) return table def set_collection_visibility(arlas: str, collection: str, public: bool): @@ -227,6 +228,35 @@ def describe_index(arlas: str, index: str) -> list[list[str]]: table.extend(Service.__get_fields__([], description.get(index, {}).get("mappings", {}).get("properties", {}))) return table + def clone_index(arlas: str, index: str, name: str) -> list[list[str]]: + Service.__es__(arlas, "/".join([index, "_block", "write"]), put="") + Service.__es__(arlas, "/".join([index, "_clone", name]), put="") + Service.__es__(arlas, "/".join([index, "_settings"]), put='{"index.blocks.write": false}') + return Service.list_indices(arlas, keeponly=name) + + def migrate_index(arlas: str, index: str, target_arlas: str, target_name: str) -> list[list[str]]: + source = Configuration.settings.arlas.get(arlas) + migration = { + "source": { + "remote": { + "host": source.elastic.location, + "username": source.elastic.login, + "password": source.elastic.password, + }, + "index": index, + "query": {"match_all": {}}, + }, + "dest": {"index": target_name}, + } + print("1/3: fetch mapping ...") + mapping = Service.__es__(arlas, "/".join([index, "_mapping"])) + mapping = json.dumps(json.loads(mapping).get(index)) + print("2/3: copy mapping ...") + Service.__es__(target_arlas, "/".join([target_name]), put=mapping) + print("3/3: copy data ...") + print(Service.__es__(target_arlas, "/".join(["_reindex"]), post=json.dumps(migration))) + return Service.list_indices(target_arlas, keeponly=target_name) + def sample_collection(arlas: str, collection: str, pretty: bool, size: int) -> dict: sample = Service.__arlas__(arlas, "/".join(["explore", collection, "_search"]) + "?size={}".format(size)) return sample @@ -432,6 +462,7 @@ def __arlas__(arlas: str, suffix, post=None, put=None, patch=None, delete=None, else: print("Error: request {} failed with status {}: {}".format(method, str(r.status_code), str(r.reason)), file=sys.stderr) print(" url: {}".format(url), file=sys.stderr) + print(r.content) exit(1) except Exception as e: print("Error: request {} failed on {}".format(method, e), file=sys.stderr) @@ -452,13 +483,13 @@ def __es__(arlas: str, suffix, post=None, put=None, delete=None, exit_on_failure auth = (endpoint.elastic.login, endpoint.elastic.password) if endpoint.elastic.login else None method = "GET" data = None - if post: + if post is not None: data = post method = "POST" - if put: + if put is not None: data = put method = "PUT" - if delete: + if delete is not None: method = "DELETE" r: requests.Response = Service.__request__(url, method, data, __headers, auth) if r.ok: @@ -466,6 +497,7 @@ def __es__(arlas: str, suffix, post=None, put=None, delete=None, exit_on_failure elif exit_on_failure: print("Error: request {} failed with status {}: {}".format(method, str(r.status_code), str(r.reason)), file=sys.stderr) print(" url: {}".format(url), file=sys.stderr) + print(r.content) exit(1) else: raise RequestException(r.status_code, r.content)