copy an index or migrate between clusters

arlas/cli/index.py

@@ -37,6 +37,31 @@ def describe(
     print(tab)
 
 
+@indices.command(help="Clone an index and set its name")
+def clone(
+    index: str = typer.Argument(help="Source index name"),
+    name: str = typer.Argument(help="Target cloned index name")
+):
+    config = variables["arlas"]
+    indices = Service.clone_index(config, index, name)
+    tab = PrettyTable(indices[0], sortby="name", align="l")
+    tab.add_rows(indices[1:])
+    print(tab)
+
+
+@indices.command(help="Migrate an index on another arlas configuration, and set the target index name")
+def migrate(
+    index: str = typer.Argument(help="Source index name"),
+    arlas_target: str = typer.Argument(help="Target ARLAS Configuration name"),
+    name: str = typer.Argument(help="Target migrated index name")
+):
+    config = variables["arlas"]
+    indices = Service.migrate_index(config, index, arlas_target, name)
+    tab = PrettyTable(indices[0], sortby="name", align="l")
+    tab.add_rows(indices[1:])
+    print(tab)
+
+
 @indices.command(help="Display a sample of an index")
 def sample(
     index: str = typer.Argument(help="index's name"),
@@ -87,6 +112,7 @@ def data(
         Service.index_hits(config, index=index, file_path=file, bulk_size=bulk, count=count)
         i = i + 1
 
+
 @indices.command(help="Generate the mapping based on the data")
 def mapping(
     file: str = typer.Argument(help="Path to the file conaining the data. Format: NDJSON"),

arlas/cli/service.py

@@ -135,16 +135,17 @@ def list_collections(arlas: str) -> list[list[str]]:
             ])
         return table
 
-    def list_indices(arlas: str) -> list[list[str]]:
+    def list_indices(arlas: str, keep_only: str = None) -> list[list[str]]:
         data = json.loads(Service.__es__(arlas, "_cat/indices?format=json"))
         table = [["name", "status", "count", "size"]]
         for index in data:
-            table.append([
-                index.get("index"),
-                index.get("status"),
-                index.get("docs.count"),
-                index.get("store.size")
-            ])
+            if keep_only is None or keep_only == index.get("index"):
+                table.append([
+                    index.get("index"),
+                    index.get("status"),
+                    index.get("docs.count"),
+                    index.get("store.size")
+                ])
         return table
 
     def set_collection_visibility(arlas: str, collection: str, public: bool):
@@ -227,6 +228,35 @@ def describe_index(arlas: str, index: str) -> list[list[str]]:
         table.extend(Service.__get_fields__([], description.get(index, {}).get("mappings", {}).get("properties", {})))
         return table
 
+    def clone_index(arlas: str, index: str, name: str) -> list[list[str]]:
+        Service.__es__(arlas, "/".join([index, "_block", "write"]), put="")
+        Service.__es__(arlas, "/".join([index, "_clone", name]), put="")
+        Service.__es__(arlas, "/".join([index, "_settings"]), put='{"index.blocks.write": false}')
+        return Service.list_indices(arlas, keep_only=name)
+
+    def migrate_index(arlas: str, index: str, target_arlas: str, target_name: str) -> list[list[str]]:
+        source = Configuration.settings.arlas.get(arlas)
+        migration = {
+            "source": {
+                "remote": {
+                    "host": source.elastic.location,
+                    "username": source.elastic.login,
+                    "password": source.elastic.password,
+                },
+                "index": index,
+                "query": {"match_all": {}},
+            },
+            "dest": {"index": target_name},
+        }
+        print("1/3: fetch mapping ...")
+        mapping = Service.__es__(arlas, "/".join([index, "_mapping"]))
+        mapping = json.dumps(json.loads(mapping).get(index))
+        print("2/3: copy mapping ...")
+        Service.__es__(target_arlas, "/".join([target_name]), put=mapping)
+        print("3/3: copy data ...")
+        print(Service.__es__(target_arlas, "/".join(["_reindex"]), post=json.dumps(migration)))
+        return Service.list_indices(target_arlas, keep_only=target_name)
+
     def sample_collection(arlas: str, collection: str, pretty: bool, size: int) -> dict:
         sample = Service.__arlas__(arlas, "/".join(["explore", collection, "_search"]) + "?size={}".format(size))
         return sample
@@ -432,6 +462,7 @@ def __arlas__(arlas: str, suffix, post=None, put=None, patch=None, delete=None,
             else:
                 print("Error: request {} failed with status {}: {}".format(method, str(r.status_code), str(r.reason)), file=sys.stderr)
                 print("   url: {}".format(url), file=sys.stderr)
+                print(r.content)
                 exit(1)
         except Exception as e:
             print("Error: request {} failed on {}".format(method, e), file=sys.stderr)
@@ -452,20 +483,21 @@ def __es__(arlas: str, suffix, post=None, put=None, delete=None, exit_on_failure
         auth = (endpoint.elastic.login, endpoint.elastic.password) if endpoint.elastic.login else None
         method = "GET"
         data = None
-        if post:
+        if post is not None:
             data = post
             method = "POST"
-        if put:
+        if put is not None:
             data = put
             method = "PUT"
-        if delete:
+        if delete is not None:
             method = "DELETE"
         r: requests.Response = Service.__request__(url, method, data, __headers, auth)
         if r.ok:
             return r.content
         elif exit_on_failure:
             print("Error: request {} failed with status {}: {}".format(method, str(r.status_code), str(r.reason)), file=sys.stderr)
             print("   url: {}".format(url), file=sys.stderr)
+            print(r.content)
             exit(1)
         else:
             raise RequestException(r.status_code, r.content)

scripts/tests.sh

@@ -96,6 +96,16 @@ else
     exit 1
 fi
 
+# ----------------------------------------------------------
+echo "TEST clone index"
+if python3.10 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml indices --config tests clone courses courses2 | grep courses2 ; then
+    echo "OK: course2 found"
+    yes | python3.10 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml indices --config tests delete courses2
+else
+    echo "ERROR: course2 not found"
+    exit 1
+fi
+
 
 # ----------------------------------------------------------
 echo "TEST add collection"