Merge pull request #4 from gisaia/feat/persist

Feat/persist

arlas/cli/cli.py

@@ -4,6 +4,7 @@
 
 from arlas.cli.collections import collections
 from arlas.cli.configurations import configurations
+from arlas.cli.persist import persist
 from arlas.cli.index import indices
 from arlas.cli.variables import variables
 from arlas.cli.settings import ARLAS, Configuration, Resource, Settings
@@ -36,7 +37,8 @@ def init(
             arlas={
                 "demo": ARLAS(server=Resource(location="https://demo.cloud.arlas.io/arlas/server", headers={"Content-Type": "application/json"})),
                 "local": ARLAS(
-                    server=Resource(location="http://localhost:9999/arlas", headers={"Content-Type": "application/json"}),
+                    server=Resource(location="http://localhost/server", headers={"Content-Type": "application/json"}),
+                    persistence=Resource(location="http://localhost/persist", headers={"Content-Type": "application/json"}),
                     elastic=Resource(location="http://localhost:9200", headers={"Content-Type": "application/json"}),
                     allow_delete=True
                 )
@@ -56,6 +58,7 @@ def init(
 def main():
     app.add_typer(collections, name="collections")
     app.add_typer(indices, name="indices")
+    app.add_typer(persist, name="persist")
     app.add_typer(configurations, name="confs")
     app()
 

arlas/cli/configurations.py

@@ -22,6 +22,7 @@ def list_configurations():
 def create_configuration(
     name: str = typer.Argument(help="Name of the configuration"),
     server: str = typer.Option(help="ARLAS Server url"),
+    persistence: str = typer.Option(default=None, help="ARLAS Persistence url"),
     headers: list[str] = typer.Option([], help="header (name:value)"),
     elastic: str = typer.Option(default=None, help="dictionary of name/es resources"),
     elastic_headers: list[str] = typer.Option([], help="header (name:value)"),
@@ -42,6 +43,9 @@ def create_configuration(
     conf = ARLAS(
         server=Resource(location=server, headers=dict(map(lambda h: (h.split(":")[0], h.split(":")[1]), headers))),
         allow_delete=allow_delete)
+    if persistence:
+        conf.persistence = Resource(location=persistence, headers=dict(map(lambda h: (h.split(":")[0], h.split(":")[1]), headers)))
+
     if auth_token_url:
         conf.authorization = AuthorizationService(
             token_url=Resource(location=auth_token_url, headers=dict(map(lambda h: (h.split(":")[0], h.split(":")[1]), auth_headers))),

arlas/cli/index.py

@@ -52,7 +52,8 @@ def sample(
 def create(
     index: str = typer.Argument(help="index's name"),
     mapping: str = typer.Option(help="Name of the mapping within your configuration, or URL or file path"),
-    shards: int = typer.Option(default=1, help="Number of shards for the index")
+    shards: int = typer.Option(default=1, help="Number of shards for the index"),
+    add_uuid: str = typer.Argument(default=None, help="Set a UUID for the provided json path field")
 ):
     config = variables["arlas"]
     mapping_resource = Configuration.settings.mappings.get(mapping, None)
@@ -66,7 +67,8 @@ def create(
         config,
         index=index,
         mapping_resource=mapping_resource,
-        number_of_shards=shards)
+        number_of_shards=shards,
+        add_uuid=add_uuid)
     print("Index {} created on {}".format(index, config))
 
 
@@ -121,10 +123,10 @@ def delete(
 ):
     config = variables["arlas"]
     if not Configuration.settings.arlas.get(config).allow_delete:
-        print("Error: delete on \"{}\" is not allowed. To allow delete, change your configuration file ({}).".format(config, configuration_file), file=sys.stderr)
+        print("Error: delete on \"{}\" is not allowed. To allow delete, change your configuration file ({}).".format(config, variables["configuration_file"]), file=sys.stderr)
         exit(1)
 
-    if typer.confirm("You are about to delete the index '{}' on the '{}' configuration.\n".format(index, config),
+    if typer.confirm("You are about to delete the index '{}' on  '{}' configuration.\n".format(index, config),
                      prompt_suffix="Do you want to continue (del {} on {})?".format(index, config),
                      default=False, ):
         if config != "local" and config.find("test") < 0:

arlas/cli/persist.py

@@ -0,0 +1,92 @@
+import json
+import typer
+import os
+import sys
+from prettytable import PrettyTable
+
+from arlas.cli.settings import Configuration, Resource
+from arlas.cli.service import Service
+from arlas.cli.variables import variables
+
+persist = typer.Typer()
+
+
+@persist.callback()
+def configuration(config: str = typer.Option(help="Name of the ARLAS configuration to use from your configuration file ({}).".format(variables["configuration_file"]))):
+    variables["arlas"] = config
+
+
+@persist.command(help="Add an entry, returns its ID", name="add")
+def add(
+    file: str = typer.Argument(help="File path"),
+    zone: str = typer.Argument(help="zone"),
+    name: str = typer.Option(help="name", default="none"),
+    reader: list[str] = typer.Option(help="Readers", default=[]),
+    writer: list[str] = typer.Option(help="writers", default=[]),
+    encode: bool = typer.Option(help="Encode in BASE64", default=False)
+):
+    config = variables["arlas"]
+    id = Service.persistence_add_file(config, Resource(location=file), zone=zone, name=name, readers=reader, writers=writer, encode=encode)
+    print(id)
+
+
+@persist.command(help="Delete an entry", name="delete")
+def delete(
+    id: str = typer.Argument(help="entry identifier")
+):
+    config = variables["arlas"]
+    if not Configuration.settings.arlas.get(config).allow_delete:
+        print("Error: delete on \"{}\" is not allowed. To allow delete, change your configuration file ({}).".format(config, variables["configuration_file"]), file=sys.stderr)
+        exit(1)
+
+    if typer.confirm("You are about to delete the entry '{}' on '{}' configuration.\n".format(id, config),
+                     prompt_suffix="Do you want to continue (del {} on {})?".format(id, config),
+                     default=False, ):
+        if config != "local" and config.find("test") < 0:
+            if typer.prompt("WARNING: You are not on a test environment. To delete {} on {}, type the name of the configuration ({})".format(id, config, config)) != config:
+                print("Error: delete on \"{}\" cancelled.".format(config), file=sys.stderr)
+                exit(1)
+
+    Service.persistence_delete(config, id=id)
+    print("Resource {} deleted.".format(id))
+
+
+@persist.command(help="Retrieve an entry", name="get")
+def get(
+    id: str = typer.Argument(help="entry identifier")
+):
+    config = variables["arlas"]
+    print(Service.persistence_get(config, id=id).get("doc_value"), end="")
+
+
+@persist.command(help="List entries within a zone", name="zone")
+def zone(
+    zone: str = typer.Argument(help="Zone name")
+):
+    config = variables["arlas"]
+    table = Service.persistence_zone(config, zone=zone)
+    tab = PrettyTable(table[0], sortby="name", align="l")
+    tab.add_rows(table[1:])
+    print(tab)
+
+
+@persist.command(help="List groups allowed to access a zone", name="groups")
+def groups(
+    zone: str = typer.Argument(help="Zone name")
+):
+    config = variables["arlas"]
+    table = Service.persistence_groups(config, zone=zone)
+    tab = PrettyTable(table[0], sortby="group", align="l")
+    tab.add_rows(table[1:])
+    print(tab)
+
+
+@persist.command(help="Describe an entry", name="describe")
+def describe(
+    id: str = typer.Argument(help="entry identifier")
+):
+    config = variables["arlas"]
+    table = Service.persistence_describe(config, id=id)
+    tab = PrettyTable(table[0], sortby="metadata", align="l")
+    tab.add_rows(table[1:])
+    print(tab)

arlas/cli/service.py

@@ -1,10 +1,11 @@
+from enum import Enum
 import json
 import os
 import sys
 from alive_progress import alive_bar
 import requests
 
-from arlas.cli.settings import Configuration, Resource
+from arlas.cli.settings import ARLAS, Configuration, Resource
 
 
 class RequestException(Exception):
@@ -13,6 +14,11 @@ def __init__(self, code, message):
         self.message = message
 
 
+class Services(Enum):
+    arlas_server = "server"
+    persistence_server = "persistence"
+
+
 class Service:
 
     def list_collections(arlas: str) -> list[list[str]]:
@@ -99,12 +105,12 @@ def create_collection(arlas: str, collection: str, model_resource: str, index: s
         print(json.dumps(model))
         Service.__arlas__(arlas, "/".join(["collections", collection]), put=json.dumps(model))
 
-    def create_index_from_resource(arlas: str, index: str, mapping_resource: str, number_of_shards: int):
+    def create_index_from_resource(arlas: str, index: str, mapping_resource: str, number_of_shards: int, add_uuid: str = None):
         mapping = json.loads(Service.__fetch__(mapping_resource))
         if not mapping.get("mappings"):
             print("Error: mapping {} does not contain \"mappings\" at its root.".format(mapping_resource), file=sys.stderr)
             exit(1)
-        Service.create_index(arlas, index, mapping, number_of_shards)
+        Service.create_index(arlas, index, mapping, number_of_shards, add_uuid)
 
     def create_index(arlas: str, index: str, mapping: str, number_of_shards: int = 1):
         index_doc = {"mappings": mapping.get("mappings"), "settings": {"number_of_shards": number_of_shards}}
@@ -134,7 +140,50 @@ def count_hits(file_path: str) -> int:
         with open(file_path) as f:
             for line in f:
                 line_number = line_number + 1
-        return line_number                
+        return line_number
+
+    def persistence_add_file(arlas: str, file: Resource, zone: str, name: str, encode: bool = False, readers: list[str] = [], writers: list[str] = []):
+        content = Service.__fetch__(file, bytes=True)
+        url = "/".join(["persist", "resource", zone, name]) + "?" + "&readers=".join(readers) + "&writers=".join(writers)
+        return Service.__arlas__(arlas, url, post=content, service=Services.persistence_server).get("id")
+
+    def persistence_delete(arlas: str, id: str):
+        url = "/".join(["persist", "resource", "id", id])
+        return Service.__arlas__(arlas, url, delete=True, service=Services.persistence_server)
+
+    def persistence_get(arlas: str, id: str):
+        url = "/".join(["persist", "resource", "id", id])
+        return Service.__arlas__(arlas, url, service=Services.persistence_server)
+
+    def persistence_zone(arlas: str, zone: str):
+        url = "/".join(["persist", "resources", zone]) + "?size=10&page=1&order=desc&pretty=false"
+        table = [["id", "name", "zone", "last_update_date", "owner"]]
+        entries = Service.__arlas__(arlas, url, service=Services.persistence_server).get("data", [])
+        for entry in entries:
+            table.append([entry["id"], entry["doc_key"], entry["doc_zone"], entry["last_update_date"], entry["doc_owner"]])
+        return table
+
+    def persistence_groups(arlas: str, zone: str):
+        url = "/".join(["persist", "groups", zone])
+        table = [["group"]]
+        groups = Service.__arlas__(arlas, url, service=Services.persistence_server)
+        for group in groups:
+            table.append([group])
+        return table
+
+    def persistence_describe(arlas: str, id: str):
+        url = "/".join(["persist", "resource", "id", id])
+        r = Service.__arlas__(arlas, url, service=Services.persistence_server)
+        table = [["metadata", "value"]]
+        table.append(["ID", r.get("id")])
+        table.append(["name", r.get("doc_key")])
+        table.append(["zone", r.get("doc_zone")])
+        table.append(["last_update_date", r.get("last_update_date")])
+        table.append(["owner", r.get("doc_owner")])
+        table.append(["organization", r.get("doc_organization")])
+        table.append(["ispublic", r.get("ispublic")])
+        table.append(["updatable", r.get("updatable")])
+        return table
 
     def __index_bulk__(arlas: str, index: str, bulk: []):
         data = os.linesep.join([json.dumps(line) for line in bulk]) + os.linesep
@@ -185,29 +234,39 @@ def __get_fields__(origin: list[str], properties: dict[str:dict]):
                 fields.append([".".join(o), type])
         return fields
 
-    def __arlas__(arlas: str, suffix, post=None, put=None, delete=None):
-        __headers__ = Configuration.settings.arlas.get(arlas).server.headers.copy()
-        if Configuration.settings.arlas.get(arlas).authorization is not None:
-            __headers__["Authorization"] = "Bearer " + Service.__get_token__(arlas)
-        endpoint = Configuration.settings.arlas.get(arlas)
-        if endpoint is None:
-            print("Error: arlas configuration ({}) not found among [{}].".format(arlas, ", ".join(Configuration.settings.arlas.keys())), file=sys.stderr)
+    def __arlas__(arlas: str, suffix, post=None, put=None, delete=None, service=Services.arlas_server):
+        configuration: ARLAS = Configuration.settings.arlas.get(arlas, {})
+        if configuration is None:
+            print("Error: arlas configuration ({}) not found among [{}] for {}.".format(arlas, ", ".join(Configuration.settings.arlas.keys()), service.name), file=sys.stderr)
             exit(1)
-        url = "/".join([endpoint.server.location, suffix])
-        if post:
-            r = requests.post(url, data=post, headers=__headers__)
+        if service == Services.arlas_server:
+            __headers__ = configuration.server.headers.copy()
+            endpoint: Resource = configuration.server
         else:
-            if put:
-                r = requests.put(url, data=put, headers=__headers__)
+            __headers__ = configuration.persistence.headers.copy()
+            endpoint: Resource = configuration.persistence
+        if Configuration.settings.arlas.get(arlas).authorization is not None:
+            __headers__["Authorization"] = "Bearer " + Service.__get_token__(arlas)
+        url = "/".join([endpoint.location, suffix])
+        try:
+            if post:
+                r = requests.post(url, data=post, headers=__headers__)
             else:
-                if delete:
-                    r = requests.delete(url, headers=__headers__)
+                if put:
+                    r = requests.put(url, data=put, headers=__headers__)
                 else:
-                    r = requests.get(url, headers=__headers__)
-        if r.ok:
-            return r.json()
-        else:
-            print("Error: request failed with status {}: {}".format(str(r.status_code), r.content), file=sys.stderr)
+                    if delete:
+                        r = requests.delete(url, headers=__headers__)
+                    else:
+                        r = requests.get(url, headers=__headers__)
+            if r.ok:
+                return r.json()
+            else:
+                print("Error: request failed with status {}: {}".format(str(r.status_code), r.content), file=sys.stderr)
+                print("   url: {}".format(url), file=sys.stderr)
+                exit(1)
+        except Exception as e:
+            print("Error: request failed: {}".format(e), file=sys.stderr)
             print("   url: {}".format(url), file=sys.stderr)
             exit(1)
 
@@ -242,10 +301,13 @@ def __es__(arlas: str, suffix, post=None, put=None, delete=None, exit_on_failure
             else:
                 raise RequestException(r.status_code, r.content)
 
-    def __fetch__(resource: Resource):
+    def __fetch__(resource: Resource, bytes: bool = False):
         if os.path.exists(resource.location):
             content = None
-            with open(resource.location) as f:
+            mode = "r"
+            if bytes:
+                mode = "rb"
+            with open(resource.location, mode) as f:
                 content = f.read()
             return content
         r = requests.get(resource.location, headers=resource.headers)

arlas/cli/settings.py

@@ -20,6 +20,7 @@ class AuthorizationService(BaseModel):
 
 
 class ARLAS(BaseModel):
+    persistence: Resource | None = Field(title="ARLAS Persistence Server", default=None)
     server: Resource = Field(title="ARLAS Server")
     authorization: AuthorizationService | None = Field(default=None, title="Keycloak URL")
     elastic: Resource | None = Field(default=None, title="dictionary of name/es resources")

docker/dc-test.yaml

@@ -35,8 +35,6 @@ services:
       - arlas-net
     logging:
       driver: ${DOCKER_LOGGING_DRIVER:-json-file}
-      options:
-        tag: "urms"
     healthcheck:
           test: ["CMD","java","HttpHealthcheck.java","http://localhost:9999/admin/healthcheck"]
           interval: 5s
@@ -51,7 +49,7 @@ services:
       - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
       - discovery.type=single-node
       - cluster.name=arlas-es-cluster
-      - node.name=urms-data-node-1
+      - node.name=data-node-1
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
       - xpack.security.enabled=false
       - tracing.apm.enabled=false
@@ -68,10 +66,29 @@ services:
       - arlas-net
     logging:
       driver: ${DOCKER_LOGGING_DRIVER:-json-file}
-      options:
-        tag: "urms"
     healthcheck:
       test: "curl -s --user ${ELASTIC_USER}:${ELASTIC_PASSWORD} -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'"
       interval: 10s
       timeout: 10s
       retries: 24
+
+  arlas-persistence:
+    image: ${ARLAS_PERSISTENCE_SERVER_VERSION}
+    container_name: arlas-persistence
+    restart: always
+    environment:
+      - ARLAS_PERSISTENCE_HIBERNATE_DRIVER=org.postgresql.Driver
+      - ARLAS_PERSISTENCE_LOCAL_FOLDER=/tmp/
+    ports:
+      - "9997:9997"
+    expose:
+      - "9997"
+    networks:
+      - arlas-net
+    logging:
+      driver: ${DOCKER_LOGGING_DRIVER:-json-file}
+    healthcheck:
+          test: ["CMD","java","HttpHealthcheck.java","http://localhost:9997/arlas_persistence_server/swagger.json"]
+          interval: 5s
+          timeout: 10s
+          retries: 3