From ead7c0fbf6260f58c63405e3d1aa58649a72c65f Mon Sep 17 00:00:00 2001 From: Sylvain Gaudan Date: Fri, 8 Mar 2024 16:11:01 +0100 Subject: [PATCH] fix persist --- arlas/cli/cli.py | 2 +- arlas/cli/collections.py | 2 +- arlas/cli/index.py | 6 ++++-- arlas/cli/persist.py | 4 ++-- arlas/cli/service.py | 16 ++++++++++------ scripts/tests.sh | 13 ++++++++++++- 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/arlas/cli/cli.py b/arlas/cli/cli.py index c250c17..c04b317 100644 --- a/arlas/cli/cli.py +++ b/arlas/cli/cli.py @@ -37,7 +37,7 @@ def init( arlas={ "demo": ARLAS(server=Resource(location="https://demo.cloud.arlas.io/arlas/server", headers={"Content-Type": "application/json"})), "local": ARLAS( - server=Resource(location="http://localhost/arlas", headers={"Content-Type": "application/json"}), + server=Resource(location="http://localhost/server", headers={"Content-Type": "application/json"}), persistence=Resource(location="http://localhost/persist", headers={"Content-Type": "application/json"}), elastic=Resource(location="http://localhost:9200", headers={"Content-Type": "application/json"}), allow_delete=True diff --git a/arlas/cli/collections.py b/arlas/cli/collections.py index 024daa5..2eeac96 100644 --- a/arlas/cli/collections.py +++ b/arlas/cli/collections.py @@ -13,7 +13,7 @@ @collections.callback() def configuration(config: str = typer.Option(help="Name of the ARLAS configuration to use from your configuration file ({}).".format(variables["configuration_file"]))): - variables["arlas"] = config + variables["arlas"] = config @collections.command(help="List collections", name="list") diff --git a/arlas/cli/index.py b/arlas/cli/index.py index 4acea86..1ba3913 100644 --- a/arlas/cli/index.py +++ b/arlas/cli/index.py @@ -52,7 +52,8 @@ def sample( def create( index: str = typer.Argument(help="index's name"), mapping: str = typer.Option(help="Name of the mapping within your configuration, or URL or file path"), - shards: int = typer.Option(default=1, help="Number of shards for the index") + shards: int = typer.Option(default=1, help="Number of shards for the index"), + add_uuid: str = typer.Argument(default=None, help="Set a UUID for the provided json path field") ): config = variables["arlas"] mapping_resource = Configuration.settings.mappings.get(mapping, None) @@ -66,7 +67,8 @@ def create( config, index=index, mapping_resource=mapping_resource, - number_of_shards=shards) + number_of_shards=shards, + add_uuid=add_uuid) print("Index {} created on {}".format(index, config)) diff --git a/arlas/cli/persist.py b/arlas/cli/persist.py index 6b24d68..5cf0ebf 100644 --- a/arlas/cli/persist.py +++ b/arlas/cli/persist.py @@ -26,7 +26,7 @@ def add( encode: bool = typer.Option(help="Encode in BASE64", default=False) ): config = variables["arlas"] - id = Service.persistence_add_file(config, Resource(location=file), zone=zone, name=name, readers=reader, encode=encode) + id = Service.persistence_add_file(config, Resource(location=file), zone=zone, name=name, readers=reader, writers=writer, encode=encode) print(id) @@ -56,7 +56,7 @@ def get( id: str = typer.Argument(help="entry identifier") ): config = variables["arlas"] - print(Service.persistence_get(config, id=id).get("doc_value")) + print(Service.persistence_get(config, id=id).get("doc_value"), end="") @persist.command(help="List entries within a zone", name="zone") diff --git a/arlas/cli/service.py b/arlas/cli/service.py index 62ffb3f..965d4ed 100644 --- a/arlas/cli/service.py +++ b/arlas/cli/service.py @@ -105,12 +105,12 @@ def create_collection(arlas: str, collection: str, model_resource: str, index: s print(json.dumps(model)) Service.__arlas__(arlas, "/".join(["collections", collection]), put=json.dumps(model)) - def create_index_from_resource(arlas: str, index: str, mapping_resource: str, number_of_shards: int): + def create_index_from_resource(arlas: str, index: str, mapping_resource: str, number_of_shards: int, add_uuid: str = None): mapping = json.loads(Service.__fetch__(mapping_resource)) if not mapping.get("mappings"): print("Error: mapping {} does not contain \"mappings\" at its root.".format(mapping_resource), file=sys.stderr) exit(1) - Service.create_index(arlas, index, mapping, number_of_shards) + Service.create_index(arlas, index, mapping, number_of_shards, add_uuid) def create_index(arlas: str, index: str, mapping: str, number_of_shards: int = 1): index_doc = {"mappings": mapping.get("mappings"), "settings": {"number_of_shards": number_of_shards}} @@ -143,7 +143,7 @@ def count_hits(file_path: str) -> int: return line_number def persistence_add_file(arlas: str, file: Resource, zone: str, name: str, encode: bool = False, readers: list[str] = [], writers: list[str] = []): - content = Service.__fetch__(file) + content = Service.__fetch__(file, bytes=True) url = "/".join(["persist", "resource", zone, name]) + "?" + "&readers=".join(readers) + "&writers=".join(writers) return Service.__arlas__(arlas, url, post=content, service=Services.persistence_server).get("id") @@ -240,10 +240,11 @@ def __arlas__(arlas: str, suffix, post=None, put=None, delete=None, service=Serv print("Error: arlas configuration ({}) not found among [{}] for {}.".format(arlas, ", ".join(Configuration.settings.arlas.keys()), service.name), file=sys.stderr) exit(1) if service == Services.arlas_server: + __headers__ = configuration.server.headers.copy() endpoint: Resource = configuration.server else: + __headers__ = configuration.persistence.headers.copy() endpoint: Resource = configuration.persistence - __headers__ = Configuration.settings.arlas.get(arlas).server.headers.copy() if Configuration.settings.arlas.get(arlas).authorization is not None: __headers__["Authorization"] = "Bearer " + Service.__get_token__(arlas) url = "/".join([endpoint.location, suffix]) @@ -300,10 +301,13 @@ def __es__(arlas: str, suffix, post=None, put=None, delete=None, exit_on_failure else: raise RequestException(r.status_code, r.content) - def __fetch__(resource: Resource): + def __fetch__(resource: Resource, bytes: bool = False): if os.path.exists(resource.location): content = None - with open(resource.location) as f: + mode = "r" + if bytes: + mode = "rb" + with open(resource.location, mode) as f: content = f.read() return content r = requests.get(resource.location, headers=resource.headers) diff --git a/scripts/tests.sh b/scripts/tests.sh index 6e4a85e..baf3ea8 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -213,6 +213,17 @@ else exit 1 fi +# ---------------------------------------------------------- +echo "TEST entry content is same as sent" +python3 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml persist --config tests get $id > /tmp/result + +if cmp --silent -- /tmp/result README.md; then + echo "OK: content is the same" +else + echo "ERROR: content differs" + exit 1 +fi + # ---------------------------------------------------------- echo "TEST entry describe" if python3 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml persist --config tests describe $id | grep toto ; then @@ -223,7 +234,7 @@ else fi # ---------------------------------------------------------- -echo "TEST entry describe" +echo "TEST list groups" if python3 -m arlas.cli.cli --config-file /tmp/arlas_cli.yaml persist --config tests groups my_zone | grep "group/public" ; then echo "OK: groups found " else