From 2887a7b9f4788a5785e18f2120e4818a0b6c431c Mon Sep 17 00:00:00 2001 From: Ahmed Nader Date: Tue, 24 Dec 2024 17:09:21 +0300 Subject: [PATCH 1/4] Added the test_rest_catalog suite of tests. Covering the functionality migration from the mock tests --- tests/integration/test_rest_catalog.py | 827 ++++++++++++++++++++++++- 1 file changed, 815 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_rest_catalog.py b/tests/integration/test_rest_catalog.py index 24a8d9f6e..8ce0e64f3 100644 --- a/tests/integration/test_rest_catalog.py +++ b/tests/integration/test_rest_catalog.py @@ -16,34 +16,806 @@ # under the License. # pylint:disable=redefined-outer-name + +from typing import Any, Dict + import pytest +from pyiceberg.catalog import PropertiesUpdateSummary from pyiceberg.catalog.rest import RestCatalog +from pyiceberg.exceptions import ( + BadRequestError, + NamespaceAlreadyExistsError, + NoSuchIdentifierError, + NoSuchNamespaceError, + NoSuchTableError, + NoSuchViewError, + TableAlreadyExistsError, +) +from pyiceberg.io import load_file_io +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import Schema +from pyiceberg.table import Table +from pyiceberg.table.metadata import TableMetadataV2 +from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder +from pyiceberg.transforms import IdentityTransform, TruncateTransform + +TEST_NAMESPACE_IDENTIFIER = ("rest_integration_ns",) +TEST_TABLE_IDENTIFIER = ("rest_integration_ns", "rest_integration_tbl") +TEST_TABLE_IDENTIFIER_RENAME = ("rest_integration_ns", "renamed_rest_integration_tbl") + +EXAMPLE_table_metadata_no_snapshot_v2 = { + "format-version": 2, + "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29", + "location": f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}", + "last-updated-ms": 1657810967051, + "last-column-id": 3, + "schema": { + "type": "struct", + "schema-id": 0, + "identifier-field-ids": [2], + "fields": [ + {"id": 1, "name": "foo", "required": False, "type": "string"}, + {"id": 2, "name": "bar", "required": True, "type": "int"}, + {"id": 3, "name": "baz", "required": False, "type": "boolean"}, + ], + }, + "current-schema-id": 0, + "schemas": [ + { + "type": "struct", + "fields": ( + {"id": 1, "name": "foo", "type": "string", "required": False}, + {"id": 2, "name": "bar", "type": "int", "required": True}, + {"id": 3, "name": "baz", "type": "boolean", "required": False}, + ), + "schema-id": 0, + "identifier-field-ids": [2], + } + ], + "partition-specs": [{"spec-id": 0, "fields": ()}], + "default-spec-id": 0, + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": { + "write.parquet.compression-codec": "zstd", + }, + "refs": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], +} + +EXAMPLE_table_metadata_no_snapshot_partitioned_v2 = { + "format-version": 2, + "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29", + "location": f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}", + "last-updated-ms": 1657810967051, + "last-column-id": 3, + "schema": { + "type": "struct", + "schema-id": 0, + "identifier-field-ids": [2], + "fields": [ + {"id": 1, "name": "foo", "required": False, "type": "string"}, + {"id": 2, "name": "bar", "required": True, "type": "int"}, + {"id": 3, "name": "baz", "required": False, "type": "boolean"}, + ], + }, + "current-schema-id": 0, + "schemas": [ + { + "type": "struct", + "fields": ( + {"id": 1, "name": "foo", "type": "string", "required": False}, + {"id": 2, "name": "bar", "type": "int", "required": True}, + {"id": 3, "name": "baz", "type": "boolean", "required": False}, + ), + "schema-id": 0, + "identifier-field-ids": [2], + } + ], + "partition-specs": [ + {"spec-id": 0, "fields": ({"source-id": 1, "field-id": 1000, "transform": "truncate[3]", "name": "id"},)} + ], + "default-spec-id": 0, + "last-partition-id": 1000, + "default-sort-order-id": 1, + "sort-orders": [ + { + "order-id": 1, + "fields": [ + {"source-id": 2, "transform": "identity", "direction": SortDirection.ASC, "null-order": NullOrder.NULLS_FIRST} + ], + } + ], + "properties": { + "owner": "fokko", + "write.parquet.compression-codec": "zstd", + }, + "refs": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], +} + + +@pytest.fixture +def table_metadata_no_snapshot_v2() -> Dict[str, Any]: + return EXAMPLE_table_metadata_no_snapshot_v2 + + +@pytest.fixture +def table_metadata_no_snapshot_partitioned_v2() -> Dict[str, Any]: + return EXAMPLE_table_metadata_no_snapshot_partitioned_v2 + + +@pytest.fixture +def rest_integration_example_metadata_partitioned_v2(table_metadata_no_snapshot_partitioned_v2: Dict[str, Any]) -> Dict[str, Any]: + return { + "metadata-location": f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}", + "metadata": table_metadata_no_snapshot_partitioned_v2, + "config": { + "client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", + "region": "us-west-2", + }, + } + + +@pytest.fixture +def example_table_metadata_with_no_location(table_metadata_no_snapshot_v2: Dict[str, Any]) -> Dict[str, Any]: + return { + "metadata": table_metadata_no_snapshot_v2, + "config": { + "client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", + "region": "us-west-2", + }, + } + -TEST_NAMESPACE_IDENTIFIER = "TEST NS" +@pytest.fixture +def table_metadata_no_snapshot_rest(table_metadata_no_snapshot_v2: Dict[str, Any]) -> Dict[str, Any]: + return { + "metadata-location": f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}", + "metadata": table_metadata_no_snapshot_v2, + "config": { + "client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", + "region": "us-west-2", + }, + } @pytest.mark.integration -@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")]) -def test_namespace_exists(catalog: RestCatalog) -> None: - if not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER): +@pytest.fixture(scope="function") +@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up")]) +def test_clean_up(catalog: RestCatalog) -> None: + print("BEGINNING TEST CLEAN UP") + for namespaces_tuple in catalog.list_namespaces(): + print(namespaces_tuple) + namespace_name = namespaces_tuple[0] + print(namespace_name) + if TEST_NAMESPACE_IDENTIFIER[0] in namespace_name: + for identifier in catalog.list_tables(namespace_name): + print(identifier) + catalog.purge_table(identifier) + print(namespace_name) + print(catalog.list_namespaces()) + if catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER): + catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) + print(catalog.list_namespaces()) + + print("FINISHED TEST CLEAN UP") + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_namespace_200(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + assert TEST_NAMESPACE_IDENTIFIER in catalog.list_namespaces() + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_namespace_if_exists_409(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_namespace_if_not_exists(TEST_NAMESPACE_IDENTIFIER) + assert TEST_NAMESPACE_IDENTIFIER in catalog.list_namespaces() + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_namespace_409(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + with pytest.raises(NamespaceAlreadyExistsError) as e: catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + assert "Namespace already exists" in str(e.value) - assert catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_drop_namespace_404(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(NoSuchNamespaceError) as e: + catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) + assert "Namespace does not exist" in str(e.value) @pytest.mark.integration -@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")]) -def test_namespace_not_exists(catalog: RestCatalog) -> None: - if catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER): +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_drop_namespace_409(catalog: RestCatalog, table_schema_simple: Schema, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) + + with pytest.raises(BadRequestError) as e: catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) + assert f"Namespace {TEST_NAMESPACE_IDENTIFIER[0]} is not empty" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_load_namespace_properties_200(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER, properties={"prop": "yes"}) + + assert "yes" == catalog.load_namespace_properties(TEST_NAMESPACE_IDENTIFIER)["prop"] + + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_load_namespace_properties_404(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(NoSuchNamespaceError) as e: + catalog.load_namespace_properties(TEST_NAMESPACE_IDENTIFIER)["prop"] + assert "Namespace does not exist" in str(e.value) + + +# Update Properties +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_update_namespace_properties_200(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER, properties={"prop": "yes", "abc": "abc"}) + assert PropertiesUpdateSummary(removed=["abc"], updated=["prop"], missing=["def"]) == catalog.update_namespace_properties( + TEST_NAMESPACE_IDENTIFIER, {"abc", "def"}, {"prop": "yes"} + ) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_update_namespace_properties_404(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(NoSuchNamespaceError) as e: + catalog.update_namespace_properties(TEST_NAMESPACE_IDENTIFIER, {"abc", "def"}, {"prop": "yes"}) + assert "Namespace does not exist" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_namespace_exists_204(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + assert catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_namespace_exists_404(catalog: RestCatalog, clean_up: Any) -> None: assert not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) @pytest.mark.integration -@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")]) -def test_create_namespace_if_not_exists(catalog: RestCatalog) -> None: +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_namespace_empty(catalog: RestCatalog, clean_up: Any) -> None: + assert not catalog.namespace_exists("") + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_load_table_200( + catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema, table_metadata_no_snapshot_rest: Dict[str, Any] +) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) + + actual = catalog.load_table(TEST_TABLE_IDENTIFIER) + + expected_metadata = table_metadata_no_snapshot_rest["metadata"] + expected_metadata["table-uuid"] = actual.metadata.model_dump()["table-uuid"] # Note! Generated ID + expected_metadata["last-updated-ms"] = actual.metadata.model_dump()["last-updated-ms"] # Note! Generated TIMESTAMP + + expected = Table( + identifier=TEST_TABLE_IDENTIFIER, + metadata_location=table_metadata_no_snapshot_rest["metadata-location"], + metadata=TableMetadataV2(**expected_metadata), + io=load_file_io(), + catalog=catalog, + ) + assert actual.metadata.model_dump() == expected.metadata.model_dump() + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_load_table_honor_access_delegation( + catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema, table_metadata_no_snapshot_rest: Dict[str, Any] +) -> None: + catalog = RestCatalog( + "rest", + **{ + "uri": "http://localhost:8181", + "token": "Some-jwt-token", + "header.X-Iceberg-Access-Delegation": "remote-signing", + }, + ) + + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) + + actual = catalog.load_table(TEST_TABLE_IDENTIFIER) + + expected_metadata = table_metadata_no_snapshot_rest["metadata"] + expected_metadata["table-uuid"] = actual.metadata.model_dump()["table-uuid"] # Note! Generated ID + expected_metadata["last-updated-ms"] = actual.metadata.model_dump()["last-updated-ms"] # Note! Generated TIMESTAMP + + expected = Table( + identifier=TEST_TABLE_IDENTIFIER, + metadata_location=table_metadata_no_snapshot_rest["metadata-location"], + metadata=TableMetadataV2(**expected_metadata), + io=load_file_io(), + catalog=catalog, + ) + + assert actual.metadata.model_dump() == expected.metadata.model_dump() + assert "remote-signing" == catalog._session.headers["X-Iceberg-Access-Delegation"] + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_load_table_from_self_identifier_200( + catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema, table_metadata_no_snapshot_rest: Dict[str, Any] +) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) + + table = catalog.load_table(TEST_TABLE_IDENTIFIER) + actual = catalog.load_table(table.name()) + + expected_metadata = table_metadata_no_snapshot_rest["metadata"] + expected_metadata["table-uuid"] = actual.metadata.model_dump()["table-uuid"] # Note! Generated ID + expected_metadata["last-updated-ms"] = actual.metadata.model_dump()["last-updated-ms"] # Note! Generated TIMESTAMP + + expected = Table( + identifier=TEST_TABLE_IDENTIFIER, + metadata_location=table_metadata_no_snapshot_rest["metadata-location"], + metadata=TableMetadataV2(**expected_metadata), + io=load_file_io(), + catalog=catalog, + ) + assert actual.metadata.model_dump() == expected.metadata.model_dump() + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_load_table_404(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + with pytest.raises(NoSuchTableError) as e: + catalog.load_table((TEST_NAMESPACE_IDENTIFIER[0], "does_not_exist")) + assert "Table does not exist" in str(e.value) + + + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_table_exists_204(catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) + + assert catalog.table_exists(TEST_TABLE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_table_exists_404(catalog: RestCatalog, clean_up: Any) -> None: + assert not catalog.table_exists(TEST_TABLE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_drop_table_404(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + with pytest.raises(NoSuchTableError) as e: + catalog.drop_table(TEST_TABLE_IDENTIFIER) + assert "Table does not exist" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_table_200( + catalog: RestCatalog, + clean_up: Any, + table_schema_simple: Schema, + rest_integration_example_metadata_partitioned_v2: Dict[str, Any], +) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + actual = catalog.create_table( + identifier=TEST_TABLE_IDENTIFIER, + schema=table_schema_simple, + location=None, + partition_spec=PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 + ), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), + properties={"owner": "fokko"}, + ) + + expected_metadata = rest_integration_example_metadata_partitioned_v2["metadata"] + expected_metadata["table-uuid"] = actual.metadata.model_dump()["table-uuid"] # Note! Generated ID + expected_metadata["last-updated-ms"] = actual.metadata.model_dump()["last-updated-ms"] # Note! Generated TIMESTAMP + + expected = Table( + identifier=TEST_TABLE_IDENTIFIER, + metadata_location=rest_integration_example_metadata_partitioned_v2["metadata-location"], + metadata=TableMetadataV2(**expected_metadata), + io=load_file_io(), + catalog=catalog, + ) + + assert actual.metadata.model_dump() == expected.metadata.model_dump() + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_table_with_given_location_removes_trailing_slash_200( + catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema +) -> None: + location = "s3://warehouse/database/table-custom-location" + + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + actual = catalog.create_table( + identifier=TEST_TABLE_IDENTIFIER, + schema=table_schema_simple, + location=f"{location}/", + partition_spec=PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 + ), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), + properties={"owner": "fokko"}, + ) + + assert actual.metadata.model_dump()["location"] == location + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_staged_table_200( + catalog: RestCatalog, + clean_up: Any, + table_schema_simple: Schema, + rest_integration_example_metadata_partitioned_v2: Dict[str, Any], +) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + txn = catalog.create_table_transaction( + identifier=TEST_TABLE_IDENTIFIER, + schema=table_schema_simple, + location=None, + partition_spec=PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 + ), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), + properties={"owner": "fokko"}, + ) + + actual = txn.commit_transaction() + + assert actual.metadata.model_dump()["properties"]["created-at"] + + expected_metadata = rest_integration_example_metadata_partitioned_v2["metadata"] + expected_metadata["table-uuid"] = actual.metadata.model_dump()["table-uuid"] # Note! Generated ID + expected_metadata["last-updated-ms"] = actual.metadata.model_dump()["last-updated-ms"] # Note! Generated TIMESTAMP + expected_metadata["properties"]["created-at"] = actual.metadata.model_dump()["properties"][ + "created-at" + ] # Note! Generated TIMESTAMP + + expected = Table( + identifier=TEST_TABLE_IDENTIFIER, + metadata_location=rest_integration_example_metadata_partitioned_v2["metadata-location"], + metadata=TableMetadataV2(**expected_metadata), + io=load_file_io(), + catalog=catalog, + ) + + assert actual.metadata.model_dump() == expected.metadata.model_dump() + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_table_409(catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_table( + identifier=TEST_TABLE_IDENTIFIER, + schema=table_schema_simple, + location=None, + partition_spec=PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 + ), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), + properties={"owner": "fokko"}, + ) + + with pytest.raises(TableAlreadyExistsError) as e: + catalog.create_table( + identifier=TEST_TABLE_IDENTIFIER, + schema=table_schema_simple, + location=None, + partition_spec=PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 + ), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), + properties={"owner": "fokko"}, + ) + + assert "Table already exists" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_table_if_not_exists_200(catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + table1 = catalog.create_table( + identifier=TEST_TABLE_IDENTIFIER, + schema=table_schema_simple, + location=None, + partition_spec=PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 + ), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), + properties={"owner": "fokko"}, + ) + + table2 = catalog.create_table_if_not_exists( + identifier=TEST_TABLE_IDENTIFIER, + schema=table_schema_simple, + location=None, + partition_spec=PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 + ), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), + properties={"owner": "fokko"}, + ) + + assert table1 == table2 + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_delete_namespace_204(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) + + assert not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_delete_table_204(catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) + + catalog.drop_table(TEST_TABLE_IDENTIFIER) + + assert not catalog.table_exists(TEST_TABLE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_delete_table_from_self_identifier_204(catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) + table = catalog.load_table(TEST_TABLE_IDENTIFIER) + catalog.drop_table(table.name()) + + assert not catalog.table_exists(table.name()) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_delete_table_404(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + with pytest.raises(NoSuchTableError) as e: + catalog.drop_table(TEST_TABLE_IDENTIFIER) + + assert "Table does not exist" in str(e.value) + + + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_rename_table_200( + catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema, table_metadata_no_snapshot_rest: Dict[str, Any] +) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + catalog.create_table(TEST_TABLE_IDENTIFIER, table_schema_simple) + + actual = catalog.rename_table(TEST_TABLE_IDENTIFIER, TEST_TABLE_IDENTIFIER_RENAME) + + expected_metadata = table_metadata_no_snapshot_rest["metadata"] + expected_metadata["table-uuid"] = actual.metadata.model_dump()["table-uuid"] # Note! Generated ID + expected_metadata["last-updated-ms"] = actual.metadata.model_dump()["last-updated-ms"] # Note! Generated TIMESTAMP + + expected = Table( + identifier=TEST_TABLE_IDENTIFIER_RENAME, + metadata_location=table_metadata_no_snapshot_rest["metadata-location"], + metadata=TableMetadataV2(**expected_metadata), + io=load_file_io(), + catalog=catalog, + ) + + assert not catalog.table_exists(TEST_TABLE_IDENTIFIER) + assert actual.metadata.model_dump() == expected.metadata.model_dump() + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_rename_table_from_self_identifier_200( + catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema, table_metadata_no_snapshot_rest: Dict[str, Any] +) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + original_table = catalog.create_table(TEST_TABLE_IDENTIFIER, table_schema_simple) + + actual = catalog.rename_table(original_table.name(), TEST_TABLE_IDENTIFIER_RENAME) + + expected_metadata = table_metadata_no_snapshot_rest["metadata"] + expected_metadata["table-uuid"] = actual.metadata.model_dump()["table-uuid"] # Note! Generated ID + expected_metadata["last-updated-ms"] = actual.metadata.model_dump()["last-updated-ms"] # Note! Generated TIMESTAMP + + expected = Table( + identifier=TEST_TABLE_IDENTIFIER_RENAME, + metadata_location=table_metadata_no_snapshot_rest["metadata-location"], + metadata=TableMetadataV2(**expected_metadata), + io=load_file_io(), + catalog=catalog, + ) + + assert not catalog.table_exists(TEST_TABLE_IDENTIFIER) + assert actual.metadata.model_dump() == expected.metadata.model_dump() + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_table_missing_namespace(catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema) -> None: + assert not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) + + catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) + assert catalog.table_exists(TEST_TABLE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_load_table_invalid_namespace(catalog: RestCatalog, clean_up: Any) -> None: + table = "table" + with pytest.raises(NoSuchIdentifierError) as e: + catalog.load_table(table) + assert f"Missing namespace or invalid identifier: {table}" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_drop_table_invalid_namespace(catalog: RestCatalog, clean_up: Any) -> None: + table = "table" + with pytest.raises(NoSuchIdentifierError) as e: + catalog.drop_table(table) + assert f"Missing namespace or invalid identifier: {table}" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_purge_table_invalid_namespace(catalog: RestCatalog, clean_up: Any) -> None: + table = "table" + with pytest.raises(NoSuchIdentifierError) as e: + catalog.purge_table(table) + assert f"Missing namespace or invalid identifier: {table}" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_namespace_invalid_namespace(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(NoSuchNamespaceError) as e: + catalog.create_namespace(()) + assert "Empty namespace identifier" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_drop_namespace_invalid_namespace(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(NoSuchNamespaceError) as e: + catalog.drop_namespace(()) + assert "Empty namespace identifier" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_load_namespace_properties_invalid_namespace(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(NoSuchNamespaceError) as e: + catalog.load_namespace_properties(()) + assert "Empty namespace identifier" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_update_namespace_properties_invalid_namespace(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(NoSuchNamespaceError) as e: + catalog.update_namespace_properties(()) + assert "Empty namespace identifier" in str(e.value) + + +# @pytest.mark.integration +# def test_request_session_with_ssl_ca_bundle() -> None: +# catalog_properties = { +# "uri": "http://localhost:8181", +# "token": "some-jwt-token", +# "ssl": { +# "cabundle": "path_to_ca_bundle", +# }, +# } +# catalog = RestCatalog("rest", **catalog_properties) # type: ignore +# print(catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER)) +# with pytest.raises(NoSuchNamespaceError) as e: +# RestCatalog("rest", **catalog_properties) # type: ignore +# assert "Empty namespace identifier" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_request_session_with_ssl_client_cert(catalog: RestCatalog, clean_up: Any) -> None: + catalog_properties = { + "uri": "http://localhost:8181", + "token": "some-jwt-token", + "ssl": { + "client": { + "cert": "path_to_client_cert", + "key": "path_to_client_key", + } + }, + } + with pytest.raises(OSError) as e: + RestCatalog("rest", **catalog_properties) # type: ignore + assert "Could not find the TLS certificate file, invalid path: path_to_client_cert" in str(e.value) + + +# @pytest.mark.integration +# @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +# def test_table_identifier_in_commit_table_request( +# catalog: RestCatalog, clean_up: Any,table_schema_simple: Schema, table_metadata_no_snapshot_rest: Dict[str, Any] +# ) -> None: +# catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) +# catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) +# table = Table( +# identifier=TEST_TABLE_IDENTIFIER, +# metadata=table_metadata_no_snapshot_rest, # type:ignore +# metadata_location=None, # type:ignore +# io=None, # type:ignore +# catalog=catalog, +# ) + +# actual = catalog.commit_table(table, (), ()) +# print("ACTUAL") +# print(actual) +# catalog._session. +# assert not catalog.table_exists(TEST_TABLE_IDENTIFIER) + +# catalog.drop_table(TEST_TABLE_IDENTIFIER) +# catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_namespace_if_not_exists(catalog: RestCatalog, clean_up: Any) -> None: if catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER): catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) @@ -53,11 +825,42 @@ def test_create_namespace_if_not_exists(catalog: RestCatalog) -> None: @pytest.mark.integration -@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")]) -def test_create_namespace_if_already_existing(catalog: RestCatalog) -> None: +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_create_namespace_if_already_existing(catalog: RestCatalog, clean_up: Any) -> None: if not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER): catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) catalog.create_namespace_if_not_exists(TEST_NAMESPACE_IDENTIFIER) assert catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_drop_view_invalid_namespace(catalog: RestCatalog, clean_up: Any) -> None: + view = "view" + + with pytest.raises(NoSuchIdentifierError) as e: + catalog.drop_view(view) + + assert f"Missing namespace or invalid identifier: {view}" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_drop_view_404(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(NoSuchViewError) as e: + catalog.drop_view((TEST_NAMESPACE_IDENTIFIER[0], "NO_VIEW")) + + assert "View does not exist" in str(e.value) + + +# @pytest.mark.integration +# @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +# def test_drop_view_204(catalog: RestCatalog, clean_up: Any,table_schema_simple: Schema) -> None: +# catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) +# catalog.create_table(TEST_TABLE_IDENTIFIER) +# print(catalog.list_views(TEST_NAMESPACE_IDENTIFIER)) + +# assert False +# catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) From 265fa4f435b88f1d58dd99b6873e1933a26c434a Mon Sep 17 00:00:00 2001 From: Ahmed Nader Date: Tue, 24 Dec 2024 18:58:55 +0300 Subject: [PATCH 2/4] Removed ca_bundle and commit_table tests due to being unapplicable --- tests/integration/test_rest_catalog.py | 68 +------------------------- 1 file changed, 1 insertion(+), 67 deletions(-) diff --git a/tests/integration/test_rest_catalog.py b/tests/integration/test_rest_catalog.py index 8ce0e64f3..c9424169f 100644 --- a/tests/integration/test_rest_catalog.py +++ b/tests/integration/test_rest_catalog.py @@ -190,22 +190,13 @@ def table_metadata_no_snapshot_rest(table_metadata_no_snapshot_v2: Dict[str, Any @pytest.fixture(scope="function") @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up")]) def test_clean_up(catalog: RestCatalog) -> None: - print("BEGINNING TEST CLEAN UP") for namespaces_tuple in catalog.list_namespaces(): - print(namespaces_tuple) namespace_name = namespaces_tuple[0] - print(namespace_name) if TEST_NAMESPACE_IDENTIFIER[0] in namespace_name: for identifier in catalog.list_tables(namespace_name): - print(identifier) catalog.purge_table(identifier) - print(namespace_name) - print(catalog.list_namespaces()) if catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER): catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) - print(catalog.list_namespaces()) - - print("FINISHED TEST CLEAN UP") @pytest.mark.integration @@ -247,7 +238,7 @@ def test_drop_namespace_409(catalog: RestCatalog, table_schema_simple: Schema, c catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) - with pytest.raises(BadRequestError) as e: + with pytest.raises(BadRequestError) as e: catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) assert f"Namespace {TEST_NAMESPACE_IDENTIFIER[0]} is not empty" in str(e.value) @@ -260,7 +251,6 @@ def test_load_namespace_properties_200(catalog: RestCatalog, clean_up: Any) -> N assert "yes" == catalog.load_namespace_properties(TEST_NAMESPACE_IDENTIFIER)["prop"] - @pytest.mark.integration @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) def test_load_namespace_properties_404(catalog: RestCatalog, clean_up: Any) -> None: @@ -401,8 +391,6 @@ def test_load_table_404(catalog: RestCatalog, clean_up: Any) -> None: assert "Table does not exist" in str(e.value) - - @pytest.mark.integration @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) def test_table_exists_204(catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema) -> None: @@ -630,8 +618,6 @@ def test_delete_table_404(catalog: RestCatalog, clean_up: Any) -> None: assert "Table does not exist" in str(e.value) - - @pytest.mark.integration @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) def test_rename_table_200( @@ -754,22 +740,6 @@ def test_update_namespace_properties_invalid_namespace(catalog: RestCatalog, cle assert "Empty namespace identifier" in str(e.value) -# @pytest.mark.integration -# def test_request_session_with_ssl_ca_bundle() -> None: -# catalog_properties = { -# "uri": "http://localhost:8181", -# "token": "some-jwt-token", -# "ssl": { -# "cabundle": "path_to_ca_bundle", -# }, -# } -# catalog = RestCatalog("rest", **catalog_properties) # type: ignore -# print(catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER)) -# with pytest.raises(NoSuchNamespaceError) as e: -# RestCatalog("rest", **catalog_properties) # type: ignore -# assert "Empty namespace identifier" in str(e.value) - - @pytest.mark.integration @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) def test_request_session_with_ssl_client_cert(catalog: RestCatalog, clean_up: Any) -> None: @@ -788,31 +758,6 @@ def test_request_session_with_ssl_client_cert(catalog: RestCatalog, clean_up: An assert "Could not find the TLS certificate file, invalid path: path_to_client_cert" in str(e.value) -# @pytest.mark.integration -# @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) -# def test_table_identifier_in_commit_table_request( -# catalog: RestCatalog, clean_up: Any,table_schema_simple: Schema, table_metadata_no_snapshot_rest: Dict[str, Any] -# ) -> None: -# catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) -# catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) -# table = Table( -# identifier=TEST_TABLE_IDENTIFIER, -# metadata=table_metadata_no_snapshot_rest, # type:ignore -# metadata_location=None, # type:ignore -# io=None, # type:ignore -# catalog=catalog, -# ) - -# actual = catalog.commit_table(table, (), ()) -# print("ACTUAL") -# print(actual) -# catalog._session. -# assert not catalog.table_exists(TEST_TABLE_IDENTIFIER) - -# catalog.drop_table(TEST_TABLE_IDENTIFIER) -# catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) - - @pytest.mark.integration @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) def test_create_namespace_if_not_exists(catalog: RestCatalog, clean_up: Any) -> None: @@ -853,14 +798,3 @@ def test_drop_view_404(catalog: RestCatalog, clean_up: Any) -> None: catalog.drop_view((TEST_NAMESPACE_IDENTIFIER[0], "NO_VIEW")) assert "View does not exist" in str(e.value) - - -# @pytest.mark.integration -# @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) -# def test_drop_view_204(catalog: RestCatalog, clean_up: Any,table_schema_simple: Schema) -> None: -# catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) -# catalog.create_table(TEST_TABLE_IDENTIFIER) -# print(catalog.list_views(TEST_NAMESPACE_IDENTIFIER)) - -# assert False -# catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) From 614acdc6297bb5f66b73d783c3733bcf7163358e Mon Sep 17 00:00:00 2001 From: Ahmed Nader Date: Tue, 24 Dec 2024 20:05:32 +0300 Subject: [PATCH 3/4] Added skipping for tests that are not producing correct result. --- tests/integration/test_rest_catalog.py | 59 +++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_rest_catalog.py b/tests/integration/test_rest_catalog.py index c9424169f..e634c4294 100644 --- a/tests/integration/test_rest_catalog.py +++ b/tests/integration/test_rest_catalog.py @@ -30,6 +30,7 @@ NoSuchNamespaceError, NoSuchTableError, NoSuchViewError, + OAuthError, TableAlreadyExistsError, ) from pyiceberg.io import load_file_io @@ -43,6 +44,8 @@ TEST_NAMESPACE_IDENTIFIER = ("rest_integration_ns",) TEST_TABLE_IDENTIFIER = ("rest_integration_ns", "rest_integration_tbl") TEST_TABLE_IDENTIFIER_RENAME = ("rest_integration_ns", "renamed_rest_integration_tbl") +TEST_URI = "http://localhost:8181" +TEST_CREDENTIALS = "client:secret" EXAMPLE_table_metadata_no_snapshot_v2 = { "format-version": 2, @@ -329,7 +332,7 @@ def test_load_table_honor_access_delegation( catalog = RestCatalog( "rest", **{ - "uri": "http://localhost:8181", + "uri": TEST_URI, "token": "Some-jwt-token", "header.X-Iceberg-Access-Delegation": "remote-signing", }, @@ -744,7 +747,7 @@ def test_update_namespace_properties_invalid_namespace(catalog: RestCatalog, cle @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) def test_request_session_with_ssl_client_cert(catalog: RestCatalog, clean_up: Any) -> None: catalog_properties = { - "uri": "http://localhost:8181", + "uri": TEST_URI, "token": "some-jwt-token", "ssl": { "client": { @@ -798,3 +801,55 @@ def test_drop_view_404(catalog: RestCatalog, clean_up: Any) -> None: catalog.drop_view((TEST_NAMESPACE_IDENTIFIER[0], "NO_VIEW")) assert "View does not exist" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_properties_sets_headers(catalog: RestCatalog, clean_up: Any) -> None: + catalog = RestCatalog( + "rest", + uri=TEST_URI, + warehouse="s3://some-bucket", + **{"header.Content-Type": "application/vnd.api+json", "header.Customized-Header": "some/value"}, + ) + assert catalog._session.headers.get("Content-type") == "application/json", ( + "Expected 'Content-Type' default header not to be overwritten" + ) + assert catalog._session.headers.get("Customized-Header") == "some/value", ( + "Expected 'Customized-Header' header to be 'some/value'" + ) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +@pytest.mark.skip(reason="Not raising OAuthError") +def test_token_400(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(OAuthError) as e: + RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS) + assert str(e.value) == "invalid_client: Credentials for key invalid_key do not match" + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +@pytest.mark.skip(reason="Not raising OAuthError") +def test_token_401(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(OAuthError) as e: + RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS) + assert "invalid_client" in str(e.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +@pytest.mark.skip(reason="Not raising OSError") +def test_request_session_with_ssl_ca_bundle(catalog: RestCatalog, clean_up: Any) -> None: + catalog_properties = { + "uri": "http://localhost:8181", + "token": "some-jwt-token", + "ssl": { + "cabundle": "path_to_ca_bundle", + }, + } + + with pytest.raises(OSError) as e: + RestCatalog("rest", **catalog_properties) # type: ignore + assert "Empty namespace identifier" in str(e.value) From b29132805073f2014fcb46df6df9fe8768a5c7bf Mon Sep 17 00:00:00 2001 From: Ahmed Nader Date: Tue, 24 Dec 2024 21:25:26 +0300 Subject: [PATCH 4/4] Added missing tests to the rest integration catalog --- tests/integration/test_rest_catalog.py | 59 +++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_rest_catalog.py b/tests/integration/test_rest_catalog.py index e634c4294..f3ccccfb6 100644 --- a/tests/integration/test_rest_catalog.py +++ b/tests/integration/test_rest_catalog.py @@ -217,6 +217,13 @@ def test_create_namespace_if_exists_409(catalog: RestCatalog, clean_up: Any) -> assert TEST_NAMESPACE_IDENTIFIER in catalog.list_namespaces() +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_list_namespaces_200(catalog: RestCatalog, clean_up: Any) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + assert catalog.list_namespaces() == [("default",), TEST_NAMESPACE_IDENTIFIER] + + @pytest.mark.integration @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) def test_create_namespace_409(catalog: RestCatalog, clean_up: Any) -> None: @@ -262,7 +269,6 @@ def test_load_namespace_properties_404(catalog: RestCatalog, clean_up: Any) -> N assert "Namespace does not exist" in str(e.value) -# Update Properties @pytest.mark.integration @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) def test_update_namespace_properties_200(catalog: RestCatalog, clean_up: Any) -> None: @@ -384,6 +390,33 @@ def test_load_table_from_self_identifier_200( assert actual.metadata.model_dump() == expected.metadata.model_dump() +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_list_tables_200(catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema) -> None: + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) + + assert TEST_TABLE_IDENTIFIER in catalog.list_tables(TEST_NAMESPACE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_list_tables_200_sigv4(catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema) -> None: + catalog = RestCatalog("rest", **{"uri": TEST_URI, "token": "some-jwt-token", "rest.sigv4-enabled": "true"}) + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + catalog.create_table(TEST_TABLE_IDENTIFIER, schema=table_schema_simple) + + assert TEST_TABLE_IDENTIFIER in catalog.list_tables(TEST_NAMESPACE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_list_tables_404(catalog: RestCatalog, clean_up: Any, table_schema_simple: Schema) -> None: + with pytest.raises(NoSuchNamespaceError) as e: + catalog.list_tables(TEST_NAMESPACE_IDENTIFIER) + assert "Namespace does not exist" in str(e.value) + + @pytest.mark.integration @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) def test_load_table_404(catalog: RestCatalog, clean_up: Any) -> None: @@ -783,6 +816,22 @@ def test_create_namespace_if_already_existing(catalog: RestCatalog, clean_up: An assert catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_list_views_200_sigv4(catalog: RestCatalog, clean_up: Any) -> None: + catalog = RestCatalog("rest", **{"uri": TEST_URI, "token": "some-jwt-token", "rest.sigv4-enabled": "true"}) + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + assert [] == catalog.list_views(TEST_NAMESPACE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_list_views_404(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(NoSuchNamespaceError) as e: + RestCatalog("rest", uri=TEST_URI, token="some-jwt-token").list_views(TEST_NAMESPACE_IDENTIFIER) + assert "Namespace does not exist" in str(e.value) + + @pytest.mark.integration @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) def test_drop_view_invalid_namespace(catalog: RestCatalog, clean_up: Any) -> None: @@ -820,6 +869,14 @@ def test_properties_sets_headers(catalog: RestCatalog, clean_up: Any) -> None: ) +@pytest.mark.integration +@pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) +def test_no_uri_supplied(catalog: RestCatalog, clean_up: Any) -> None: + with pytest.raises(KeyError) as e: + RestCatalog("production") + assert "uri" in str(e.value) + + @pytest.mark.integration @pytest.mark.parametrize("catalog,clean_up", [(pytest.lazy_fixture("session_catalog"), pytest.lazy_fixture("test_clean_up"))]) @pytest.mark.skip(reason="Not raising OAuthError")