Merge branch 'agis/test-fix' into agis/test-of-db/sc-50707/aarch64

CMakeLists.txt

@@ -46,8 +46,8 @@ if (NOT TileDB_FOUND)
         message(STATUS "Downloading TileDB default version ...")
         # Download latest release
         fetch_prebuilt_tiledb(
-                VERSION 2.26.2
-                RELLIST_HASH SHA256=86c19d7c5246cb18e370a4272cead63ea84bd651789842e618de4d57d4510522
+                VERSION 2.27.0
+                RELLIST_HASH SHA256=8056514b1949cdab19405376e32e299578491a6d3e953321d12d761f94dc19b9
         )
     endif()
     find_package(TileDB REQUIRED)

tiledb/__init__.py

@@ -21,7 +21,7 @@
 
 from tiledb.libtiledb import version as libtiledb_version
 
-if libtiledb_version()[0] == 2 and libtiledb_version()[1] >= 25:
+if libtiledb_version()[0] == 2 and libtiledb_version()[1] >= 26:
     from .current_domain import CurrentDomain
     from .ndrectangle import NDRectangle
 

tiledb/array_schema.py

@@ -14,7 +14,7 @@
 from .domain import Domain
 from .filter import Filter, FilterList
 
-if libtiledb_version()[0] == 2 and libtiledb_version()[1] >= 25:
+if libtiledb_version()[0] == 2 and libtiledb_version()[1] >= 26:
     from .current_domain import CurrentDomain
 
 _tiledb_order_to_string = {
@@ -388,7 +388,7 @@ def has_dim_label(self, name: str) -> bool:
         """
         return self._has_dim_label(self._ctx, name)
 
-    if libtiledb_version()[0] == 2 and libtiledb_version()[1] >= 25:
+    if libtiledb_version()[0] == 2 and libtiledb_version()[1] >= 26:
 
         @property
         def current_domain(self) -> CurrentDomain:

tiledb/cc/current_domain.cc

@@ -16,7 +16,7 @@ using namespace tiledbpy::common;
 namespace py = pybind11;
 
 void init_current_domain(py::module &m) {
-#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 25
+#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 26
   py::class_<NDRectangle>(m, "NDRectangle")
       .def(py::init<NDRectangle>())
 

tiledb/cc/enum.cc

@@ -175,7 +175,7 @@ void init_enums(py::module &m) {
       .value("TIFF", TILEDB_MIME_TIFF)
       .value("PDF", TILEDB_MIME_PDF);
 
-#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 25
+#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 26
   py::enum_<tiledb_current_domain_type_t>(m, "CurrentDomainType")
       .value("NDRECTANGLE", TILEDB_NDRECTANGLE);
 #endif

tiledb/cc/schema.cc

@@ -278,7 +278,7 @@ void init_schema(py::module &m) {
              ArraySchemaExperimental::add_enumeration(ctx, schema, enmr);
            })
 
-#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 25
+#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 26
       .def("_current_domain",
            [](const ArraySchema &schema, const Context &ctx) {
              return ArraySchemaExperimental::current_domain(ctx, schema);

tiledb/ctx.py

@@ -293,6 +293,20 @@ def save(self, uri: str):
         """
         self.save_to_file(uri)
 
+    def __reduce__(self):
+        """
+        Customize the pickling process by defining how to serialize
+        and reconstruct the Config object.
+        """
+        state = self.dict()
+        return (self.__class__, (), state)
+
+    def __setstate__(self, state):
+        """
+        Customize how the Config object is restored from its serialized state.
+        """
+        self.__init__(state)
+
 
 class ConfigKeys:
     """

tiledb/fragment.py

@@ -188,6 +188,9 @@ def __init__(self, fragments):
         self._fragments = fragments
         self._index = 0
 
+    def __iter__(self):
+        return self
+
     def __next__(self):
         if self._index < len(self._fragments):
             fi = FragmentInfo(self._fragments, self._index)

tiledb/schema_evolution.cc

@@ -100,7 +100,7 @@ void init_schema_evolution(py::module &m) {
              }
            })
 
-#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 25
+#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 26
       .def("expand_current_domain",
            [](ArraySchemaEvolution &inst, py::object current_domain_py) {
              tiledb_current_domain_t *current_domain_c =

tiledb/schema_evolution.py

@@ -52,7 +52,7 @@ def extend_enumeration(self, enmr: Enumeration):
 
         self.ase.extend_enumeration(enmr)
 
-    if libtiledb_version()[0] == 2 and libtiledb_version()[1] >= 25:
+    if libtiledb_version()[0] == 2 and libtiledb_version()[1] >= 26:
         from .current_domain import CurrentDomain
 
         def expand_current_domain(self, current_domain: CurrentDomain):

tiledb/tests/cc/test_group.py

@@ -28,7 +28,6 @@ def test_group_metadata(tmp_path):
     assert_array_equal(grp._get_metadata("flt", False)[0], flt_data)
     grp._close()
 
-    time.sleep(0.001)
     grp._open(lt.QueryType.WRITE)
     grp._delete_metadata("int")
     grp._close()

tiledb/tests/conftest.py

@@ -52,54 +52,6 @@ def pytest_configure(config):
     # default must be set here rather than globally
     pytest.tiledb_vfs = "file"
 
-    vfs_config(config)
-
-
-def vfs_config(pytestconfig):
-    vfs_config_override = {}
-
-    vfs = pytestconfig.getoption("vfs")
-    if vfs == "s3":
-        pytest.tiledb_vfs = "s3"
-
-        vfs_config_override.update(
-            {
-                "vfs.s3.endpoint_override": "localhost:9999",
-                "vfs.s3.aws_access_key_id": "minio",
-                "vfs.s3.aws_secret_access_key": "miniosecretkey",
-                "vfs.s3.scheme": "https",
-                "vfs.s3.verify_ssl": False,
-                "vfs.s3.use_virtual_addressing": False,
-            }
-        )
-
-    vfs_config_arg = pytestconfig.getoption("vfs-config", None)
-    if vfs_config_arg:
-        pass
-
-    tiledb._orig_ctx = tiledb.Ctx
-
-    def get_config(config):
-        final_config = {}
-        if isinstance(config, tiledb.Config):
-            final_config = config.dict()
-        elif config:
-            final_config = config
-
-        final_config.update(vfs_config_override)
-        return final_config
-
-    class PatchedCtx(tiledb.Ctx):
-        def __init__(self, config=None):
-            super().__init__(get_config(config))
-
-    class PatchedConfig(tiledb.Config):
-        def __init__(self, params=None):
-            super().__init__(get_config(params))
-
-    tiledb.Ctx = PatchedCtx
-    tiledb.Config = PatchedConfig
-
 
 @pytest.fixture(scope="function", autouse=True)
 def isolate_os_fork(original_os_fork):

tiledb/tests/test_context_and_config.py

@@ -1,4 +1,6 @@
+import io
 import os
+import pickle
 import subprocess
 import sys
 import xml
@@ -261,3 +263,20 @@ def test_config_repr_html(self):
             pytest.fail(
                 f"Could not parse config._repr_html_(). Saw {config._repr_html_()}"
             )
+
+    def test_config_pickle(self):
+        # test that Config can be pickled and unpickled
+        config = tiledb.Config(
+            {
+                "rest.use_refactored_array_open": "false",
+                "rest.use_refactored_array_open_and_query_submit": "true",
+                "vfs.azure.storage_account_name": "myaccount",
+            }
+        )
+        with io.BytesIO() as buf:
+            pickle.dump(config, buf)
+            buf.seek(0)
+            config2 = pickle.load(buf)
+
+            self.assertIsInstance(config2, tiledb.Config)
+            self.assertEqual(config2.dict(), config.dict())

tiledb/tests/test_current_domain.py

@@ -7,9 +7,9 @@
 import tiledb
 import tiledb.cc as lt
 
-if not (tiledb.libtiledb.version()[0] == 2 and tiledb.libtiledb.version()[1] >= 25):
+if not (tiledb.libtiledb.version()[0] == 2 and tiledb.libtiledb.version()[1] >= 26):
     pytest.skip(
-        "CurrentDomain is only available in TileDB 2.25 and later",
+        "CurrentDomain is only available in TileDB 2.26 and later",
         allow_module_level=True,
     )
 

tiledb/tests/test_fixes.py

@@ -364,26 +364,16 @@ class SOMA919Test(DiskTestCase):
     We've distilled @atolopko-czi's gist example using the TileDB-Py API directly.
     """
 
-    def run_test(self, use_timestamps):
+    def run_test(self):
         import tempfile
 
         import numpy as np
 
         import tiledb
 
         root_uri = tempfile.mkdtemp()
-
-        if use_timestamps:
-            group_ctx100 = tiledb.Ctx(
-                {
-                    "sm.group.timestamp_start": 100,
-                    "sm.group.timestamp_end": 100,
-                }
-            )
-            timestamp = 100
-        else:
-            group_ctx100 = tiledb.Ctx()
-            timestamp = None
+        group_ctx100 = tiledb.Ctx()
+        timestamp = None
 
         # create the group and add a dummy subgroup "causes_bug"
         tiledb.Group.create(root_uri, ctx=group_ctx100)
@@ -411,13 +401,12 @@ def run_test(self, use_timestamps):
         tiledb.libtiledb.version() < (2, 15, 0),
         reason="SOMA919 fix implemented in libtiledb 2.15",
     )
-    @pytest.mark.parametrize("use_timestamps", [True, False])
-    def test_soma919(self, use_timestamps):
+    def test_soma919(self):
         N = 100
         fails = 0
         for i in range(N):
             try:
-                self.run_test(use_timestamps)
+                self.run_test()
             except AssertionError:
                 fails += 1
         if fails > 0:

tiledb/tests/test_group.py

@@ -120,10 +120,7 @@ def test_move_group(self):
             ),
         ),
     )
-    @pytest.mark.parametrize("use_timestamps", [True, False])
-    def test_group_metadata(
-        self, int_data, flt_data, str_data, str_type, capfd, use_timestamps
-    ):
+    def test_group_metadata(self, int_data, flt_data, str_data, str_type, capfd):
         def values_equal(lhs, rhs):
             if isinstance(lhs, np.ndarray):
                 if not isinstance(rhs, np.ndarray):
@@ -139,13 +136,13 @@ def values_equal(lhs, rhs):
         grp_path = self.path("test_group_metadata")
         tiledb.Group.create(grp_path)
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 1} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(grp_path, "w", cfg) as grp:
             grp.meta["int"] = int_data
             grp.meta["flt"] = flt_data
             grp.meta["str"] = str_data
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 1} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(grp_path, "r", cfg) as grp:
             assert len(grp.meta) == 3
             assert "int" in grp.meta
@@ -162,11 +159,11 @@ def values_equal(lhs, rhs):
             assert "Type: DataType.INT" in metadata_dump
             assert f"Type: DataType.{str_type}" in metadata_dump
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 2} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(grp_path, "w", cfg) as grp:
             del grp.meta["int"]
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 2} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(grp_path, "r", cfg) as grp:
             assert len(grp.meta) == 2
             assert "int" not in grp.meta
@@ -373,8 +370,7 @@ class GroupMetadataTest(GroupTestCase):
             (np.array([1, 2, 3]), np.array([1.5, 2.5, 3.5]), np.array(["x"])),
         ),
     )
-    @pytest.mark.parametrize("use_timestamps", [True, False])
-    def test_group_metadata(self, int_data, flt_data, str_data, use_timestamps):
+    def test_group_metadata(self, int_data, flt_data, str_data):
         def values_equal(lhs, rhs):
             if isinstance(lhs, np.ndarray):
                 if not isinstance(rhs, np.ndarray):
@@ -390,13 +386,13 @@ def values_equal(lhs, rhs):
         grp_path = self.path("test_group_metadata")
         tiledb.Group.create(grp_path)
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 1} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(grp_path, "w", cfg) as grp:
             grp.meta["int"] = int_data
             grp.meta["flt"] = flt_data
             grp.meta["str"] = str_data
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 1} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(grp_path, "r", cfg) as grp:
             assert grp.meta.keys() == {"int", "flt", "str"}
             assert len(grp.meta) == 3
@@ -407,11 +403,11 @@ def values_equal(lhs, rhs):
             assert "str" in grp.meta
             assert values_equal(grp.meta["str"], str_data)
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 2} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(grp_path, "w", cfg) as grp:
             del grp.meta["int"]
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 2} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(grp_path, "r", cfg) as grp:
             assert len(grp.meta) == 2
             assert "int" not in grp.meta
@@ -641,21 +637,20 @@ def test_numpy(self, test_vals, ndarray):
         self.assert_metadata_roundtrip(grp.meta, test_vals)
         grp.close()
 
-    @pytest.mark.parametrize("use_timestamps", [True, False])
-    def test_consolidation_and_vac(self, use_timestamps):
+    def test_consolidation_and_vac(self):
         vfs = tiledb.VFS()
         path = self.path("test_consolidation_and_vac")
         tiledb.Group.create(path)
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 1} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(path, "w", cfg) as grp:
             grp.meta["meta"] = 1
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 2} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(path, "w", cfg) as grp:
             grp.meta["meta"] = 2
 
-        cfg = tiledb.Config({"sm.group.timestamp_end": 3} if use_timestamps else {})
+        cfg = tiledb.Config()
         with tiledb.Group(path, "w", cfg) as grp:
             grp.meta["meta"] = 3
 

tiledb/tests/test_libtiledb.py

@@ -823,8 +823,7 @@ def test_ncell_int(self):
                 assert_array_equal(T, R)
                 assert_array_equal(T, R.multi_index[0:2][""])
 
-    @pytest.mark.parametrize("use_timestamps", [True, False])
-    def test_open_with_timestamp(self, use_timestamps):
+    def test_open_with_timestamp(self):
         A = np.zeros(3)
 
         dom = tiledb.Domain(tiledb.Dim(domain=(0, 2), tile=3, dtype=np.int64))
@@ -841,15 +840,9 @@ def test_open_with_timestamp(self, use_timestamps):
             self.assertEqual(T[1], 0)
             self.assertEqual(T[2], 0)
 
-        if use_timestamps:
-            # sleep 200ms and write
-            time.sleep(0.2)
         with tiledb.DenseArray(self.path("foo"), mode="w") as T:
             T[0:1] = 1
 
-        if use_timestamps:
-            # sleep 200ms and write
-            time.sleep(0.2)
         with tiledb.DenseArray(self.path("foo"), mode="w") as T:
             T[1:2] = 2