diff --git a/asdf/_asdf.py b/asdf/_asdf.py index 3a5d4a3fb..434733f87 100644 --- a/asdf/_asdf.py +++ b/asdf/_asdf.py @@ -57,7 +57,7 @@ def __init__( extensions=None, version=None, ignore_unrecognized_tag=False, - memmap=True, + memmap=False, lazy_load=True, custom_schema=None, ): @@ -88,7 +88,7 @@ def __init__( memmap : bool, optional When `True`, when reading files, attempt to memmap underlying data - arrays when possible. Defaults to ``True``. + arrays when possible. Defaults to ``False``. lazy_load : bool, optional When `True` and the underlying file handle is seekable, data @@ -1510,7 +1510,7 @@ def open_asdf( extensions=None, ignore_unrecognized_tag=False, _force_raw_types=False, - memmap=True, + memmap=False, lazy_tree=NotSet, lazy_load=True, custom_schema=None, @@ -1549,7 +1549,7 @@ def open_asdf( memmap : bool, optional When `True`, when reading files, attempt to memmap underlying data - arrays when possible. Defaults to ``True``. + arrays when possible. Defaults to ``False``. lazy_load : bool, optional When `True` and the underlying file handle is seekable, data diff --git a/asdf/_tests/tags/core/tests/test_ndarray.py b/asdf/_tests/tags/core/tests/test_ndarray.py index 08a2202fe..d0f4b80ab 100644 --- a/asdf/_tests/tags/core/tests/test_ndarray.py +++ b/asdf/_tests/tags/core/tests/test_ndarray.py @@ -976,7 +976,7 @@ def test_memmap_write(tmp_path): tmpfile = str(tmp_path / "data.asdf") tree = {"data": np.zeros(100)} - with asdf.AsdfFile(tree) as af: + with asdf.AsdfFile(tree, memmap=False) as af: # Make sure we're actually writing to an internal array for this test af.write_to(tmpfile, all_array_storage="internal") @@ -1002,7 +1002,7 @@ def test_readonly(tmp_path): af.write_to(tmpfile, all_array_storage="internal") # Opening in read mode (the default) should mean array is readonly - with asdf.open(tmpfile) as af: + with asdf.open(tmpfile, memmap=True) as af: assert af["data"].flags.writeable is False with pytest.raises(ValueError, match=r"assignment destination is read-only"): af["data"][0] = 41 @@ -1046,7 +1046,7 @@ def test_block_data_change(pad_blocks, tmp_path): with asdf.AsdfFile(tree) as af: af.write_to(tmpfile, pad_blocks=pad_blocks) - with asdf.open(tmpfile, mode="rw") as af: + with asdf.open(tmpfile, mode="rw", memmap=True) as af: assert np.all(af.tree["data"] == 0) array_before = af.tree["data"].__array__() af.tree["data"][:5] = 1 diff --git a/asdf/_tests/test_api.py b/asdf/_tests/test_api.py index b33059511..6c849db0c 100644 --- a/asdf/_tests/test_api.py +++ b/asdf/_tests/test_api.py @@ -60,7 +60,7 @@ def test_open_readonly(tmp_path): os.chmod(tmpfile, 0o440) assert os.access(tmpfile, os.W_OK) is False - with asdf.open(tmpfile) as af: + with asdf.open(tmpfile, memmap=True) as af: assert af["baz"].flags.writeable is False with pytest.raises(PermissionError, match=r".* Permission denied: .*"), asdf.open(tmpfile, mode="rw"): diff --git a/asdf/_tests/test_array_blocks.py b/asdf/_tests/test_array_blocks.py index e8d8cf3c4..e162e668d 100644 --- a/asdf/_tests/test_array_blocks.py +++ b/asdf/_tests/test_array_blocks.py @@ -811,9 +811,11 @@ def filename_with_array(tmp_path_factory): @pytest.mark.parametrize( "open_kwargs,should_memmap", [ - ({}, True), - ({"memmap": True}, True), - ({"memmap": False}, False), + ({}, False), + ({"lazy_load": True, "memmap": True}, True), + ({"lazy_load": False, "memmap": True}, True), + ({"lazy_load": True, "memmap": False}, False), + ({"lazy_load": False, "memmap": False}, False), ], ) def test_open_no_memmap(filename_with_array, open_kwargs, should_memmap): @@ -823,7 +825,7 @@ def test_open_no_memmap(filename_with_array, open_kwargs, should_memmap): default (no kwargs) memmap """ - with asdf.open(filename_with_array, lazy_load=False, **open_kwargs) as af: + with asdf.open(filename_with_array, **open_kwargs) as af: array = af.tree["array"] if should_memmap: assert isinstance(array.base, np.memmap) diff --git a/asdf/_tests/test_generic_io.py b/asdf/_tests/test_generic_io.py index fea00295f..25158b632 100644 --- a/asdf/_tests/test_generic_io.py +++ b/asdf/_tests/test_generic_io.py @@ -110,7 +110,7 @@ def get_read_fd(): f.read(0) return f - with _roundtrip(tree, get_write_fd, get_read_fd) as ff: + with _roundtrip(tree, get_write_fd, get_read_fd, read_options={"memmap": True}) as ff: assert len(ff._blocks.blocks) == 2 assert isinstance(ff._blocks.blocks[0].cached_data, np.memmap) @@ -133,7 +133,7 @@ def get_read_fd(): assert f._uri == path.as_uri() return f - with _roundtrip(tree, get_write_fd, get_read_fd) as ff: + with _roundtrip(tree, get_write_fd, get_read_fd, read_options={"memmap": True}) as ff: assert len(ff._blocks.blocks) == 2 assert isinstance(ff._blocks.blocks[0].cached_data, np.memmap) @@ -173,7 +173,7 @@ def get_read_fd(): assert f._uri == path.as_uri() return f - with _roundtrip(tree, get_write_fd, get_read_fd) as ff: + with _roundtrip(tree, get_write_fd, get_read_fd, read_options={"memmap": True}) as ff: assert len(ff._blocks.blocks) == 2 assert isinstance(ff._blocks.blocks[0].cached_data, np.memmap) ff.tree["science_data"][0] = 42 diff --git a/asdf/_tests/test_reference.py b/asdf/_tests/test_reference.py index f618865fd..37dee54f4 100644 --- a/asdf/_tests/test_reference.py +++ b/asdf/_tests/test_reference.py @@ -56,9 +56,6 @@ def do_asserts(ff): assert_array_equal(ff.tree["science_data"], exttree["cool_stuff"]["a"]) assert len(ff._external_asdf_by_uri) == 1 - with pytest.raises((ValueError, RuntimeError), match=r"assignment destination is read-only"): - # Assignment destination is readonly - ff.tree["science_data"][0] = 42 assert_array_equal(ff.tree["science_data2"], exttree["cool_stuff"]["a"]) assert len(ff._external_asdf_by_uri) == 2 diff --git a/changes/1801.general.rst b/changes/1801.general.rst new file mode 100644 index 000000000..36c8bc4f7 --- /dev/null +++ b/changes/1801.general.rst @@ -0,0 +1 @@ +Set ``memmap=False`` to default for ``asdf.open`` and ``AsdfFile.__init__``. diff --git a/docs/asdf/whats_new.rst b/docs/asdf/whats_new.rst index 8f9328cbf..5db2ecdc1 100644 --- a/docs/asdf/whats_new.rst +++ b/docs/asdf/whats_new.rst @@ -46,6 +46,13 @@ Removed API New Defaults ------------ +.. _whats_new_4.0.0_memmap: + +Memory mapping disabled by default +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Calls to ``asdf.open() and ``AsdfFile()`` will now default to ``memmap=False``, disabling memory mapping of arrays by default. + .. _whats_new_4.0.0_validation: Validation