diff --git a/MICROLIB.md b/MICROLIB.md index ef6d41d..866c065 100644 --- a/MICROLIB.md +++ b/MICROLIB.md @@ -29,6 +29,9 @@ String API: 3. [Open an existing StringsMmap](#open-an-existing-stringsmmap) 4. [Append new samples to a StringsMmap](#append-new-samples-to-a-stringsmmap) + +## Utils API + ### Wrapped The `Wrapped` class allows you to lazily apply a function @@ -38,7 +41,7 @@ For example: ```python import numpy as np import torch -from mmap_ninja.base import Wrapped +from mmap_ninja import Wrapped dataset = [np.array([1, 2, 3], np.array([-1, 10]))] wrapped = Wrapped(dataset, wrapper_fn=torch.tensor) @@ -54,7 +57,7 @@ For example: ```python import numpy as np import torch -from mmap_ninja.base import Wrapped +from mmap_ninja import Wrapped dataset = [np.array([1, 2, 3], np.array([-1, 10]))] wrapped = Wrapped(dataset, wrapper_fn=torch.tensor, copy_before_wrapper_fn=False) @@ -62,6 +65,8 @@ wrapped = Wrapped(dataset, wrapper_fn=torch.tensor, copy_before_wrapper_fn=False print(wrapped[14]) ``` +## Numpy API + ### Create a Numpy memmap from a Numpy array The `mmap_ninja.numpy` module provides utilities for @@ -71,10 +76,10 @@ this example: ```python import numpy as np -from mmap_ninja import numpy as np_ninja +import mmap_ninja arr = np.random.randn(200, 224, 224, 3) -np_ninja.from_ndarray('imgs_mmap', arr) +mmap_ninja.np_from_ndarray('imgs_mmap', arr) ``` ### Create a Numpy memmap from a generator @@ -92,9 +97,11 @@ from mmap_ninja import numpy as np_ninja from pathlib import Path from os import listdir +import mmap_ninja + imgs_dir = Path('./path_to_img_dir') -np_ninja.from_generator( +mmap_ninja.np_from_generator( out_dir='imgs_mmap', sample_generator=map(mpimg.imread, imgs_dir.iterdir()), batch_size=32, @@ -133,6 +140,8 @@ np_ninja.extend(memmap, np.arange(11, 13)) np_ninja.extend_dir("growable", np.arange(14, 16)) ``` +## Ragged API + ### Create a RaggedMmap from list of samples If your samples are of different shapes, then you should use @@ -223,6 +232,8 @@ new_samples = [np.array([123, -1]), np.array([-1, 0, 123, 92, 12])] mmap.extend(new_samples) ``` +## String API + ### Create a StringsMmap from list of strings To create a `StringsMmap` from a list of strings, use the diff --git a/mmap_ninja/tests/test_base.py b/mmap_ninja/tests/test_base.py index 3f4cb86..c72af4f 100644 --- a/mmap_ninja/tests/test_base.py +++ b/mmap_ninja/tests/test_base.py @@ -1,6 +1,6 @@ import numpy as np -from mmap_ninja import base +from mmap_ninja import base, Wrapped def test_int_conversions(): @@ -45,7 +45,7 @@ def test_shape_file(tmp_path): def test_wrapper(): arr = np.array([1, 2, 3]) - wrapped = base.Wrapped(arr, lambda x: -x) + wrapped = Wrapped(arr, lambda x: -x) assert wrapped[0] == -1 assert wrapped[1] == -2 assert wrapped[2] == -3 @@ -54,7 +54,7 @@ def test_wrapper(): def test_wrapper_skip_copy(): arr = np.array([1, 2, 3]) - wrapped = base.Wrapped(arr, lambda x: -x, copy_before_wrapper_fn=False) + wrapped = Wrapped(arr, lambda x: -x, copy_before_wrapper_fn=False) assert wrapped[0] == -1 assert wrapped[1] == -2 assert wrapped[2] == -3 diff --git a/mmap_ninja/tests/test_numpy.py b/mmap_ninja/tests/test_numpy.py index 1b19248..bf75e3d 100644 --- a/mmap_ninja/tests/test_numpy.py +++ b/mmap_ninja/tests/test_numpy.py @@ -2,7 +2,6 @@ import stat import numpy as np - from mmap_ninja import generic, numpy as np_ninja