From c7a68a4c0fcccc5fe054e1294e236a485941b9bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Mon, 6 May 2024 15:54:13 +0200 Subject: [PATCH 01/21] add functionality to specify temp directory location when creating memory mapped temp arrays this allows you to also utilize memory mapped temp arrays in multithreaded processes by explicitly passing the directory name without needing to rely on the global TEMP_DIR_NAME variable still being accurate --- alphabase/io/tempmmap.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index f8be53bc..602c47bb 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -56,7 +56,7 @@ def redefine_temp_location(path): return TEMP_DIR_NAME -def array(shape: tuple, dtype: np.dtype) -> np.ndarray: +def array(shape: tuple, dtype: np.dtype, tmp_dir_name : str = None) -> np.ndarray: """Create a writable temporary mmapped array. Parameters @@ -65,12 +65,23 @@ def array(shape: tuple, dtype: np.dtype) -> np.ndarray: A tuple with the shape of the array. dtype : type The np.dtype of the array. + tmp_dir_name : str, optional + If specified the memory mapped array will be created in this directory. Defaults to None. Returns ------- type A writable temporary mmapped array. """ + # redefine the temporary directory if a new location is given otherwise read from global variable + # this allows you to ensure that the correct temp directory location is used when working with multiple threads + if tmp_dir_name is not None: + if os.path.isdir(tmp_dir_name): #ensure that directory exists + TEMP_DIR_NAME = tmp_dir_name + else: + raise ValueError( + "The directory in which the file should be created does not exist." + ) temp_file_name = os.path.join( TEMP_DIR_NAME, f"temp_mmap_{np.random.randint(2**63)}.hdf" ) @@ -87,7 +98,7 @@ def array(shape: tuple, dtype: np.dtype) -> np.ndarray: ).reshape(shape) -def create_empty_mmap(shape: tuple, dtype: np.dtype, path: str = None, overwrite=False): +def create_empty_mmap(shape: tuple, dtype: np.dtype, path: str = None, overwrite: bool = False, tmp_dir_name: str = None): """Initialize a new HDF5 file compatible with mmap. Returns the path to the initialized file. File can be mapped using the mmap_array_from_path function. @@ -104,12 +115,23 @@ def create_empty_mmap(shape: tuple, dtype: np.dtype, path: str = None, overwrite overwrite : bool , optional If True the file will be overwritten if it already exists. Defaults to False. + tmp_dir_name : str, optional + If specified the memory mapped array will be created in this directory. Returns ------- str path to the newly created file. """ + # redefine the temporary directory if a new location is given otherwise read from global variable + # this allows you to ensure that the correct temp directory location is used when working with multiple threads + if tmp_dir_name is not None: + if os.path.isdir(tmp_dir_name): #ensure that directory exists + TEMP_DIR_NAME = tmp_dir_name + else: + raise ValueError( + "The directory in which the file should be created does not exist." + ) # if path does not exist generate a random file name in the TEMP directory if path is None: From 7b652fdaef4fd088a64c4c17042c4a389cfd9eb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Mon, 6 May 2024 16:21:23 +0200 Subject: [PATCH 02/21] ruff formating --- alphabase/io/tempmmap.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index 602c47bb..dea37952 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -56,7 +56,7 @@ def redefine_temp_location(path): return TEMP_DIR_NAME -def array(shape: tuple, dtype: np.dtype, tmp_dir_name : str = None) -> np.ndarray: +def array(shape: tuple, dtype: np.dtype, tmp_dir_name: str = None) -> np.ndarray: """Create a writable temporary mmapped array. Parameters @@ -76,12 +76,12 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_name : str = None) -> np.ndarra # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_name is not None: - if os.path.isdir(tmp_dir_name): #ensure that directory exists + if os.path.isdir(tmp_dir_name): # ensure that directory exists TEMP_DIR_NAME = tmp_dir_name else: raise ValueError( - "The directory in which the file should be created does not exist." - ) + "The directory in which the file should be created does not exist." + ) temp_file_name = os.path.join( TEMP_DIR_NAME, f"temp_mmap_{np.random.randint(2**63)}.hdf" ) @@ -98,7 +98,13 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_name : str = None) -> np.ndarra ).reshape(shape) -def create_empty_mmap(shape: tuple, dtype: np.dtype, path: str = None, overwrite: bool = False, tmp_dir_name: str = None): +def create_empty_mmap( + shape: tuple, + dtype: np.dtype, + path: str = None, + overwrite: bool = False, + tmp_dir_name: str = None + ): """Initialize a new HDF5 file compatible with mmap. Returns the path to the initialized file. File can be mapped using the mmap_array_from_path function. @@ -126,12 +132,12 @@ def create_empty_mmap(shape: tuple, dtype: np.dtype, path: str = None, overwrite # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_name is not None: - if os.path.isdir(tmp_dir_name): #ensure that directory exists + if os.path.isdir(tmp_dir_name): # ensure that directory exists TEMP_DIR_NAME = tmp_dir_name else: raise ValueError( - "The directory in which the file should be created does not exist." - ) + "The directory in which the file should be created does not exist." + ) # if path does not exist generate a random file name in the TEMP directory if path is None: From a0be3359d8e9af6f8959490ca72e26bc745ae6b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Mon, 6 May 2024 16:23:46 +0200 Subject: [PATCH 03/21] ruff formatting part 2 --- alphabase/io/tempmmap.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index dea37952..6241cd34 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -76,7 +76,7 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_name: str = None) -> np.ndarray # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_name is not None: - if os.path.isdir(tmp_dir_name): # ensure that directory exists + if os.path.isdir(tmp_dir_name): # ensure that directory exists TEMP_DIR_NAME = tmp_dir_name else: raise ValueError( @@ -99,12 +99,12 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_name: str = None) -> np.ndarray def create_empty_mmap( - shape: tuple, - dtype: np.dtype, - path: str = None, - overwrite: bool = False, - tmp_dir_name: str = None - ): + shape: tuple, + dtype: np.dtype, + path: str = None, + overwrite: bool = False, + tmp_dir_name: str = None +): """Initialize a new HDF5 file compatible with mmap. Returns the path to the initialized file. File can be mapped using the mmap_array_from_path function. @@ -132,7 +132,7 @@ def create_empty_mmap( # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_name is not None: - if os.path.isdir(tmp_dir_name): # ensure that directory exists + if os.path.isdir(tmp_dir_name): # ensure that directory exists TEMP_DIR_NAME = tmp_dir_name else: raise ValueError( From 08a90e1b65daa8f461a3bc9bd4f361a37c1350bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Wed, 8 May 2024 12:01:25 +0200 Subject: [PATCH 04/21] address review changes - updated name to temp_dir_abs_path - moved code to check if directory exists to seperate function --- alphabase/io/tempmmap.py | 65 +++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index 6241cd34..81f14956 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -22,6 +22,35 @@ "and might need to be triggered manually!" ) +def _check_temp_dir_existence(abs_path: str) -> str: + """ + Check if the directory to which the temp arrays should be written exists. If not raise a value error. + + Parameters + ---------- + abs_path : str + The absolute path to the new temporary directory. + + Returns + ------ + str + The directory path if it exists. + """ + #ensure that the path exists + if os.path.exists(abs_path): + # ensure that the path points to a directory + if os.path.isdir(abs_path): + TEMP_DIR_NAME = abs_path + return(TEMP_DIR_NAME) + else: + raise ValueError( + f"The path {abs_path} does not point to a directory." + ) + else: + raise ValueError( + f"The directory {abs_path} in which the file should be created does not exist." + ) + def redefine_temp_location(path): """ @@ -56,7 +85,7 @@ def redefine_temp_location(path): return TEMP_DIR_NAME -def array(shape: tuple, dtype: np.dtype, tmp_dir_name: str = None) -> np.ndarray: +def array(shape: tuple, dtype: np.dtype, tmp_dir_abs_path: str = None) -> np.ndarray: """Create a writable temporary mmapped array. Parameters @@ -65,23 +94,22 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_name: str = None) -> np.ndarray A tuple with the shape of the array. dtype : type The np.dtype of the array. - tmp_dir_name : str, optional - If specified the memory mapped array will be created in this directory. Defaults to None. + tmp_dir_abs_path : str, optional + If specified the memory mapped array will be created in this directory. + An absolute path is expected. + Defaults to None. If not specified the global TEMP_DIR_NAME location will be used. Returns ------- type A writable temporary mmapped array. """ + # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads - if tmp_dir_name is not None: - if os.path.isdir(tmp_dir_name): # ensure that directory exists - TEMP_DIR_NAME = tmp_dir_name - else: - raise ValueError( - "The directory in which the file should be created does not exist." - ) + if tmp_dir_abs_path is not None: + TEMP_DIR_NAME = _check_temp_dir_existence(tmp_dir_abs_path) + temp_file_name = os.path.join( TEMP_DIR_NAME, f"temp_mmap_{np.random.randint(2**63)}.hdf" ) @@ -103,7 +131,7 @@ def create_empty_mmap( dtype: np.dtype, path: str = None, overwrite: bool = False, - tmp_dir_name: str = None + tmp_dir_abs_path: str = None ): """Initialize a new HDF5 file compatible with mmap. Returns the path to the initialized file. File can be mapped using the mmap_array_from_path function. @@ -117,12 +145,12 @@ def create_empty_mmap( path : str, optional The path to the file that should be created. Defaults to None. - If None a random file name will be generated. + If None a random file name will be generated in the default tempdir location. overwrite : bool , optional If True the file will be overwritten if it already exists. Defaults to False. - tmp_dir_name : str, optional - If specified the memory mapped array will be created in this directory. + tmp_dir_abs_path : str, optional + If specified the default tempdir location will be updated to this path. Defaults to None. An absolute path to a directory is expected. Returns ------- @@ -131,13 +159,8 @@ def create_empty_mmap( """ # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads - if tmp_dir_name is not None: - if os.path.isdir(tmp_dir_name): # ensure that directory exists - TEMP_DIR_NAME = tmp_dir_name - else: - raise ValueError( - "The directory in which the file should be created does not exist." - ) + if tmp_dir_abs_path is not None: + TEMP_DIR_NAME = _check_temp_dir_existence(tmp_dir_abs_path) # if path does not exist generate a random file name in the TEMP directory if path is None: From 63135f4696297928620c661e6366e7d9401ba03c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Wed, 8 May 2024 12:02:40 +0200 Subject: [PATCH 05/21] fix punctuation mistakes in generated warning message --- alphabase/io/tempmmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index 81f14956..3f404b37 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -69,7 +69,7 @@ def redefine_temp_location(path): global _TEMP_DIR, _TEMP_DIR, TEMP_DIR_NAME logging.warning( - f"""Folder {TEMP_DIR_NAME} with temp mmap arrays is being deleted.All existing temp mmapp arrays will be unusable!""" + f"""Folder {TEMP_DIR_NAME} with temp mmap arrays is being deleted. All existing temp mmapp arrays will be unusable!""" ) # cleaup old temporary directory From 4ae6339476692df6cb93354eae085231a731c9de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Wed, 8 May 2024 12:02:59 +0200 Subject: [PATCH 06/21] ensure consistent directory naming convention between functions --- alphabase/io/tempmmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index 3f404b37..092fd790 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -77,7 +77,7 @@ def redefine_temp_location(path): del _TEMP_DIR # create new tempfile at desired location - _TEMP_DIR = tempfile.TemporaryDirectory(prefix=os.path.join(path, "temp_mmap")) + _TEMP_DIR = tempfile.TemporaryDirectory(prefix=os.path.join(path, "temp_mmap_")) TEMP_DIR_NAME = _TEMP_DIR.name logging.warning( f"""New temp folder location. Temp mmap arrays are written to {TEMP_DIR_NAME}. Cleanup of this folder is OS dependant, and might need to be triggered manually!""" From 88bba016664ebe4fde2e539a1eff77539be87dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Wed, 8 May 2024 12:03:55 +0200 Subject: [PATCH 07/21] add _check_file_location function move code to check if file path for creating a new empty temp directory is valid to a seperate helper function and update variable name to be more understandable --- alphabase/io/tempmmap.py | 61 ++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index 092fd790..e37fe3aa 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -51,6 +51,44 @@ def _check_temp_dir_existence(abs_path: str) -> str: f"The directory {abs_path} in which the file should be created does not exist." ) +def _check_file_location(abs_file_path: str, overwrite = False) -> str: + """ + Check if the path specified for the new temporary file is valid. If not raise a value error. + + Valid file paths need to: + 1. be contained in directories that exist + 2. end in .hdf + 3. not exist if overwrite is set to False + + Parameters + ---------- + abs_path : str + The absolute path to the new temporary file. + + Returns + ------ + str + The file path if it is valid. + """ + #check overwrite status and existence of file + if not overwrite: + if os.path.exists(abs_file_path): + raise ValueError( + "The file already exists. Set overwrite to True to overwrite the file or choose a different name." + ) + + #ensure that the filename conforms to the naming convention + if not os.path.basename.endswith(".hdf"): + raise ValueError("The chosen file name needs to end with .hdf") + + #ensure that the directory in which the file should be created exists + if os.path.isdir(os.path.commonpath(abs_file_path)): + return(abs_file_path) + else: + raise ValueError( + f"The directory {os.path.commonpath(abs_file_path)} in which the file should be created does not exist." + ) + def redefine_temp_location(path): """ @@ -129,7 +167,7 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_abs_path: str = None) -> np.nda def create_empty_mmap( shape: tuple, dtype: np.dtype, - path: str = None, + file_path: str = None, overwrite: bool = False, tmp_dir_abs_path: str = None ): @@ -142,8 +180,8 @@ def create_empty_mmap( A tuple with the shape of the array. dtype : type The np.dtype of the array. - path : str, optional - The path to the file that should be created. + file_path : str, optional + The absolute path to the file that should be created. This includes the file name. Defaults to None. If None a random file name will be generated in the default tempdir location. overwrite : bool , optional @@ -163,25 +201,12 @@ def create_empty_mmap( TEMP_DIR_NAME = _check_temp_dir_existence(tmp_dir_abs_path) # if path does not exist generate a random file name in the TEMP directory - if path is None: + if file_path is None: temp_file_name = os.path.join( TEMP_DIR_NAME, f"temp_mmap_{np.random.randint(2**63)}.hdf" ) else: - # check that if overwrite is false the file does not already exist - if not overwrite: - if os.path.exists(path): - raise ValueError( - "The file already exists. Set overwrite to True to overwrite the file or choose a different name." - ) - if not os.path.basename.endswith(".hdf"): - raise ValueError("The chosen file name needs to end with .hdf") - if os.path.isdir(os.path.commonpath(path)): - temp_file_name = path - else: - raise ValueError( - "The directory in which the file should be created does not exist." - ) + temp_file_name= _check_file_location(file_path: str, overwrite = False) with h5py.File(temp_file_name, "w") as hdf_file: array = hdf_file.create_dataset("array", shape=shape, dtype=dtype) From 0eef46cce30c05ba36e3f3004f3376a19fdc0cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Wed, 8 May 2024 13:01:34 +0200 Subject: [PATCH 08/21] fix missing bracket --- alphabase/io/tempmmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index e37fe3aa..356b3938 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -206,7 +206,7 @@ def create_empty_mmap( TEMP_DIR_NAME, f"temp_mmap_{np.random.randint(2**63)}.hdf" ) else: - temp_file_name= _check_file_location(file_path: str, overwrite = False) + temp_file_name= _check_file_location(file_path, overwrite = False) with h5py.File(temp_file_name, "w") as hdf_file: array = hdf_file.create_dataset("array", shape=shape, dtype=dtype) From a42ec3e4e42ccd74e635465e312b05a87fead9bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Mon, 13 May 2024 16:55:05 +0200 Subject: [PATCH 09/21] ruff formatting --- alphabase/io/tempmmap.py | 52 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index 356b3938..c7d5ba1f 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -22,6 +22,7 @@ "and might need to be triggered manually!" ) + def _check_temp_dir_existence(abs_path: str) -> str: """ Check if the directory to which the temp arrays should be written exists. If not raise a value error. @@ -33,25 +34,24 @@ def _check_temp_dir_existence(abs_path: str) -> str: Returns ------ - str + str The directory path if it exists. """ - #ensure that the path exists + # ensure that the path exists if os.path.exists(abs_path): # ensure that the path points to a directory - if os.path.isdir(abs_path): + if os.path.isdir(abs_path): TEMP_DIR_NAME = abs_path - return(TEMP_DIR_NAME) + return TEMP_DIR_NAME else: - raise ValueError( - f"The path {abs_path} does not point to a directory." - ) + raise ValueError(f"The path {abs_path} does not point to a directory.") else: raise ValueError( f"The directory {abs_path} in which the file should be created does not exist." ) -def _check_file_location(abs_file_path: str, overwrite = False) -> str: + +def _check_file_location(abs_file_path: str, overwrite=False) -> str: """ Check if the path specified for the new temporary file is valid. If not raise a value error. @@ -67,28 +67,28 @@ def _check_file_location(abs_file_path: str, overwrite = False) -> str: Returns ------ - str + str The file path if it is valid. """ - #check overwrite status and existence of file + # check overwrite status and existence of file if not overwrite: if os.path.exists(abs_file_path): - raise ValueError( - "The file already exists. Set overwrite to True to overwrite the file or choose a different name." - ) + raise ValueError( + "The file already exists. Set overwrite to True to overwrite the file or choose a different name." + ) - #ensure that the filename conforms to the naming convention + # ensure that the filename conforms to the naming convention if not os.path.basename.endswith(".hdf"): raise ValueError("The chosen file name needs to end with .hdf") - - #ensure that the directory in which the file should be created exists + + # ensure that the directory in which the file should be created exists if os.path.isdir(os.path.commonpath(abs_file_path)): - return(abs_file_path) + return abs_file_path else: raise ValueError( f"The directory {os.path.commonpath(abs_file_path)} in which the file should be created does not exist." ) - + def redefine_temp_location(path): """ @@ -133,7 +133,7 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_abs_path: str = None) -> np.nda dtype : type The np.dtype of the array. tmp_dir_abs_path : str, optional - If specified the memory mapped array will be created in this directory. + If specified the memory mapped array will be created in this directory. An absolute path is expected. Defaults to None. If not specified the global TEMP_DIR_NAME location will be used. @@ -147,7 +147,7 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_abs_path: str = None) -> np.nda # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_abs_path is not None: TEMP_DIR_NAME = _check_temp_dir_existence(tmp_dir_abs_path) - + temp_file_name = os.path.join( TEMP_DIR_NAME, f"temp_mmap_{np.random.randint(2**63)}.hdf" ) @@ -165,11 +165,11 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_abs_path: str = None) -> np.nda def create_empty_mmap( - shape: tuple, - dtype: np.dtype, - file_path: str = None, - overwrite: bool = False, - tmp_dir_abs_path: str = None + shape: tuple, + dtype: np.dtype, + file_path: str = None, + overwrite: bool = False, + tmp_dir_abs_path: str = None, ): """Initialize a new HDF5 file compatible with mmap. Returns the path to the initialized file. File can be mapped using the mmap_array_from_path function. @@ -206,7 +206,7 @@ def create_empty_mmap( TEMP_DIR_NAME, f"temp_mmap_{np.random.randint(2**63)}.hdf" ) else: - temp_file_name= _check_file_location(file_path, overwrite = False) + temp_file_name = _check_file_location(file_path, overwrite=False) with h5py.File(temp_file_name, "w") as hdf_file: array = hdf_file.create_dataset("array", shape=shape, dtype=dtype) From 70138d02fffd6b9c02b5a5660021325c330d0944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Wed, 15 May 2024 17:33:47 +0200 Subject: [PATCH 10/21] ensure that TEMP_DIR_NAME is considered global in cases where no abs_path is provided this otherwise throws an error. --- alphabase/io/tempmmap.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index c7d5ba1f..c896b886 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -143,6 +143,8 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_abs_path: str = None) -> np.nda A writable temporary mmapped array. """ + global TEMP_DIR_NAME + # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_abs_path is not None: @@ -195,6 +197,8 @@ def create_empty_mmap( str path to the newly created file. """ + global TEMP_DIR_NAME + # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_abs_path is not None: From 4f39dc0d2102378320766001cf3ce5173ceed53f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Thu, 23 May 2024 11:29:15 +0200 Subject: [PATCH 11/21] replace _check_temp_dir_existance with _change_temp_dir_location rename function to better reflect what the code does --- alphabase/io/tempmmap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index c896b886..f57b3f61 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -23,7 +23,7 @@ ) -def _check_temp_dir_existence(abs_path: str) -> str: +def _change_temp_dir_location(abs_path: str) -> str: """ Check if the directory to which the temp arrays should be written exists. If not raise a value error. @@ -148,7 +148,7 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_abs_path: str = None) -> np.nda # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_abs_path is not None: - TEMP_DIR_NAME = _check_temp_dir_existence(tmp_dir_abs_path) + TEMP_DIR_NAME = _change_temp_dir_location(tmp_dir_abs_path) temp_file_name = os.path.join( TEMP_DIR_NAME, f"temp_mmap_{np.random.randint(2**63)}.hdf" @@ -198,11 +198,11 @@ def create_empty_mmap( path to the newly created file. """ global TEMP_DIR_NAME - + # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_abs_path is not None: - TEMP_DIR_NAME = _check_temp_dir_existence(tmp_dir_abs_path) + TEMP_DIR_NAME = _change_temp_dir_location(tmp_dir_abs_path) # if path does not exist generate a random file name in the TEMP directory if file_path is None: From 0193779a6fa94584eb4909373714a6d65a3ada54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Thu, 23 May 2024 11:29:39 +0200 Subject: [PATCH 12/21] rename _check_file_location to _get_file_location name better reflects what the code does --- alphabase/io/tempmmap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index f57b3f61..45aa20ea 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -51,7 +51,7 @@ def _change_temp_dir_location(abs_path: str) -> str: ) -def _check_file_location(abs_file_path: str, overwrite=False) -> str: +def _get_file_location(abs_file_path: str, overwrite=False) -> str: """ Check if the path specified for the new temporary file is valid. If not raise a value error. @@ -210,7 +210,7 @@ def create_empty_mmap( TEMP_DIR_NAME, f"temp_mmap_{np.random.randint(2**63)}.hdf" ) else: - temp_file_name = _check_file_location(file_path, overwrite=False) + temp_file_name = _get_file_location(file_path, overwrite=False) with h5py.File(temp_file_name, "w") as hdf_file: array = hdf_file.create_dataset("array", shape=shape, dtype=dtype) From 5f55d6929dbec9866822dc3199ccd8f88ea84bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Thu, 23 May 2024 15:36:10 +0200 Subject: [PATCH 13/21] move definition of global parameter into _change_temp__dir_location function --- alphabase/io/tempmmap.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index 45aa20ea..38258e92 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -25,24 +25,22 @@ def _change_temp_dir_location(abs_path: str) -> str: """ - Check if the directory to which the temp arrays should be written exists. If not raise a value error. + Check if the directory to which the temp arrays should be written exists, if so defines this as the new temp dir location. If not raise a value error. Parameters ---------- abs_path : str The absolute path to the new temporary directory. - Returns - ------ - str - The directory path if it exists. """ + + global TEMP_DIR_NAME + # ensure that the path exists if os.path.exists(abs_path): # ensure that the path points to a directory if os.path.isdir(abs_path): TEMP_DIR_NAME = abs_path - return TEMP_DIR_NAME else: raise ValueError(f"The path {abs_path} does not point to a directory.") else: @@ -148,7 +146,7 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_abs_path: str = None) -> np.nda # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_abs_path is not None: - TEMP_DIR_NAME = _change_temp_dir_location(tmp_dir_abs_path) + _change_temp_dir_location(tmp_dir_abs_path) temp_file_name = os.path.join( TEMP_DIR_NAME, f"temp_mmap_{np.random.randint(2**63)}.hdf" @@ -202,7 +200,7 @@ def create_empty_mmap( # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_abs_path is not None: - TEMP_DIR_NAME = _change_temp_dir_location(tmp_dir_abs_path) + _change_temp_dir_location(tmp_dir_abs_path) # if path does not exist generate a random file name in the TEMP directory if file_path is None: From bac4ba7bd1d4992681130be3b978a48d72dc8d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Thu, 23 May 2024 15:37:06 +0200 Subject: [PATCH 14/21] remove duplicate variable definition statement in global and repetitive code --- alphabase/io/tempmmap.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index 38258e92..661a37f2 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -22,7 +22,6 @@ "and might need to be triggered manually!" ) - def _change_temp_dir_location(abs_path: str) -> str: """ Check if the directory to which the temp arrays should be written exists, if so defines this as the new temp dir location. If not raise a value error. @@ -48,7 +47,6 @@ def _change_temp_dir_location(abs_path: str) -> str: f"The directory {abs_path} in which the file should be created does not exist." ) - def _get_file_location(abs_file_path: str, overwrite=False) -> str: """ Check if the path specified for the new temporary file is valid. If not raise a value error. @@ -103,7 +101,8 @@ def redefine_temp_location(path): """ - global _TEMP_DIR, _TEMP_DIR, TEMP_DIR_NAME + global _TEMP_DIR, TEMP_DIR_NAME + logging.warning( f"""Folder {TEMP_DIR_NAME} with temp mmap arrays is being deleted. All existing temp mmapp arrays will be unusable!""" ) @@ -115,9 +114,11 @@ def redefine_temp_location(path): # create new tempfile at desired location _TEMP_DIR = tempfile.TemporaryDirectory(prefix=os.path.join(path, "temp_mmap_")) TEMP_DIR_NAME = _TEMP_DIR.name + logging.warning( f"""New temp folder location. Temp mmap arrays are written to {TEMP_DIR_NAME}. Cleanup of this folder is OS dependant, and might need to be triggered manually!""" ) + return TEMP_DIR_NAME @@ -298,13 +299,15 @@ def clear() -> str: str The name of the new temporary folder. """ - global _TEMP_DIR - global TEMP_DIR_NAME + global _TEMP_DIR, TEMP_DIR_NAME + logging.warning( f"Folder {TEMP_DIR_NAME} with temp mmap arrays is being deleted. " "All existing temp mmapp arrays will be unusable!" ) + del _TEMP_DIR + _TEMP_DIR = tempfile.TemporaryDirectory(prefix="temp_mmap_") TEMP_DIR_NAME = _TEMP_DIR.name return TEMP_DIR_NAME From 275ee28753beb078e6a1044b51d933037f14067d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Thu, 23 May 2024 15:38:15 +0200 Subject: [PATCH 15/21] delete unnecessary del statement --- alphabase/io/tempmmap.py | 1 - 1 file changed, 1 deletion(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index 661a37f2..7be01fff 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -109,7 +109,6 @@ def redefine_temp_location(path): # cleaup old temporary directory shutil.rmtree(TEMP_DIR_NAME, ignore_errors=True) - del _TEMP_DIR # create new tempfile at desired location _TEMP_DIR = tempfile.TemporaryDirectory(prefix=os.path.join(path, "temp_mmap_")) From 60ea09bed070c9080b5be3f4200897f68fd3b19c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophia=20M=C3=A4dler?= <15019107+sophiamaedler@users.noreply.github.com> Date: Thu, 23 May 2024 15:40:00 +0200 Subject: [PATCH 16/21] ruff formatting --- alphabase/io/tempmmap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index 7be01fff..b7bd8af9 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -22,6 +22,7 @@ "and might need to be triggered manually!" ) + def _change_temp_dir_location(abs_path: str) -> str: """ Check if the directory to which the temp arrays should be written exists, if so defines this as the new temp dir location. If not raise a value error. @@ -47,6 +48,7 @@ def _change_temp_dir_location(abs_path: str) -> str: f"The directory {abs_path} in which the file should be created does not exist." ) + def _get_file_location(abs_file_path: str, overwrite=False) -> str: """ Check if the path specified for the new temporary file is valid. If not raise a value error. @@ -306,7 +308,7 @@ def clear() -> str: ) del _TEMP_DIR - + _TEMP_DIR = tempfile.TemporaryDirectory(prefix="temp_mmap_") TEMP_DIR_NAME = _TEMP_DIR.name return TEMP_DIR_NAME From 280f7bac888be608f33af4721aa49c0235e60147 Mon Sep 17 00:00:00 2001 From: mschwoerer <82171591+mschwoer@users.noreply.github.com> Date: Fri, 31 May 2024 12:11:32 +0200 Subject: [PATCH 17/21] fix formatting --- alphabase/io/tempmmap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alphabase/io/tempmmap.py b/alphabase/io/tempmmap.py index fc2c9a79..054ded1a 100644 --- a/alphabase/io/tempmmap.py +++ b/alphabase/io/tempmmap.py @@ -151,7 +151,7 @@ def array(shape: tuple, dtype: np.dtype, tmp_dir_abs_path: str = None) -> np.nda A writable temporary mmapped array. """ global TEMP_DIR_NAME - + _log_cleanup_info_once() # redefine the temporary directory if a new location is given otherwise read from global variable @@ -209,7 +209,7 @@ def create_empty_mmap( global TEMP_DIR_NAME _log_cleanup_info_once() - + # redefine the temporary directory if a new location is given otherwise read from global variable # this allows you to ensure that the correct temp directory location is used when working with multiple threads if tmp_dir_abs_path is not None: From aab05a045c48606a09763a552651e9df59b5e8a7 Mon Sep 17 00:00:00 2001 From: GeorgWa Date: Wed, 12 Jun 2024 18:27:14 -0700 Subject: [PATCH 18/21] fix #181 --- alphabase/protein/fasta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alphabase/protein/fasta.py b/alphabase/protein/fasta.py index a7e7f9df..81f3145d 100644 --- a/alphabase/protein/fasta.py +++ b/alphabase/protein/fasta.py @@ -46,7 +46,7 @@ def read_fasta_file(fasta_filename: str = ""): protein information, {protein_id:str, full_name:str, gene_name:str, description:str, sequence:str} """ - with open(fasta_filename, "rt") as handle: + with open(fasta_filename, "rt", encoding="utf-8", errors="ignore") as handle: iterator = SeqIO.parse(handle, "fasta") while iterator: try: From 6998c66aa6fa81e626cb9902d795fc58b9e800bf Mon Sep 17 00:00:00 2001 From: GeorgWa Date: Thu, 13 Jun 2024 12:12:21 -0700 Subject: [PATCH 19/21] raise on error --- alphabase/protein/fasta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alphabase/protein/fasta.py b/alphabase/protein/fasta.py index 81f3145d..06b9e4cc 100644 --- a/alphabase/protein/fasta.py +++ b/alphabase/protein/fasta.py @@ -46,7 +46,7 @@ def read_fasta_file(fasta_filename: str = ""): protein information, {protein_id:str, full_name:str, gene_name:str, description:str, sequence:str} """ - with open(fasta_filename, "rt", encoding="utf-8", errors="ignore") as handle: + with open(fasta_filename, "rt", encoding="utf-8") as handle: iterator = SeqIO.parse(handle, "fasta") while iterator: try: From b7dd06795ad8bbb7df7d2b6d9a664ff1059718fb Mon Sep 17 00:00:00 2001 From: jalew188 Date: Fri, 14 Jun 2024 05:29:54 +0200 Subject: [PATCH 20/21] =?UTF-8?q?Bump=20version:=201.2.4=20=E2=86=92=201.2?= =?UTF-8?q?.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 3 +-- .pre-commit-config.yaml | 2 ++ alphabase/__init__.py | 2 +- docs/conf.py | 2 +- release/one_click_linux_gui/control | 2 +- release/one_click_linux_gui/create_installer_linux.sh | 2 +- release/one_click_macos_gui/Info.plist | 4 ++-- release/one_click_macos_gui/create_installer_macos.sh | 4 ++-- release/one_click_macos_gui/distribution.xml | 2 +- release/one_click_windows_gui/alphabase_innoinstaller.iss | 2 +- release/one_click_windows_gui/create_installer_windows.sh | 2 +- 11 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0db58b8f..e4500ed2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.4 +current_version = 1.2.5 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? @@ -28,6 +28,5 @@ serialize = [bumpversion:file:./release/one_click_windows_gui/create_installer_windows.sh] [bumpversion:file:./release/one_click_windows_gui/alphabase_innoinstaller.iss] - search = {current_version} replace = {new_version} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e466212..f136e64f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,3 +12,5 @@ repos: hooks: - id: ruff-format - id: ruff + + exclude: .bumpversion.cfg diff --git a/alphabase/__init__.py b/alphabase/__init__.py index 1541389d..8a48bb30 100644 --- a/alphabase/__init__.py +++ b/alphabase/__init__.py @@ -2,7 +2,7 @@ __project__ = "alphabase" -__version__ = "1.2.4" +__version__ = "1.2.5" __license__ = "Apache" __description__ = "An infrastructure Python package of the AlphaX ecosystem" __author__ = "Mann Labs" diff --git a/docs/conf.py b/docs/conf.py index e900aa78..7138ffaa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,7 @@ copyright = "2022, Mann Labs, MPIB" author = "Mann Labs, MPIB" -release = "1.2.4" +release = "1.2.5" # -- General configuration --------------------------------------------------- diff --git a/release/one_click_linux_gui/control b/release/one_click_linux_gui/control index ee039cdc..e447a1c6 100644 --- a/release/one_click_linux_gui/control +++ b/release/one_click_linux_gui/control @@ -1,5 +1,5 @@ Package: AlphaBase -Version: 1.2.4 +Version: 1.2.5 Architecture: all Maintainer: Mann Labs Description: AlphaBase diff --git a/release/one_click_linux_gui/create_installer_linux.sh b/release/one_click_linux_gui/create_installer_linux.sh index f0418304..69a18e50 100644 --- a/release/one_click_linux_gui/create_installer_linux.sh +++ b/release/one_click_linux_gui/create_installer_linux.sh @@ -17,7 +17,7 @@ python setup.py sdist bdist_wheel # Setting up the local package cd release/one_click_linux_gui # Make sure you include the required extra packages and always use the stable or very-stable options! -pip install "../../dist/alphabase-1.2.4-py3-none-any.whl[stable]" +pip install "../../dist/alphabase-1.2.5-py3-none-any.whl[stable]" # Creating the stand-alone pyinstaller folder pip install pyinstaller diff --git a/release/one_click_macos_gui/Info.plist b/release/one_click_macos_gui/Info.plist index ab2af814..c76b5ef3 100644 --- a/release/one_click_macos_gui/Info.plist +++ b/release/one_click_macos_gui/Info.plist @@ -9,9 +9,9 @@ CFBundleIconFile alpha_logo.icns CFBundleIdentifier - alphabase.1.2.4 + alphabase.1.2.5 CFBundleShortVersionString - 1.2.4 + 1.2.5 CFBundleInfoDictionaryVersion 6.0 CFBundleName diff --git a/release/one_click_macos_gui/create_installer_macos.sh b/release/one_click_macos_gui/create_installer_macos.sh index 4165b9c2..0c3627fd 100644 --- a/release/one_click_macos_gui/create_installer_macos.sh +++ b/release/one_click_macos_gui/create_installer_macos.sh @@ -20,7 +20,7 @@ python setup.py sdist bdist_wheel # Setting up the local package cd release/one_click_macos_gui -pip install "../../dist/alphabase-1.2.4-py3-none-any.whl[stable]" +pip install "../../dist/alphabase-1.2.5-py3-none-any.whl[stable]" # Creating the stand-alone pyinstaller folder pip install pyinstaller @@ -40,5 +40,5 @@ cp ../../LICENSE.txt Resources/LICENSE.txt cp ../logos/alpha_logo.png Resources/alpha_logo.png chmod 777 scripts/* -pkgbuild --root dist/alphabase --identifier de.mpg.biochem.alphabase.app --version 1.2.4 --install-location /Applications/AlphaBase.app --scripts scripts AlphaBase.pkg +pkgbuild --root dist/alphabase --identifier de.mpg.biochem.alphabase.app --version 1.2.5 --install-location /Applications/AlphaBase.app --scripts scripts AlphaBase.pkg productbuild --distribution distribution.xml --resources Resources --package-path AlphaBase.pkg dist/alphabase_gui_installer_macos.pkg diff --git a/release/one_click_macos_gui/distribution.xml b/release/one_click_macos_gui/distribution.xml index a129f66d..abf03388 100644 --- a/release/one_click_macos_gui/distribution.xml +++ b/release/one_click_macos_gui/distribution.xml @@ -1,6 +1,6 @@ - AlphaBase 1.2.4 + AlphaBase 1.2.5 diff --git a/release/one_click_windows_gui/alphabase_innoinstaller.iss b/release/one_click_windows_gui/alphabase_innoinstaller.iss index 542da524..12bed322 100644 --- a/release/one_click_windows_gui/alphabase_innoinstaller.iss +++ b/release/one_click_windows_gui/alphabase_innoinstaller.iss @@ -2,7 +2,7 @@ ; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES! #define MyAppName "AlphaBase" -#define MyAppVersion "1.2.4" +#define MyAppVersion "1.2.5" #define MyAppPublisher "Max Planck Institute of Biochemistry and the University of Copenhagen, Mann Labs" #define MyAppURL "https://github.com/MannLabs/alphabase" #define MyAppExeName "alphabase_gui.exe" diff --git a/release/one_click_windows_gui/create_installer_windows.sh b/release/one_click_windows_gui/create_installer_windows.sh index 4734bd75..81229509 100644 --- a/release/one_click_windows_gui/create_installer_windows.sh +++ b/release/one_click_windows_gui/create_installer_windows.sh @@ -17,7 +17,7 @@ python setup.py sdist bdist_wheel # Setting up the local package cd release/one_click_windows_gui # Make sure you include the required extra packages and always use the stable or very-stable options! -pip install "../../dist/alphabase-1.2.4-py3-none-any.whl[stable]" +pip install "../../dist/alphabase-1.2.5-py3-none-any.whl[stable]" # Creating the stand-alone pyinstaller folder pip install pyinstaller From 0272da94e20227afe997928eff8bf0c9200a1696 Mon Sep 17 00:00:00 2001 From: jalew188 Date: Fri, 14 Jun 2024 05:31:00 +0200 Subject: [PATCH 21/21] =?UTF-8?q?Bump=20version:=201.2.4=20=E2=86=92=201.2?= =?UTF-8?q?.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f136e64f..86124e24 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,4 +13,4 @@ repos: - id: ruff-format - id: ruff - exclude: .bumpversion.cfg +exclude: .bumpversion.cfg