diff --git a/changelog.md b/changelog.md index d499de5fc..6fc380eb7 100644 --- a/changelog.md +++ b/changelog.md @@ -15,6 +15,8 @@ * Added parameter `titles` to customize title of each subplot in [`Kymo.plot_with_channels()`](https://lumicks-pylake.readthedocs.io/en/latest/_api/lumicks.pylake.kymo.Kymo.html#lumicks.pylake.kymo.Kymo.plot_with_channels). * Added [`KymoTrack.sample_from_channel()`](https://lumicks-pylake.readthedocs.io/en/latest/_api/lumicks.pylake.kymotracker.kymotrack.KymoTrack.html#lumicks.pylake.kymotracker.kymotrack.KymoTrack.sample_from_channel) to downsample channel data to the time points of a kymotrack. * Added support for file names with spaces in [`lk.download_from_doi()`](https://lumicks-pylake.readthedocs.io/en/latest/_api/lumicks.pylake.download_from_doi.html#lumicks.pylake.download_from_doi). +* Added function to import a [`KymoTrackGroup`](https://lumicks-pylake.readthedocs.io/en/latest/_api/lumicks.pylake.kymotracker.kymotrack.KymoTrackGroup.html) from a `CSV` file using [`load_tracks`](https://lumicks-pylake.readthedocs.io/en/latest/_api/lumicks.pylake.load_tracks.html). +* Added function to load tracks into the kymotracker widget using [`KymoWidgetGreedy.load_tracks()`](https://lumicks-pylake.readthedocs.io/en/latest/_api/lumicks.pylake.KymoWidgetGreedy.html#lumicks.pylake.KymoWidgetGreedy.load_tracks). #### Improvements diff --git a/docs/api.rst b/docs/api.rst index 37e31b845..278ed7fb8 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -111,6 +111,8 @@ Kymotracking filter_tracks refine_tracks_centroid refine_tracks_gaussian + load_tracks + Notebook widgets ---------------- diff --git a/docs/tutorial/kymotracking.rst b/docs/tutorial/kymotracking.rst index bbe057624..99acf6d96 100644 --- a/docs/tutorial/kymotracking.rst +++ b/docs/tutorial/kymotracking.rst @@ -471,8 +471,8 @@ This snippet also demonstrates how we can pass keyword arguments (forwarded to : `) to format the histogram. -Exporting kymograph tracks --------------------------- +Importing and exporting kymograph tracks +---------------------------------------- We can export the coordinates of the tracks to a `csv` file using the :meth:`~lumicks.pylake.kymotracker.kymotrack.KymoTrackGroup.save` method with the desired file name:: @@ -484,6 +484,15 @@ by passing a width in pixels to sum counts over:: tracks.save("tracks_signal.csv", sampling_width=3, correct_origin=True) +We can re-import these tracks into Pylake at a later time using the :func:`~lumicks.pylake.load_tracks()` function:: + + tracks = lk.load_tracks("tracks.csv", kymo40, "green") + +.. note:: + + The `kymo` and `channel` arguments provided must be the same as the ones used to generate the tracks. + This includes any processing (e.g. cropping, downsampling, flipping) that was performed on the kymograph. + How the algorithms work ----------------------- diff --git a/lumicks/pylake/__init__.py b/lumicks/pylake/__init__.py index 1c936d22f..c4ee84517 100644 --- a/lumicks/pylake/__init__.py +++ b/lumicks/pylake/__init__.py @@ -20,6 +20,7 @@ from .image_stack import ImageStack, CorrelatedStack from .file_download import * from .fitting.models import * +from .kymotracker.kymotrack import load_tracks from .fitting.parameter_trace import parameter_trace from .kymotracker.kymotracker import * from .piezo_tracking.baseline import * diff --git a/lumicks/pylake/file.py b/lumicks/pylake/file.py index 94d66a08d..bb40417bb 100644 --- a/lumicks/pylake/file.py +++ b/lumicks/pylake/file.py @@ -41,8 +41,17 @@ class File(Group, Force, DownsampledFD, BaselineCorrectedForce, PhotonCounts, Ph file.force1x.plot() file.kymos["name"].plot() - # Open with custom detector mapping + # Open file mapping the red photon channel to red and the blue photon channel to green and + # blue (making them appear cyan). + file = pylake.File("example.h5", rgb_to_detectors={"Red": "Red", "Green": "Blue", "Blue": "Blue"}) + + # For some systems, the detectors might be named differently (in the case of custom + # detectors). file = pylake.File("example.h5", rgb_to_detectors={"Red": "Detector 1", "Green": "Detector 2", "Blue": "Detector 3"}) + + # To see which channels are available, inspect which names are available in the group + # `Photon count` or `Photon time tags`. + print(file["Photon count"]) """ SUPPORTED_FILE_FORMAT_VERSIONS = [1, 2] diff --git a/lumicks/pylake/kymotracker/kymotrack.py b/lumicks/pylake/kymotracker/kymotrack.py index b7b63d125..219bfb107 100644 --- a/lumicks/pylake/kymotracker/kymotrack.py +++ b/lumicks/pylake/kymotracker/kymotrack.py @@ -33,6 +33,10 @@ def _read_txt(file, delimiter): except TypeError: # Direct StringIO header_lines = [file.readline(), file.readline()] + except UnicodeDecodeError as e: + raise ValueError( + f"Invalid file format. Expected comma separated text file. Full error message: {e}." + ) from None # from v0.13.0, exported CSV files have an additional header line # with the pylake version and CSV version (starting at 2) @@ -52,8 +56,8 @@ def _read_txt(file, delimiter): data = {} try: raw_data = np.loadtxt(file, delimiter=delimiter, unpack=True) - except ValueError: - raise IOError("Invalid file format!") + except ValueError as e: + raise ValueError(f"Invalid file format: {str(e)}") from None header = header_lines[0].rstrip().split(delimiter) track_idx = raw_data[0, :] @@ -167,50 +171,66 @@ def store_column(column_title, format_string, new_data): def _check_summing_mismatch(track, sampling_width): """Checks calling sample_from_image on a loaded track reproduces the same result as is in the file.""" - wrong_kymo_warning = ( - "Photon counts do not match the photon counts found in the file. It is " - "possible that the loaded kymo or channel doesn't match the one used to " - "create this file." - ) try: - if not np.allclose( + if np.allclose( track.sample_from_image((sampling_width - 1) // 2, correct_origin=True), track.photon_counts, ): - if np.allclose( - track.sample_from_image((sampling_width - 1) // 2, correct_origin=False), - track.photon_counts, - ): - return RuntimeWarning( - "Photon counts do not match the photon counts found in the file. Prior to " - "Pylake v1.1.0, the method `sample_from_image` had a bug that assumed the " - "origin of a pixel to be at the edge rather than the center of the pixel. " - "Consequently, the sampled window could be off by one pixel. This file was " - "likely created using the incorrect origin. " - "Note that Pylake loaded the counts found in the file as is, so if the " - "used summing window was very small, there may be a bias in the counts." - "To recreate these counts without bias invoke:" - f"`track.sample_from_image({(sampling_width - 1) // 2}, correct_origin=True)`" - ) - else: - return RuntimeWarning(wrong_kymo_warning) + return # We're good + + if np.allclose( + track.sample_from_image((sampling_width - 1) // 2, correct_origin=False), + track.photon_counts, + ): + return RuntimeWarning( + "Photon counts do not match the photon counts found in the file. Prior to " + "Pylake v1.1.0, the method `sample_from_image` had a bug that assumed the " + "origin of a pixel to be at the edge rather than the center of the pixel. " + "Consequently, the sampled window could be off by one pixel. This file was " + "likely created using the incorrect origin. " + "Note that Pylake loaded the counts found in the file as is, so if the " + "used summing window was very small, there may be a bias in the counts." + "To recreate these counts without bias invoke:" + f"`track.sample_from_image({(sampling_width - 1) // 2}, correct_origin=True)`" + ) + else: + return RuntimeWarning( + "Photon counts do not match the photon counts found in the file. It is " + "possible that the loaded kymo or channel doesn't match the one used to " + "create this file. Note that if you processed (e.g. sliced, cropped, flipped or " + "downsampled) the kymograph prior to tracking, you will have to make sure that you " + "crop the kymograph you supply to this function in the same way." + ) except IndexError: - return RuntimeWarning(wrong_kymo_warning) + raise ValueError( + "The supplied kymograph is of a different duration or size than the one used to " + "compute these tracks. The kymograph does not match the tracks found in the file. Note " + "that if you processed (e.g. sliced, cropped or downsampled) the kymograph prior to " + "tracking, you will have to make sure that you crop the kymograph you supply to this " + "function in the same way." + ) -def import_kymotrackgroup_from_csv(filename, kymo, channel, delimiter=";"): - """Import a KymoTrackGroup from a csv file. +def load_tracks(filename, kymo, channel, delimiter=";"): + """Loads a :class:`~lumicks.pylake.kymotracker.kymotrack.KymoTrackGroup` from a csv file. The file format contains a series of columns as follows: track index, time (pixels), coordinate (pixels), time (optional), coordinate (optional), sampled_counts (optional), minimum length + .. note:: + + If you processed (e.g. cropping, flipping, downsampling) the kymograph prior to tracking, + you will have to make sure that you crop the kymograph you supply to this function in the + same way. + Parameters ---------- filename : str | os.PathLike filename to import from. kymo : Kymo - kymograph instance that the CSV data was tracked from. + Kymograph that the CSV data was tracked from. This kymograph has to be the one used to + create the tracks (this includes any processing). See the note above for more information. channel : str color channel that was used for tracking. delimiter : str @@ -223,8 +243,27 @@ def import_kymotrackgroup_from_csv(filename, kymo, channel, delimiter=";"): Raises ------ - IOError + ValueError If the file format is not as expected. + + Examples + -------- + :: + + import lumicks.pylake as lk + + file = lk.File("test_data/kymo.h5") + kymo = file.kymos["16"] # Extract the kymo named 16 + + kymo_cropped = kymo.crop_by_distance(10, 25) + tracks = lk.track_greedy(kymo_cropped, channel="red", pixel_threshold=5) + tracks.save("tracks.csv") + + loaded_tracks = lk.load_tracks("tracks.csv", kymo_cropped, "red") + + # Plot the red channel of the kymograph and tracks together + kymo_cropped.plot("red", adjustment=lk.ColorAdjustment(5, 95, "percentile")) + loaded_tracks.plot() """ # TODO: File format validation could use some improvement @@ -232,7 +271,9 @@ def import_kymotrackgroup_from_csv(filename, kymo, channel, delimiter=";"): mandatory_fields = ["time (pixels)", "coordinate (pixels)"] if not all(field_name in data for field_name in mandatory_fields): - raise IOError("Invalid file format!") + raise ValueError( + f"Invalid file format. Missing field(s): {', '.join(set(data.keys()) - set(mandatory_fields))}" + ) # We get a list of time coordinates per track for track_time in data["time (pixels)"]: @@ -292,7 +333,7 @@ def create_track(time, coord, min_length=None, counts=None): ) ) - if sampling_width is not None: + if sampling_width is not None and not resampling_mismatch: resampling_mismatch = _check_summing_mismatch(tracks[-1], sampling_width) if resampling_mismatch: @@ -1077,7 +1118,7 @@ class KymoTrackGroup: -------- :: - from lumicks import pylake + import lumicks.pylake as lk tracks = lk.track_greedy(kymo, channel="red", pixel_threshold=5) @@ -1498,6 +1539,9 @@ def plot_fit(self, frame_idx, *, fit_kwargs=None, data_kwargs=None, show_track_i import lumicks.pylake as lk + file = lk.File("test_data/kymo.h5") + kymo = file.kymos["16"] # Extract the kymo named 16 + tracks = lk.track_greedy(kymo, channel="red", pixel_threshold=5) refined = lk.refine_tracks_gaussian(tracks, window=10, refine_missing_frames=True, overlap_strategy="multiple") diff --git a/lumicks/pylake/kymotracker/tests/test_io.py b/lumicks/pylake/kymotracker/tests/test_io.py index f160eeda7..354b6a410 100644 --- a/lumicks/pylake/kymotracker/tests/test_io.py +++ b/lumicks/pylake/kymotracker/tests/test_io.py @@ -8,12 +8,7 @@ import pytest from lumicks.pylake.kymo import _kymo_from_array -from lumicks.pylake.kymotracker.kymotrack import ( - KymoTrack, - KymoTrackGroup, - _read_txt, - import_kymotrackgroup_from_csv, -) +from lumicks.pylake.kymotracker.kymotrack import KymoTrack, KymoTrackGroup, _read_txt, load_tracks from lumicks.pylake.tests.data.mock_confocal import generate_kymo @@ -80,7 +75,7 @@ def test_kymotrackgroup_io(tmpdir_factory, dt, dx, delimiter, sampling_width, sa # Test round trip through the API testfile = f"{tmpdir_factory.mktemp('pylake')}/test.csv" tracks.save(testfile, delimiter, sampling_width, correct_origin=True) - imported_tracks = import_kymotrackgroup_from_csv(testfile, kymo, "red", delimiter=delimiter) + imported_tracks = load_tracks(testfile, kymo, "red", delimiter=delimiter) data, pylake_version, csv_version = _read_txt(testfile, delimiter) compare_kymotrack_group(tracks, imported_tracks) @@ -146,9 +141,7 @@ def get_args(func): string_representation = s.getvalue() with io.StringIO(string_representation) as s: - read_tracks = import_kymotrackgroup_from_csv( - s, kymo_integration_test_data, "red", delimiter=delimiter - ) + read_tracks = load_tracks(s, kymo_integration_test_data, "red", delimiter=delimiter) compare_kymotrack_group(kymo_integration_tracks, read_tracks) @@ -166,31 +159,22 @@ def test_photon_count_validation(kymo_integration_test_data, kymo_integration_tr RuntimeWarning, match="origin of a pixel to be at the edge rather than the center of the pixel", ): - _ = import_kymotrackgroup_from_csv( - io.StringIO(biased_tracks), kymo_integration_test_data, "red" - ) + _ = load_tracks(io.StringIO(biased_tracks), kymo_integration_test_data, "red") # We can also fail by having the wrong kymo with pytest.warns( RuntimeWarning, match="loaded kymo or channel doesn't match the one used to create this file", ): - _ = import_kymotrackgroup_from_csv( - io.StringIO(good_tracks), kymo_integration_test_data, "green" - ) + _ = load_tracks(io.StringIO(good_tracks), kymo_integration_test_data, "green") # Or by having the wrong one where it actually completely fails to sample. This tests whether - # the exception inside import_kymotrackgroup_from_csv is correctly caught and handled - with pytest.warns( - RuntimeWarning, - match="loaded kymo or channel doesn't match the one used to create this file", - ): - _ = import_kymotrackgroup_from_csv( - io.StringIO(good_tracks), kymo_integration_test_data[:"1s"], "red" - ) + # the exception inside load_tracks is correctly caught and handled + with pytest.raises(ValueError, match="kymograph is of a different duration or size"): + _ = load_tracks(io.StringIO(good_tracks), kymo_integration_test_data[:"1s"], "red") # Control for good tracks - import_kymotrackgroup_from_csv(io.StringIO(good_tracks), kymo_integration_test_data, "red") + load_tracks(io.StringIO(good_tracks), kymo_integration_test_data, "red") @pytest.mark.parametrize( @@ -222,7 +206,7 @@ def test_csv_version(version, read_with_version, recwarn): ) testfile = Path(__file__).parent / f"./data/tracks_v{version}.csv" - imported_tracks = import_kymotrackgroup_from_csv(testfile, kymo, "red", delimiter=";") + imported_tracks = load_tracks(testfile, kymo, "red", delimiter=";") match version: case 3: @@ -244,11 +228,20 @@ def test_csv_version(version, read_with_version, recwarn): np.testing.assert_allclose(track.coordinate_idx, data["coordinate (pixels)"][j]) -@pytest.mark.parametrize("filename", ["csv_bad_format.csv", "csv_unparseable.csv"]) -def test_bad_csv(filename, blank_kymo): - with pytest.raises(IOError, match="Invalid file format!"): +@pytest.mark.parametrize( + "filename, error", + [ + ("csv_bad_format.csv", "Invalid file format. Missing field(s): # track index"), + ( + "csv_unparseable.csv", + "Invalid file format: could not convert string '' to float64 at row 0, column 3.", + ), + ], +) +def test_bad_csv(filename, error, blank_kymo): + with pytest.raises(ValueError, match=re.escape(error)): file = Path(__file__).parent / "data" / filename - import_kymotrackgroup_from_csv(file, blank_kymo, "red", delimiter=";") + load_tracks(file, blank_kymo, "red", delimiter=";") def test_min_obs_csv_regression(tmpdir_factory, blank_kymo): @@ -258,7 +251,7 @@ def test_min_obs_csv_regression(tmpdir_factory, blank_kymo): RuntimeWarning, match="loaded kymo or channel doesn't match the one used to create this file", ): - imported_tracks = import_kymotrackgroup_from_csv(testfile, blank_kymo, "red", delimiter=";") + imported_tracks = load_tracks(testfile, blank_kymo, "red", delimiter=";") out_file = f"{tmpdir_factory.mktemp('pylake')}/no_min_lengths.csv" @@ -268,4 +261,4 @@ def test_min_obs_csv_regression(tmpdir_factory, blank_kymo): out_file2 = f"{tmpdir_factory.mktemp('pylake')}/no_min_lengths2.csv" with pytest.warns(RuntimeWarning, match=err_msg): - import_kymotrackgroup_from_csv(out_file, blank_kymo, "red", delimiter=";").save(out_file2) + load_tracks(out_file, blank_kymo, "red", delimiter=";").save(out_file2) diff --git a/lumicks/pylake/nb_widgets/kymotracker_widgets.py b/lumicks/pylake/nb_widgets/kymotracker_widgets.py index ee40ee018..d307fed6b 100644 --- a/lumicks/pylake/nb_widgets/kymotracker_widgets.py +++ b/lumicks/pylake/nb_widgets/kymotracker_widgets.py @@ -7,7 +7,7 @@ import numpy as np from lumicks.pylake import filter_tracks, refine_tracks_centroid -from lumicks.pylake.kymotracker.kymotrack import KymoTrackGroup, import_kymotrackgroup_from_csv +from lumicks.pylake.kymotracker.kymotrack import KymoTrackGroup, load_tracks from lumicks.pylake.kymotracker.kymotracker import track_greedy, _to_half_kernel_size from lumicks.pylake.nb_widgets.detail.mouse import MouseDragCallback from lumicks.pylake.nb_widgets.detail.undostack import UndoStack @@ -361,14 +361,31 @@ def save_tracks(self, filename, delimiter=";", sampling_width=None): def _load_from_ui(self): try: - self.tracks = import_kymotrackgroup_from_csv( - self._output_filename, self._kymo, self._channel - ) + self.tracks = load_tracks(self._output_filename, self._kymo, self._channel) self._update_tracks() self._set_label("status", f"Loaded {self._output_filename}") - except (RuntimeError, IOError) as exception: + except (RuntimeError, ValueError, IOError) as exception: self._set_label("status", str(exception)) + def load_tracks(self, filename, delimiter=";"): + """Loads tracks from a csv file into the widget + + .. note:: + + If you processed the kymograph used to create these tracks, you have to ensure that + the same processing is applied to the kymograph you load into the widget. See + :func:`~lumicks.pylake.load_tracks` for more information. + + Parameters + ---------- + filename : str | os.PathLike + filename to import from. + delimiter : str + The string used to separate columns. Default is ';'. + """ + self.tracks = load_tracks(filename, self._kymo, self._channel, delimiter) + self._update_tracks() + def _add_slider(self, name, parameter): import ipywidgets diff --git a/lumicks/pylake/nb_widgets/tests/test_kymotracker_widget.py b/lumicks/pylake/nb_widgets/tests/test_kymotracker_widget.py index c222f2acb..0025c1492 100644 --- a/lumicks/pylake/nb_widgets/tests/test_kymotracker_widget.py +++ b/lumicks/pylake/nb_widgets/tests/test_kymotracker_widget.py @@ -191,6 +191,41 @@ def test_refine_from_widget(kymograph, region_select): assert len(kymo_widget.tracks) == 1 +def test_load_from_command(kymograph, region_select, tmpdir_factory): + kymo_widget = KymoWidgetGreedy( + kymograph, "red", track_width=3, axis_aspect_ratio=1, use_widgets=False, correct_origin=True + ) + kymo_widget._algorithm_parameters["pixel_threshold"].value = 2 + in_um, in_s = calibrate_to_kymo(kymograph) + kymo_widget._track_kymo(*region_select(in_s(5), in_um(12), in_s(20), in_um(13))) + testfile = f"{tmpdir_factory.mktemp('pylake')}/kymo.csv" + kymo_widget.save_tracks(testfile, sampling_width=2) + + kymo_widget = KymoWidgetGreedy(kymograph, "red", axis_aspect_ratio=1, use_widgets=False) + assert len(kymo_widget.tracks) == 0 + kymo_widget.load_tracks(testfile) + assert len(kymo_widget.tracks) == 1 + + # Definitely a different(ly cropped) kymograph + with pytest.raises(ValueError, match="kymograph is of a different duration or size"): + kymo_widget = KymoWidgetGreedy( + kymograph[:"1.2s"], "red", axis_aspect_ratio=1, use_widgets=False + ) + kymo_widget._algorithm_parameters["pixel_threshold"].value = 2 + _ = kymo_widget.load_tracks(testfile) + + # Might be a different kymograph, but can't be sure unfortunately + with pytest.warns( + RuntimeWarning, + match=r"Photon counts do not match the photon counts found in the file", + ): + kymo_widget = KymoWidgetGreedy( + kymograph.crop_by_distance(5, 1000), "red", axis_aspect_ratio=1, use_widgets=False + ) + kymo_widget.load_tracks(testfile) + assert len(kymo_widget.tracks) == 1 + + def test_stitch(kymograph, mockevent): kymo_widget = KymoWidgetGreedy(kymograph, "red", axis_aspect_ratio=1, use_widgets=False) @@ -393,9 +428,9 @@ def test_keyword_args(kymograph): "gain,line_time,pixel_size,ref_values", ( # fmt:off - (1, 2.0, 5.0, {"pixel_threshold": (97, 1, 99, None), "track_width": (4 * 5, 3 * 5, 15 * 5, "μm"), "sigma": (2 * 5, 1 * 5, 5 * 5, "μm"), "velocity": (0, -5 * 5/2, 5 * 5/2, "μm/s")}), - (0, 2.0, 5.0, {"pixel_threshold": (1, 1, 2, None), "track_width": (4 * 5, 3 * 5, 15 * 5, "μm"), "sigma": (2 * 5, 1 * 5, 5 * 5, "μm"), "velocity": (0, -5 * 5/2, 5 * 5/2, "μm/s")}), - (1, 4.0, 4.0, {"pixel_threshold": (97, 1, 99, None), "track_width": (4 * 4, 3 * 4, 15 * 4, "μm"), "sigma": (2 * 4, 1 * 4, 5 * 4, "μm"), "velocity": (0, -5 * 4/4, 5 * 4/4, "μm/s")}), + (1, 2.0, 5.0, {"pixel_threshold": (97, 1, 99, None), "track_width": (4 * 5, 3 * 5, 15 * 5, "μm"), "sigma": (2 * 5, 1 * 5, 5 * 5, "μm"), "velocity": (0, -5 * 5 / 2, 5 * 5 / 2, "μm/s")}), + (0, 2.0, 5.0, {"pixel_threshold": (1, 1, 2, None), "track_width": (4 * 5, 3 * 5, 15 * 5, "μm"), "sigma": (2 * 5, 1 * 5, 5 * 5, "μm"), "velocity": (0, -5 * 5 / 2, 5 * 5 / 2, "μm/s")}), + (1, 4.0, 4.0, {"pixel_threshold": (97, 1, 99, None), "track_width": (4 * 4, 3 * 4, 15 * 4, "μm"), "sigma": (2 * 4, 1 * 4, 5 * 4, "μm"), "velocity": (0, -5 * 4 / 4, 5 * 4 / 4, "μm/s")}), # fmt:on ), )