diff --git a/CHANGELOG b/CHANGELOG index 6025175..cef4769 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +0.6.1 + - enh: automatically detect flickering and apply offset correction + - setup: bump dcnum from 0.24.0 to 0.25.0 (flickering detection) 0.6.0 - feat: implement segmenter validation step in CLI and GUI - feat: support PyTorch segmentation models diff --git a/chipstream/cli/cli_proc.py b/chipstream/cli/cli_proc.py index 3d337d2..889377a 100644 --- a/chipstream/cli/cli_proc.py +++ b/chipstream/cli/cli_proc.py @@ -51,6 +51,14 @@ def process_dataset( # background keyword arguments bg_kwargs = validate_background_kwargs(background_method, background_kwargs) + if (background_method == "sparsemed" + and "offset_correction" not in bg_kwargs): + # We are using the 'sparsemed' background algorithm, and the user + # did not specify whether she wants to perform flickering + # correction. Thus, we automatically check whether we need that. + with dcnum.read.HDF5Data(path_in) as hd: + bg_kwargs["offset_correction"] = \ + dcnum.read.detect_flickering(hd.image) bg_cls = cm.bg_methods[background_method] bg_id = bg_cls.get_ppid_from_ppkw(bg_kwargs) click.echo(f"Background ID:\t{bg_id}") diff --git a/chipstream/gui/main_window.py b/chipstream/gui/main_window.py index 0c0d18e..94783cc 100644 --- a/chipstream/gui/main_window.py +++ b/chipstream/gui/main_window.py @@ -190,8 +190,6 @@ def get_job_kwargs(self): bg_default = feat_background.BackgroundSparseMed bg_kwargs = inspect.getfullargspec( bg_default.check_user_kwargs).kwonlydefaults - bg_kwargs["offset_correction"] = \ - self.checkBox_bg_flickering.isChecked() # populate segmenter and its kwargs segmenter = self.comboBox_segmenter.currentData() diff --git a/chipstream/gui/main_window.ui b/chipstream/gui/main_window.ui index 74df210..f1aaf61 100644 --- a/chipstream/gui/main_window.ui +++ b/chipstream/gui/main_window.ui @@ -149,28 +149,6 @@ - - - - Background Image - - - - - - Use this option if the input dataset exhibits global temporal brightness variations of a few grayscal values - - - flickering correction - - - true - - - - - - diff --git a/chipstream/gui/manager.py b/chipstream/gui/manager.py index 835556e..ac887cf 100644 --- a/chipstream/gui/manager.py +++ b/chipstream/gui/manager.py @@ -1,3 +1,4 @@ +import copy import pathlib import threading import traceback @@ -218,9 +219,17 @@ def run(self): self.callback_when_done() def run_job(self, path_in, path_out): + job_kwargs = copy.deepcopy(self.job_kwargs) + # We are using the 'sparsemed' background algorithm by default, + # and we would like to perform flickering correction if necessary. + with dcnum.read.HDF5Data(path_in) as hd: + job_kwargs.setdefault( + "background_kwargs", {})["offset_correction"] = \ + dcnum.read.detect_flickering(hd.image) + job = dclogic.DCNumPipelineJob(path_in=path_in, path_out=path_out, - **self.job_kwargs) + **job_kwargs) self.jobs.append(job) # Make sure the job will run (This must be done after adding it # to the jobs list and before adding it to the runners list) diff --git a/pyproject.toml b/pyproject.toml index a8adc8e..1f9528f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ authors = [ maintainers = [ {name = "Paul Müller", email="dev@craban.de"}, ] -description = "GUI for DC data postprocessing" +description = "GUI and CLI for DC data postprocessing" readme = "README.rst" requires-python = ">=3.10, <4" keywords = ["RT-DC", "deformability", "cytometry"] @@ -27,7 +27,7 @@ classifiers = [ ] license = {text = "GPL version 3.0 or later"} dependencies = [ - "dcnum>=0.24.0", + "dcnum>=0.25.0", "h5py>=3.0.0, <4", "numpy>=1.21, <2", # CVE-2021-33430 ] diff --git a/tests/test_cli.py b/tests/test_cli.py index 0fe121b..b79e8ce 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -45,6 +45,53 @@ def test_cli_basins(cli_runner, drain): assert feat in h5["events"] +@pytest.mark.parametrize("add_flickering", [True, False]) +def test_cli_flickering_correction(cli_runner, add_flickering): + path_temp = retrieve_data( + "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip") + path = path_temp.with_name("input_path.rtdc") + + # create a test file for more than 100 events + with dcnum.read.concatenated_hdf5_data( + paths=3*[path_temp], + path_out=path, + compute_frame=True): + pass + + if add_flickering: + # provoke offset correction in CLI + with h5py.File(path, "a") as h5: + size = len(h5["events/image"]) + images_orig = h5["events/image"] + del h5["events/image"] + offset = np.zeros((size, 1, 1), dtype=np.uint8) + # add flickering every five frames + offset[::5] += 5 + h5["events/image"] = np.array( + images_orig + offset, + dtype=np.uint8 + ) + + path_out = path.with_name("flickering_test.rtdc") + args = [str(path), + str(path_out), + "-s", "thresh", + ] + + result = cli_runner.invoke(cli_main.chipstream_cli, args) + assert result.exit_code == 0 + + with h5py.File(path_out) as h5: + if add_flickering: + assert h5.attrs["pipeline:dcnum background"] \ + == "sparsemed:k=200^s=1^t=0^f=0.8^o=1" + assert "bg_off" in h5["events"] + else: + assert h5.attrs["pipeline:dcnum background"] \ + == "sparsemed:k=200^s=1^t=0^f=0.8^o=0" + assert "bg_off" not in h5["events"] + + @pytest.mark.parametrize("limit_events,dcnum_mapping,dcnum_yield,f0", [ # this is the default ["0", "0", 36, 1], @@ -77,6 +124,7 @@ def test_cli_limit_events(cli_runner, limit_events, dcnum_yield, result = cli_runner.invoke(cli_main.chipstream_cli, [str(path), str(path_out), + "-kb", "offset_correction=true", "-s", "thresh", "--limit-events", limit_events, "--drain-basins", @@ -85,6 +133,8 @@ def test_cli_limit_events(cli_runner, limit_events, dcnum_yield, with h5py.File(path_out) as h5: assert h5["events/frame"][0] == f0 + assert h5.attrs["pipeline:dcnum background"] == \ + "sparsemed:k=200^s=1^t=0^f=0.8^o=1" assert h5.attrs["pipeline:dcnum yield"] == dcnum_yield assert h5.attrs["pipeline:dcnum mapping"] == dcnum_mapping assert h5.attrs["experiment:event count"] == dcnum_yield diff --git a/tests/test_gui.py b/tests/test_gui.py index 9986e0b..faf1c40 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -1,5 +1,6 @@ import time +import dcnum.read import h5py import numpy as np import pytest @@ -62,13 +63,35 @@ def test_gui_basins(mw, use_basins): assert feat in h5["events"] -@pytest.mark.parametrize("correct_offset", [True, False]) -def test_gui_correct_offset(mw, correct_offset): - path = retrieve_data( +@pytest.mark.parametrize("add_flickering", [True, False]) +def test_gui_correct_offset(mw, add_flickering): + """Offset correction is done automatically""" + path_temp = retrieve_data( "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip") + path = path_temp.with_name("input_path.rtdc") + + # create a test file for more than 100 events + with dcnum.read.concatenated_hdf5_data( + paths=3*[path_temp], + path_out=path, + compute_frame=True): + pass + + if add_flickering: + # provoke offset correction in CLI + with h5py.File(path, "a") as h5: + size = len(h5["events/image"]) + images_orig = h5["events/image"] + del h5["events/image"] + offset = np.zeros((size, 1, 1), dtype=np.uint8) + # add flickering every five frames + offset[::5] += 5 + h5["events/image"] = np.array( + images_orig + offset, + dtype=np.uint8 + ) mw.append_paths([path]) mw.checkBox_pixel_size.setChecked(True) - mw.checkBox_bg_flickering.setChecked(correct_offset) mw.doubleSpinBox_pixel_size.setValue(0.666) mw.on_run() while mw.job_manager.is_busy(): @@ -80,7 +103,14 @@ def test_gui_correct_offset(mw, correct_offset): assert np.allclose(h5.attrs["imaging:pixel size"], 0.666, atol=0, rtol=1e-5) - assert ("bg_off" in h5["events"]) == correct_offset + if add_flickering: + assert h5.attrs["pipeline:dcnum background"] \ + == "sparsemed:k=200^s=1^t=0^f=0.8^o=1" + assert "bg_off" in h5["events"] + else: + assert h5.attrs["pipeline:dcnum background"] \ + == "sparsemed:k=200^s=1^t=0^f=0.8^o=0" + assert "bg_off" not in h5["events"] def test_gui_segm_torch_model(mw, qtbot, monkeypatch): diff --git a/tests/test_gui_manager.py b/tests/test_gui_manager.py index f1c3b9d..f930053 100644 --- a/tests/test_gui_manager.py +++ b/tests/test_gui_manager.py @@ -79,15 +79,15 @@ def test_manager_run_defaults(): # wait for the thread to join mg.join() - assert mg[0]["progress"] == 1 assert mg[0]["state"] == "done" + assert mg[0]["progress"] == 1 assert mg[0]["path"] == str(path) assert mg.current_index == 0 assert not mg.is_busy() # default pipeline may change in dcnum assert mg.get_runner(0).ppid == (f"{ppid.DCNUM_PPID_GENERATION}|" "hdf:p=0.2645^i=0|" - "sparsemed:k=200^s=1^t=0^f=0.8^o=1|" + "sparsemed:k=200^s=1^t=0^f=0.8^o=0|" "thresh:t=-6:cle=1^f=1^clo=2|" "legacy:b=1^h=1^v=1|" "norm:o=0^s=10")