enh: automatically detect flickering and apply offset correction

CHANGELOG

@@ -1,3 +1,6 @@
+0.6.1
+ - enh: automatically detect flickering and apply offset correction
+ - setup: bump dcnum from 0.24.0 to 0.25.0 (flickering detection)
 0.6.0
  - feat: implement segmenter validation step in CLI and GUI
  - feat: support PyTorch segmentation models

chipstream/cli/cli_proc.py

@@ -51,6 +51,14 @@ def process_dataset(
     # background keyword arguments
     bg_kwargs = validate_background_kwargs(background_method,
                                            background_kwargs)
+    if (background_method == "sparsemed"
+            and "offset_correction" not in bg_kwargs):
+        # We are using the 'sparsemed' background algorithm, and the user
+        # did not specify whether she wants to perform flickering
+        # correction. Thus, we automatically check whether we need that.
+        with dcnum.read.HDF5Data(path_in) as hd:
+            bg_kwargs["offset_correction"] = \
+                dcnum.read.detect_flickering(hd.image)
     bg_cls = cm.bg_methods[background_method]
     bg_id = bg_cls.get_ppid_from_ppkw(bg_kwargs)
     click.echo(f"Background ID:\t{bg_id}")

chipstream/gui/main_window.py

@@ -190,8 +190,6 @@ def get_job_kwargs(self):
         bg_default = feat_background.BackgroundSparseMed
         bg_kwargs = inspect.getfullargspec(
             bg_default.check_user_kwargs).kwonlydefaults
-        bg_kwargs["offset_correction"] = \
-            self.checkBox_bg_flickering.isChecked()
 
         # populate segmenter and its kwargs
         segmenter = self.comboBox_segmenter.currentData()

chipstream/gui/main_window.ui

@@ -149,28 +149,6 @@
          </property>
         </spacer>
        </item>
-       <item>
-        <widget class="QGroupBox" name="groupBox">
-         <property name="title">
-          <string>Background Image</string>
-         </property>
-         <layout class="QVBoxLayout" name="verticalLayout_2">
-          <item>
-           <widget class="QCheckBox" name="checkBox_bg_flickering">
-            <property name="toolTip">
-             <string>Use this option if the input dataset exhibits global temporal brightness variations of a few grayscal values</string>
-            </property>
-            <property name="text">
-             <string>flickering correction</string>
-            </property>
-            <property name="checked">
-             <bool>true</bool>
-            </property>
-           </widget>
-          </item>
-         </layout>
-        </widget>
-       </item>
        <item>
         <widget class="QGroupBox" name="groupBox_4">
          <property name="sizePolicy">

chipstream/gui/manager.py

@@ -1,3 +1,4 @@
+import copy
 import pathlib
 import threading
 import traceback
@@ -218,9 +219,17 @@ def run(self):
             self.callback_when_done()
 
     def run_job(self, path_in, path_out):
+        job_kwargs = copy.deepcopy(self.job_kwargs)
+        # We are using the 'sparsemed' background algorithm by default,
+        # and we would like to perform flickering correction if necessary.
+        with dcnum.read.HDF5Data(path_in) as hd:
+            job_kwargs.setdefault(
+                "background_kwargs", {})["offset_correction"] = \
+                    dcnum.read.detect_flickering(hd.image)
+
         job = dclogic.DCNumPipelineJob(path_in=path_in,
                                        path_out=path_out,
-                                       **self.job_kwargs)
+                                       **job_kwargs)
         self.jobs.append(job)
         # Make sure the job will run (This must be done after adding it
         # to the jobs list and before adding it to the runners list)

pyproject.toml

@@ -15,7 +15,7 @@ authors = [
 maintainers = [
     {name = "Paul Müller", email="[email protected]"},
 ]
-description = "GUI for DC data postprocessing"
+description = "GUI and CLI for DC data postprocessing"
 readme = "README.rst"
 requires-python = ">=3.10, <4"
 keywords = ["RT-DC", "deformability", "cytometry"]
@@ -27,7 +27,7 @@ classifiers = [
 ]
 license = {text = "GPL version 3.0 or later"}
 dependencies = [
-    "dcnum>=0.24.0",
+    "dcnum>=0.25.0",
     "h5py>=3.0.0, <4",
     "numpy>=1.21, <2",  # CVE-2021-33430
 ]

tests/test_cli.py

@@ -45,6 +45,53 @@ def test_cli_basins(cli_runner, drain):
             assert feat in h5["events"]
 
 
+@pytest.mark.parametrize("add_flickering", [True, False])
+def test_cli_flickering_correction(cli_runner, add_flickering):
+    path_temp = retrieve_data(
+        "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+    path = path_temp.with_name("input_path.rtdc")
+
+    # create a test file for more than 100 events
+    with dcnum.read.concatenated_hdf5_data(
+        paths=3*[path_temp],
+        path_out=path,
+            compute_frame=True):
+        pass
+
+    if add_flickering:
+        # provoke offset correction in CLI
+        with h5py.File(path, "a") as h5:
+            size = len(h5["events/image"])
+            images_orig = h5["events/image"]
+            del h5["events/image"]
+            offset = np.zeros((size, 1, 1), dtype=np.uint8)
+            # add flickering every five frames
+            offset[::5] += 5
+            h5["events/image"] = np.array(
+                images_orig + offset,
+                dtype=np.uint8
+            )
+
+    path_out = path.with_name("flickering_test.rtdc")
+    args = [str(path),
+            str(path_out),
+            "-s", "thresh",
+            ]
+
+    result = cli_runner.invoke(cli_main.chipstream_cli, args)
+    assert result.exit_code == 0
+
+    with h5py.File(path_out) as h5:
+        if add_flickering:
+            assert h5.attrs["pipeline:dcnum background"] \
+                   == "sparsemed:k=200^s=1^t=0^f=0.8^o=1"
+            assert "bg_off" in h5["events"]
+        else:
+            assert h5.attrs["pipeline:dcnum background"] \
+                   == "sparsemed:k=200^s=1^t=0^f=0.8^o=0"
+            assert "bg_off" not in h5["events"]
+
+
 @pytest.mark.parametrize("limit_events,dcnum_mapping,dcnum_yield,f0", [
     # this is the default
     ["0", "0", 36, 1],
@@ -77,6 +124,7 @@ def test_cli_limit_events(cli_runner, limit_events, dcnum_yield,
     result = cli_runner.invoke(cli_main.chipstream_cli,
                                [str(path),
                                 str(path_out),
+                                "-kb", "offset_correction=true",
                                 "-s", "thresh",
                                 "--limit-events", limit_events,
                                 "--drain-basins",
@@ -85,6 +133,8 @@ def test_cli_limit_events(cli_runner, limit_events, dcnum_yield,
 
     with h5py.File(path_out) as h5:
         assert h5["events/frame"][0] == f0
+        assert h5.attrs["pipeline:dcnum background"] == \
+               "sparsemed:k=200^s=1^t=0^f=0.8^o=1"
         assert h5.attrs["pipeline:dcnum yield"] == dcnum_yield
         assert h5.attrs["pipeline:dcnum mapping"] == dcnum_mapping
         assert h5.attrs["experiment:event count"] == dcnum_yield

tests/test_gui.py

@@ -1,5 +1,6 @@
 import time
 
+import dcnum.read
 import h5py
 import numpy as np
 import pytest
@@ -62,13 +63,35 @@ def test_gui_basins(mw, use_basins):
             assert feat in h5["events"]
 
 
-@pytest.mark.parametrize("correct_offset", [True, False])
-def test_gui_correct_offset(mw, correct_offset):
-    path = retrieve_data(
+@pytest.mark.parametrize("add_flickering", [True, False])
+def test_gui_correct_offset(mw, add_flickering):
+    """Offset correction is done automatically"""
+    path_temp = retrieve_data(
         "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+    path = path_temp.with_name("input_path.rtdc")
+
+    # create a test file for more than 100 events
+    with dcnum.read.concatenated_hdf5_data(
+        paths=3*[path_temp],
+        path_out=path,
+            compute_frame=True):
+        pass
+
+    if add_flickering:
+        # provoke offset correction in CLI
+        with h5py.File(path, "a") as h5:
+            size = len(h5["events/image"])
+            images_orig = h5["events/image"]
+            del h5["events/image"]
+            offset = np.zeros((size, 1, 1), dtype=np.uint8)
+            # add flickering every five frames
+            offset[::5] += 5
+            h5["events/image"] = np.array(
+                images_orig + offset,
+                dtype=np.uint8
+            )
     mw.append_paths([path])
     mw.checkBox_pixel_size.setChecked(True)
-    mw.checkBox_bg_flickering.setChecked(correct_offset)
     mw.doubleSpinBox_pixel_size.setValue(0.666)
     mw.on_run()
     while mw.job_manager.is_busy():
@@ -80,7 +103,14 @@ def test_gui_correct_offset(mw, correct_offset):
         assert np.allclose(h5.attrs["imaging:pixel size"],
                            0.666,
                            atol=0, rtol=1e-5)
-        assert ("bg_off" in h5["events"]) == correct_offset
+        if add_flickering:
+            assert h5.attrs["pipeline:dcnum background"] \
+                   == "sparsemed:k=200^s=1^t=0^f=0.8^o=1"
+            assert "bg_off" in h5["events"]
+        else:
+            assert h5.attrs["pipeline:dcnum background"] \
+                   == "sparsemed:k=200^s=1^t=0^f=0.8^o=0"
+            assert "bg_off" not in h5["events"]
 
 
 def test_gui_segm_torch_model(mw, qtbot, monkeypatch):

tests/test_gui_manager.py

@@ -79,15 +79,15 @@ def test_manager_run_defaults():
     # wait for the thread to join
     mg.join()
 
-    assert mg[0]["progress"] == 1
     assert mg[0]["state"] == "done"
+    assert mg[0]["progress"] == 1
     assert mg[0]["path"] == str(path)
     assert mg.current_index == 0
     assert not mg.is_busy()
     # default pipeline may change in dcnum
     assert mg.get_runner(0).ppid == (f"{ppid.DCNUM_PPID_GENERATION}|"
                                      "hdf:p=0.2645^i=0|"
-                                     "sparsemed:k=200^s=1^t=0^f=0.8^o=1|"
+                                     "sparsemed:k=200^s=1^t=0^f=0.8^o=0|"
                                      "thresh:t=-6:cle=1^f=1^clo=2|"
                                      "legacy:b=1^h=1^v=1|"
                                      "norm:o=0^s=10")