Revert "Merge branch 'stacked-images' of github.com:LCOGT/banzai into…

… actions-e2e"

This reverts commit 321a547, reversing
changes made to 433ecbf.

banzai/bias.py

@@ -31,9 +31,9 @@ def calibration_type(self):
         return 'bias'
 
     def apply_master_calibration(self, image, master_calibration_image):
-        image -= master_calibration_image.bias_level * image.n_sub_exposures
+        image -= master_calibration_image.bias_level
         image.meta['BIASLVL'] = master_calibration_image.bias_level, 'Bias level that was removed after overscan'
-        image -= master_calibration_image * image.n_sub_exposures
+        image -= master_calibration_image
         image.meta['L1IDBIAS'] = master_calibration_image.filename, 'ID of bias frame'
         image.meta['L1STATBI'] = 1, "Status flag for bias frame correction"
         return image
@@ -65,7 +65,7 @@ def __init__(self, runtime_context):
     def do_stage(self, image):
         bias_level = stats.sigma_clipped_mean(image.data, 3.5, mask=image.mask)
         image -= bias_level
-        image.meta['BIASLVL'] = bias_level / image.n_sub_exposures, 'Bias level that was removed after overscan'
+        image.meta['BIASLVL'] = bias_level, 'Bias level that was removed after overscan'
         return image
 
 

banzai/data.py

@@ -132,7 +132,7 @@ def __init__(self, data: Union[np.array, Table], meta: fits.Header,
                  mask: np.array = None, name: str = '', uncertainty: np.array = None, memmap=True):
         super().__init__(data=data, meta=meta, mask=mask, name=name, memmap=memmap)
         if uncertainty is None:
-            uncertainty = self.read_noise * self.n_sub_exposures * np.ones(data.shape, dtype=data.dtype) / self.gain
+            uncertainty = self.read_noise * np.ones(data.shape, dtype=data.dtype) / self.gain
         self.uncertainty = self._init_array(uncertainty)
         self._detector_section = Section.parse_region_keyword(self.meta.get('DETSEC'))
         self._data_section = Section.parse_region_keyword(self.meta.get('DATASEC'))
@@ -155,15 +155,6 @@ def __imul__(self, value):
         self.meta['MAXLIN'] *= value
         return self
 
-    def __mul__(self, value):
-        output = CCDData(self.data * value, meta=self.meta.copy(),
-                         name=self.name, uncertainty=self.uncertainty * value,
-                         mask=self.mask.copy(), memmap=self.memmap)
-        output.meta['SATURATE'] *= value
-        output.meta['GAIN'] /= value
-        output.meta['MAXLIN'] *= value
-        return output
-
     def __itruediv__(self, value):
         if isinstance(value, CCDData):
             self.uncertainty = np.abs(self.data / value.data) * \
@@ -311,10 +302,6 @@ def data_section(self, section):
         self.meta['DATASEC'] = section.to_region_keyword()
         self._data_section = section
 
-    @property
-    def n_sub_exposures(self):
-        return self.meta.get('NSUBREAD', 1)
-
     def rebin(self, binning):
         # TODO: Implement me
         return self

banzai/frames.py

@@ -195,10 +195,7 @@ def binning(self):
         return [min(x_binnings), min(y_binnings)]
 
     def __sub__(self, other):
-        if isinstance(other, ObservationFrame):
-            return self.primary_hdu - other.primary_hdu
-        else:
-            return self.primary_hdu - other
+        return self.primary_hdu - other.primary_hdu
 
     def __isub__(self, other):
         if isinstance(other, ObservationFrame):
@@ -207,12 +204,6 @@ def __isub__(self, other):
             self.primary_hdu.__isub__(other)
         return self
 
-    def __mul__(self, other):
-        if isinstance(other, ObservationFrame):
-            return self.primary_hdu.__mul__(other.primary_hdu)
-        else:
-            return self.primary_hdu.__mul__(other)
-
     def __imul__(self, other):
         if isinstance(other, ObservationFrame):
             self.primary_hdu.__imul__(other.primary_hdu)

banzai/lco.py

@@ -450,7 +450,7 @@ def open(self, file_info, runtime_context) -> Optional[ObservationFrame]:
                     # update datasec/trimsec for fs01
                     if hdu.header.get('INSTRUME') == 'fs01':
                         self._update_fs01_sections(hdu)
-                    if hdu.data.dtype == np.uint16 or hdu.data.dtype == np.uint32:
+                    if hdu.data.dtype == np.uint16:
                         hdu.data = hdu.data.astype(np.float64)
                     # check if we need to propagate any header keywords from the primary header
                     if primary_hdu is not None:

banzai/readnoise.py

@@ -1,6 +1,5 @@
 from banzai.calibrations import CalibrationUser
 from banzai.logs import format_exception, get_logger
-import numpy as np
 
 logger = get_logger()
 
@@ -9,7 +8,7 @@ class ReadNoiseLoader(CalibrationUser):
     def apply_master_calibration(self, image, master_calibration_image):
         try:
             for image_extension, readnoise_extension in zip(image.ccd_hdus, master_calibration_image.ccd_hdus):
-                image_extension.add_uncertainty(readnoise_extension.data * np.sqrt(image_extension.n_sub_exposures))
+                image_extension.add_uncertainty(readnoise_extension.data)
         except:
             logger.error(f"Can't add READNOISE to image, stopping reduction: {format_exception()}", image=image)
             return None

banzai/settings.py

@@ -54,8 +54,8 @@
 EXTRA_STAGES = {'BIAS': ['banzai.bias.BiasMasterLevelSubtractor', 'banzai.bias.BiasComparer'],
                 'DARK': ['banzai.dark.DarkNormalizer', 'banzai.dark.DarkTemperatureChecker',
                          'banzai.dark.DarkComparer'],
-                'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer',
-                            'banzai.qc.PatternNoiseDetector', 'banzai.flats.FlatComparer'],
+                'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer', 'banzai.qc.PatternNoiseDetector',
+                            'banzai.flats.FlatComparer'],
                 'STANDARD': None,
                 'EXPOSE': None,
                 'EXPERIMENTAL': None}

banzai/tests/data/test_data.dat

@@ -304,20 +304,29 @@ frameid filename filter site instrument
 33347343 tst1m0XX-ep04-20200324-0018-d00.fits.fz ip*diffuser tst ep04
 33347344 tst1m0XX-ep04-20200324-0019-d00.fits.fz ip*diffuser tst ep04
 33347345 tst1m0XX-ep04-20200324-0020-d00.fits.fz ip*diffuser tst ep04
-73414350 lsc0m476-sq34-20240723-0301-e00.fits.fz B lsc sq34
-73422225 lsc0m476-sq34-20240723-0408-x00.fits.fz B lsc sq34
-73409803 lsc0m476-sq34-20240723-0057-f00.fits.fz B lsc sq34
-73409794 lsc0m476-sq34-20240723-0056-f00.fits.fz B lsc sq34
-73409788 lsc0m476-sq34-20240723-0055-f00.fits.fz B lsc sq34
-73409785 lsc0m476-sq34-20240723-0054-f00.fits.fz B lsc sq34
-73215904 lsc0m476-sq34-20240723-0053-f00.fits.fz B lsc sq34
-73429574 lsc0m476-sq34-20240723-0548-d00.fits.fz opaque lsc sq34
-73429727 lsc0m476-sq34-20240723-0549-d00.fits.fz opaque lsc sq34
-73429774 lsc0m476-sq34-20240723-0550-d00.fits.fz opaque lsc sq34
-73429865 lsc0m476-sq34-20240723-0551-d00.fits.fz opaque lsc sq34
-73429912 lsc0m476-sq34-20240723-0552-d00.fits.fz opaque lsc sq34
-73429436 lsc0m476-sq34-20240723-0535-b00.fits.fz opaque lsc sq34
-73429432 lsc0m476-sq34-20240723-0536-b00.fits.fz opaque lsc sq34
-73429437 lsc0m476-sq34-20240723-0537-b00.fits.fz opaque lsc sq34
-73429439 lsc0m476-sq34-20240723-0538-b00.fits.fz opaque lsc sq34
-73429443 lsc0m476-sq34-20240723-0539-b00.fits.fz opaque lsc sq34
+56016937 ogg0m404-sq30-20221123-0037-b00.fits.fz opaque ogg sq30
+56016931 ogg0m404-sq30-20221123-0036-b00.fits.fz opaque ogg sq30
+56016928 ogg0m404-sq30-20221123-0035-b00.fits.fz opaque ogg sq30
+56016925 ogg0m404-sq30-20221123-0034-b00.fits.fz opaque ogg sq30
+56016920 ogg0m404-sq30-20221123-0033-b00.fits.fz opaque ogg sq30
+56016916 ogg0m404-sq30-20221123-0032-b00.fits.fz opaque ogg sq30
+56016912 ogg0m404-sq30-20221123-0031-b00.fits.fz opaque ogg sq30
+56016909 ogg0m404-sq30-20221123-0030-b00.fits.fz opaque ogg sq30
+56016906 ogg0m404-sq30-20221123-0029-b00.fits.fz opaque ogg sq30
+56016903 ogg0m404-sq30-20221123-0028-b00.fits.fz opaque ogg sq30
+56016899 ogg0m404-sq30-20221123-0027-b00.fits.fz opaque ogg sq30
+56016898 ogg0m404-sq30-20221123-0026-b00.fits.fz opaque ogg sq30
+56016896 ogg0m404-sq30-20221123-0025-b00.fits.fz opaque ogg sq30
+56016891 ogg0m404-sq30-20221123-0024-b00.fits.fz opaque ogg sq30
+56016889 ogg0m404-sq30-20221123-0023-b00.fits.fz opaque ogg sq30
+56017384 ogg0m404-sq30-20221123-0042-d00.fits.fz opaque ogg sq30
+56017294 ogg0m404-sq30-20221123-0041-d00.fits.fz opaque ogg sq30
+56017205 ogg0m404-sq30-20221123-0040-d00.fits.fz opaque ogg sq30
+56017136 ogg0m404-sq30-20221123-0039-d00.fits.fz opaque ogg sq30
+56017044 ogg0m404-sq30-20221123-0038-d00.fits.fz opaque ogg sq30
+56018344 ogg0m404-sq30-20221123-0067-f00.fits.fz rp ogg sq30
+56018340 ogg0m404-sq30-20221123-0066-f00.fits.fz rp ogg sq30
+56018336 ogg0m404-sq30-20221123-0065-f00.fits.fz rp ogg sq30
+56018331 ogg0m404-sq30-20221123-0064-f00.fits.fz rp ogg sq30
+56018328 ogg0m404-sq30-20221123-0063-f00.fits.fz rp ogg sq30
+56127852 ogg0m404-sq30-20221126-0098-e00.fits.fz rp ogg sq30

banzai/tests/test_munge.py

@@ -55,9 +55,7 @@ def test_defaults_do_not_override_header():
 
 def test_image_no_saturate_header_or_default():
     with pytest.raises(MissingSaturate):
-        fake_data = FakeCCDData(meta={'INSTRUME': 'blah27'})
-        fake_data.meta.pop('SATURATE')
-        LCOFrameFactory._init_saturate(FakeLCOObservationFrame(hdu_list=[fake_data]))
+        LCOFrameFactory._init_saturate(FakeLCOObservationFrame(hdu_list=[FakeCCDData(meta={'INSTRUME': 'blah27'})]))
 
 
 def test_saturate_no_default_but_header():

banzai/tests/utils.py

@@ -43,13 +43,7 @@ def __init__(self, image_multiplier=1.0, nx=101, ny=103, name='test_image', read
             self.uncertainty = self.read_noise * np.ones(self.data.shape, dtype=self.data.dtype)
         else:
             self.uncertainty = uncertainty
-        self.memmap = True
-        if 'SATURATE' not in self.meta:
-            self.meta['SATURATE'] = 65535.0
-        if 'GAIN' not in self.meta:
-            self.meta['GAIN'] = 1.0
-        if 'MAXLIN' not in self.meta:
-            self.meta['MAXLIN'] = 65535.0
+
         for keyword in kwargs:
             setattr(self, keyword, kwargs[keyword])
 
@@ -69,7 +63,7 @@ def __init__(self, hdu_list=None, file_path='/tmp/test_image.fits', instrument=N
         self._file_path = file_path
         self.is_bad = False
         self.hdu_order = ['SCI', 'CAT', 'BPM', 'ERR']
-        self.n_sub_exposures = 1
+
         for keyword in kwargs:
             setattr(self, keyword, kwargs[keyword])
 
@@ -165,7 +159,6 @@ def __init__(self, id=0, site='', camera='', enclosure='', telescope='', type=''
         self.nx = 4096
         self.ny = 4096
 
-
 class FakeCalImage:
     def __init__(self):
         self.frameid = 1234

setup.cfg

@@ -61,7 +61,7 @@ install_requires =
     pytest>=4.0
     pyyaml
     psycopg2-binary
-    celery[redis]>5
+    celery[redis]>=4.3.1,<5
     apscheduler
     python-dateutil
     ocs_ingester>=3.0.4,<4.0.0