diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d43efe84..4ea2d474 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -43,9 +43,21 @@ jobs: # Wait for banzai to be ready kubectl wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test - - name: Test Master Bias Creation + - name: Test Super Bias Creation run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias + + - name: Test Super Dark Creation + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark + + - name: Test Super Flat Creation + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat + + - name: Test Science Frame Creation + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files - name: Cleanup run: | diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3c8e4e77..02bce075 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -18,11 +18,6 @@ jobs: python-version: 3.9 toxenv: build_docs - - name: Python 3.9 with developer version of astropy and numpy - os: ubuntu-latest - python-version: 3.9 - toxenv: py39-test-devdeps - - name: Python 3.8 with minimal dependencies os: ubuntu-latest python-version: '3.8' diff --git a/CHANGES.md b/CHANGES.md index 48faa0bf..68106182 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,18 +1,22 @@ -1.17.0 (2023-04-24) +1.18.0 (2024-08-09) +- Added support for frames that are composed of sub-exposures that + are stacked at site + +1.17.0 (2024-04-24) ------------------- - We now omit sources in the photometry stage that have an area larger than 1000 pixels as they lead to long processing times and are almost invariably spurious. -1.16.1 (2023-04-23) +1.16.1 (2024-04-23) ------------------- - Correction to aperture photometry. We were incorrectly using the radius instead of the diameter -1.16.0 (2023-04-18) +1.16.0 (2024-04-18) ------------------- - Calibration frames are now associated with output data products rather than frames so that we have more than one calibration data product produced per frame. -1.15.2 (2023-04-12) +1.15.2 (2024-04-12) ------------------- - Fix to fpacking data when the image data array is None diff --git a/Dockerfile b/Dockerfile index 2f7d5146..f9318952 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,3 +24,5 @@ COPY --chown=10087:10000 . /lco/banzai ENV PATH /home/archive/envs/banzai/bin:$PATH RUN /home/archive/envs/banzai/bin/pip install --no-cache-dir /lco/banzai/ + +RUN cp /lco/banzai/pytest.ini /home/archive/pytest.ini diff --git a/banzai/bias.py b/banzai/bias.py index 19429727..c49e5330 100755 --- a/banzai/bias.py +++ b/banzai/bias.py @@ -17,6 +17,8 @@ def calibration_type(self): return 'BIAS' def make_master_calibration_frame(self, images): + for image in images: + image /= image.n_sub_exposures master_image = super(BiasMaker, self).make_master_calibration_frame(images) master_image.bias_level = np.mean([image.bias_level for image in images if image.bias_level is not None]) return master_image @@ -31,9 +33,9 @@ def calibration_type(self): return 'bias' def apply_master_calibration(self, image, master_calibration_image): - image -= master_calibration_image.bias_level + image -= master_calibration_image.bias_level * image.n_sub_exposures image.meta['BIASLVL'] = master_calibration_image.bias_level, 'Bias level that was removed after overscan' - image -= master_calibration_image + image -= master_calibration_image * image.n_sub_exposures image.meta['L1IDBIAS'] = master_calibration_image.filename, 'ID of bias frame' image.meta['L1STATBI'] = 1, "Status flag for bias frame correction" return image @@ -65,7 +67,9 @@ def __init__(self, runtime_context): def do_stage(self, image): bias_level = stats.sigma_clipped_mean(image.data, 3.5, mask=image.mask) image -= bias_level - image.meta['BIASLVL'] = bias_level, 'Bias level that was removed after overscan' + # This is only run for bias frames and n_sub_exposures should always be 1 + # but we divide it here for consistency to cover pathological uses cases + image.meta['BIASLVL'] = bias_level / image.n_sub_exposures, 'Bias level that was removed after overscan' return image diff --git a/banzai/data.py b/banzai/data.py index 4adf7322..19f76c3d 100644 --- a/banzai/data.py +++ b/banzai/data.py @@ -132,7 +132,8 @@ def __init__(self, data: Union[np.array, Table], meta: fits.Header, mask: np.array = None, name: str = '', uncertainty: np.array = None, memmap=True): super().__init__(data=data, meta=meta, mask=mask, name=name, memmap=memmap) if uncertainty is None: - uncertainty = self.read_noise * np.ones(data.shape, dtype=data.dtype) / self.gain + uncertainty = self.read_noise * np.sqrt(self.n_sub_exposures) * np.ones(data.shape, dtype=data.dtype) + uncertainty /= self.gain self.uncertainty = self._init_array(uncertainty) self._detector_section = Section.parse_region_keyword(self.meta.get('DETSEC')) self._data_section = Section.parse_region_keyword(self.meta.get('DATASEC')) @@ -155,6 +156,15 @@ def __imul__(self, value): self.meta['MAXLIN'] *= value return self + def __mul__(self, value): + output = CCDData(self.data * value, meta=self.meta.copy(), + name=self.name, uncertainty=self.uncertainty * value, + mask=self.mask.copy(), memmap=self.memmap) + output.meta['SATURATE'] *= value + output.meta['GAIN'] /= value + output.meta['MAXLIN'] *= value + return output + def __itruediv__(self, value): if isinstance(value, CCDData): self.uncertainty = np.abs(self.data / value.data) * \ @@ -176,7 +186,7 @@ def to_fits(self, context): def __del__(self): super().__del__() - del self.uncertainty + del self._uncertainty def __isub__(self, value): if isinstance(value, CCDData): @@ -192,9 +202,14 @@ def __sub__(self, other): return type(self)(data=self.data - other.data, meta=self.meta, mask=self.mask | other.mask, uncertainty=uncertainty) - def add_uncertainty(self, readnoise: np.array): - self._validate_array(readnoise) - self.uncertainty = self._init_array(readnoise) + @property + def uncertainty(self): + return self._uncertainty + + @uncertainty.setter + def uncertainty(self, value: np.array): + self._validate_array(value) + self._uncertainty = self._init_array(value) def signal_to_noise(self): return np.abs(self.data) / self.uncertainty @@ -302,6 +317,17 @@ def data_section(self, section): self.meta['DATASEC'] = section.to_region_keyword() self._data_section = section + @property + def n_sub_exposures(self): + n_exposures = self.meta.get('NSUBREAD', 1) + if str(n_exposures).lower() in ['n/a', 'unknown', 'none', '']: + n_exposures = 1 + return n_exposures + + @n_sub_exposures.setter + def n_sub_exposures(self, value): + self.meta['NSUBREAD'] = value + def rebin(self, binning): # TODO: Implement me return self diff --git a/banzai/frames.py b/banzai/frames.py index 093a5734..47543aa5 100644 --- a/banzai/frames.py +++ b/banzai/frames.py @@ -78,6 +78,14 @@ def background(self): def background(self, value): self.primary_hdu.background = value + @property + def n_sub_exposures(self): + return self.primary_hdu.n_sub_exposures + + @n_sub_exposures.setter + def n_sub_exposures(self, value): + self.primary_hdu.n_sub_exposures = value + @abc.abstractmethod def save_processing_metadata(self, context): pass @@ -195,7 +203,10 @@ def binning(self): return [min(x_binnings), min(y_binnings)] def __sub__(self, other): - return self.primary_hdu - other.primary_hdu + if isinstance(other, ObservationFrame): + return self.primary_hdu - other.primary_hdu + else: + return self.primary_hdu - other def __isub__(self, other): if isinstance(other, ObservationFrame): @@ -204,6 +215,12 @@ def __isub__(self, other): self.primary_hdu.__isub__(other) return self + def __mul__(self, other): + if isinstance(other, ObservationFrame): + return self.primary_hdu.__mul__(other.primary_hdu) + else: + return self.primary_hdu.__mul__(other) + def __imul__(self, other): if isinstance(other, ObservationFrame): self.primary_hdu.__imul__(other.primary_hdu) diff --git a/banzai/lco.py b/banzai/lco.py index 6f801318..67447d57 100644 --- a/banzai/lco.py +++ b/banzai/lco.py @@ -450,7 +450,7 @@ def open(self, file_info, runtime_context) -> Optional[ObservationFrame]: # update datasec/trimsec for fs01 if hdu.header.get('INSTRUME') == 'fs01': self._update_fs01_sections(hdu) - if hdu.data.dtype == np.uint16: + if hdu.data.dtype == np.uint16 or hdu.data.dtype == np.uint32: hdu.data = hdu.data.astype(np.float64) # check if we need to propagate any header keywords from the primary header if primary_hdu is not None: diff --git a/banzai/readnoise.py b/banzai/readnoise.py index ba11ab69..00acd440 100644 --- a/banzai/readnoise.py +++ b/banzai/readnoise.py @@ -1,5 +1,6 @@ from banzai.calibrations import CalibrationUser from banzai.logs import format_exception, get_logger +import numpy as np logger = get_logger() @@ -8,7 +9,7 @@ class ReadNoiseLoader(CalibrationUser): def apply_master_calibration(self, image, master_calibration_image): try: for image_extension, readnoise_extension in zip(image.ccd_hdus, master_calibration_image.ccd_hdus): - image_extension.add_uncertainty(readnoise_extension.data) + image_extension.uncertainty = readnoise_extension.data * np.sqrt(image_extension.n_sub_exposures) except: logger.error(f"Can't add READNOISE to image, stopping reduction: {format_exception()}", image=image) return None diff --git a/banzai/settings.py b/banzai/settings.py index fdfb98b5..7123c5b9 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -54,8 +54,8 @@ EXTRA_STAGES = {'BIAS': ['banzai.bias.BiasMasterLevelSubtractor', 'banzai.bias.BiasComparer'], 'DARK': ['banzai.dark.DarkNormalizer', 'banzai.dark.DarkTemperatureChecker', 'banzai.dark.DarkComparer'], - 'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer', 'banzai.qc.PatternNoiseDetector', - 'banzai.flats.FlatComparer'], + 'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer', + 'banzai.qc.PatternNoiseDetector', 'banzai.flats.FlatComparer'], 'STANDARD': None, 'EXPOSE': None, 'EXPERIMENTAL': None} @@ -80,7 +80,7 @@ 'elp': {'minute': 0, 'hour': 23}, 'ogg': {'minute': 0, 'hour': 3}} -ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', ' ') +ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', 'http://astrometry.lco.gtn/catalog') CALIBRATION_FILENAME_FUNCTIONS = {'BIAS': ('banzai.utils.file_utils.config_to_filename', 'banzai.utils.file_utils.ccdsum_to_filename'), diff --git a/banzai/tests/data/configdb_example.json b/banzai/tests/data/configdb_example.json index 250900f7..b6ef51f2 100644 --- a/banzai/tests/data/configdb_example.json +++ b/banzai/tests/data/configdb_example.json @@ -2174,7 +2174,379 @@ "horizon": 15.0, "ha_limit_pos": 4.4666667, "ha_limit_neg": -4.533333, - "instrument_set": [], + "instrument_set": [{ + "id": 1496, + "code": "sq34", + "state": "SCHEDULABLE", + "telescope": "http://configdb.lco.gtn/telescopes/13/", + "autoguider_camera": { + "id": 1270, + "code": "sq34", + "camera_type": { + "id": 189, + "size": "117x78", + "pscale": 0.734, + "name": "0m4 QHY600", + "code": "0m4-SciCam-QHY600", + "pixels_x": 9576, + "pixels_y": 6388, + "max_rois": 0 + }, + "orientation": 0.0, + "optical_elements": { + "filters": "OIII,SII,Astrodon-Exo,w,opaque,up,rp,ip,gp,zs,V,B,H-Alpha" + }, + "optical_element_groups": [ + { + "name": "DeltaRho QHY600 default wheel", + "type": "filters", + "optical_elements": [ + { + "name": "OIII", + "code": "OIII", + "schedulable": true + }, + { + "name": "S[II]", + "code": "SII", + "schedulable": true + }, + { + "name": "Astrodon-Exo", + "code": "Astrodon-Exo", + "schedulable": true + }, + { + "name": "PanSTARRS-w", + "code": "w", + "schedulable": true + }, + { + "name": "Opaque", + "code": "opaque", + "schedulable": false + }, + { + "name": "SDSS-up", + "code": "up", + "schedulable": true + }, + { + "name": "SDSS-rp", + "code": "rp", + "schedulable": true + }, + { + "name": "SDSS-ip", + "code": "ip", + "schedulable": true + }, + { + "name": "SDSS-gp", + "code": "gp", + "schedulable": true + }, + { + "name": "PanSTARRS-Z", + "code": "zs", + "schedulable": true + }, + { + "name": "Bessell-V", + "code": "V", + "schedulable": true + }, + { + "name": "Bessell-B", + "code": "B", + "schedulable": true + }, + { + "name": "H Alpha", + "code": "H-Alpha", + "schedulable": true + } + ], + "element_change_overhead": 2.0, + "default": "" + } + ], + "host": "inst.0m4a.aqwa.lsc.lco.gtn" + }, + "science_cameras": [ + { + "id": 1270, + "code": "sq34", + "camera_type": { + "id": 189, + "size": "117x78", + "pscale": 0.734, + "name": "0m4 QHY600", + "code": "0m4-SciCam-QHY600", + "pixels_x": 9576, + "pixels_y": 6388, + "max_rois": 0 + }, + "orientation": 0.0, + "optical_elements": { + "filters": "OIII,SII,Astrodon-Exo,w,opaque,up,rp,ip,gp,zs,V,B,H-Alpha" + }, + "optical_element_groups": [ + { + "name": "DeltaRho QHY600 default wheel", + "type": "filters", + "optical_elements": [ + { + "name": "OIII", + "code": "OIII", + "schedulable": true + }, + { + "name": "S[II]", + "code": "SII", + "schedulable": true + }, + { + "name": "Astrodon-Exo", + "code": "Astrodon-Exo", + "schedulable": true + }, + { + "name": "PanSTARRS-w", + "code": "w", + "schedulable": true + }, + { + "name": "Opaque", + "code": "opaque", + "schedulable": false + }, + { + "name": "SDSS-up", + "code": "up", + "schedulable": true + }, + { + "name": "SDSS-rp", + "code": "rp", + "schedulable": true + }, + { + "name": "SDSS-ip", + "code": "ip", + "schedulable": true + }, + { + "name": "SDSS-gp", + "code": "gp", + "schedulable": true + }, + { + "name": "PanSTARRS-Z", + "code": "zs", + "schedulable": true + }, + { + "name": "Bessell-V", + "code": "V", + "schedulable": true + }, + { + "name": "Bessell-B", + "code": "B", + "schedulable": true + }, + { + "name": "H Alpha", + "code": "H-Alpha", + "schedulable": true + } + ], + "element_change_overhead": 2.0, + "default": "" + } + ], + "host": "inst.0m4a.aqwa.lsc.lco.gtn" + } + ], + "instrument_type": { + "id": 100, + "name": "0m4 SCICAM QHY600", + "code": "0m4-SciCam-QHY600", + "fixed_overhead_per_exposure": 1.0, + "instrument_category": "IMAGE", + "observation_front_padding": 90.0, + "acquire_exposure_time": 0.0, + "default_configuration_type": "EXPOSE", + "mode_types": [ + { + "type": "guiding", + "modes": [ + { + "name": "Guide Off", + "overhead": 0.0, + "code": "OFF", + "schedulable": true, + "validation_schema": {} + }, + { + "name": "Guide On", + "overhead": 0.0, + "code": "ON", + "schedulable": true, + "validation_schema": {} + } + ], + "default": "ON" + }, + { + "type": "readout", + "modes": [ + { + "name": "QHY600 Central 30x30 arcmin", + "overhead": 4.0, + "code": "central30x30", + "schedulable": true, + "validation_schema": {} + }, + { + "name": "QHY600 Full Frame Readout", + "overhead": 9.0, + "code": "full_frame", + "schedulable": true, + "validation_schema": { + "extra_params": { + "type": "dict", + "schema": { + "bin_x": { + "type": "integer", + "allowed": [ + 1 + ], + "default": 1 + }, + "bin_y": { + "type": "integer", + "allowed": [ + 1 + ], + "default": 1 + } + }, + "default": {}, + "required": true + }, + "exposure_count": { + "max": 50 + } + } + } + ], + "default": "central30x30" + }, + { + "type": "acquisition", + "modes": [ + { + "name": "Acquire Off", + "overhead": 0.0, + "code": "OFF", + "schedulable": true, + "validation_schema": {} + } + ], + "default": "OFF" + } + ], + "default_acceptability_threshold": 90.0, + "config_front_padding": 5.0, + "allow_self_guiding": true, + "configuration_types": [ + { + "name": "Exposure", + "code": "EXPOSE", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": false, + "requires_optical_elements": true, + "validation_schema": {} + }, + { + "name": "Exposure Sequence", + "code": "REPEAT_EXPOSE", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": false, + "requires_optical_elements": true, + "validation_schema": {} + }, + { + "name": "AUTO_FOCUS", + "code": "AUTO_FOCUS", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": false, + "requires_optical_elements": true, + "validation_schema": {} + }, + { + "name": "SKY_FLAT", + "code": "SKY_FLAT", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": false, + "requires_optical_elements": true, + "validation_schema": { + "exposure_time": { + "type": "float", + "default": 1 + } + } + }, + { + "name": "BIAS", + "code": "BIAS", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": true, + "requires_optical_elements": false, + "validation_schema": {} + }, + { + "name": "DARK", + "code": "DARK", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": true, + "requires_optical_elements": false, + "validation_schema": {} + }, + { + "name": "STANDARD", + "code": "STANDARD", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": false, + "requires_optical_elements": true, + "validation_schema": {} + } + ], + "validation_schema": { + "extra_params": { + "type": "dict", + "schema": { + "defocus": { + "max": 5.0, + "min": -5.0, + "type": "float" + } + } + } + } + }, + "__str__": "lsc.aqwa.0m4a.sq34" + } + ], "__str__": "lsc.aqwa.0m4a" } ], diff --git a/banzai/tests/data/test_data.dat b/banzai/tests/data/test_data.dat index f7bc38f7..42c74f5d 100644 --- a/banzai/tests/data/test_data.dat +++ b/banzai/tests/data/test_data.dat @@ -304,29 +304,20 @@ frameid filename filter site instrument 33347343 tst1m0XX-ep04-20200324-0018-d00.fits.fz ip*diffuser tst ep04 33347344 tst1m0XX-ep04-20200324-0019-d00.fits.fz ip*diffuser tst ep04 33347345 tst1m0XX-ep04-20200324-0020-d00.fits.fz ip*diffuser tst ep04 -56016937 ogg0m404-sq30-20221123-0037-b00.fits.fz opaque ogg sq30 -56016931 ogg0m404-sq30-20221123-0036-b00.fits.fz opaque ogg sq30 -56016928 ogg0m404-sq30-20221123-0035-b00.fits.fz opaque ogg sq30 -56016925 ogg0m404-sq30-20221123-0034-b00.fits.fz opaque ogg sq30 -56016920 ogg0m404-sq30-20221123-0033-b00.fits.fz opaque ogg sq30 -56016916 ogg0m404-sq30-20221123-0032-b00.fits.fz opaque ogg sq30 -56016912 ogg0m404-sq30-20221123-0031-b00.fits.fz opaque ogg sq30 -56016909 ogg0m404-sq30-20221123-0030-b00.fits.fz opaque ogg sq30 -56016906 ogg0m404-sq30-20221123-0029-b00.fits.fz opaque ogg sq30 -56016903 ogg0m404-sq30-20221123-0028-b00.fits.fz opaque ogg sq30 -56016899 ogg0m404-sq30-20221123-0027-b00.fits.fz opaque ogg sq30 -56016898 ogg0m404-sq30-20221123-0026-b00.fits.fz opaque ogg sq30 -56016896 ogg0m404-sq30-20221123-0025-b00.fits.fz opaque ogg sq30 -56016891 ogg0m404-sq30-20221123-0024-b00.fits.fz opaque ogg sq30 -56016889 ogg0m404-sq30-20221123-0023-b00.fits.fz opaque ogg sq30 -56017384 ogg0m404-sq30-20221123-0042-d00.fits.fz opaque ogg sq30 -56017294 ogg0m404-sq30-20221123-0041-d00.fits.fz opaque ogg sq30 -56017205 ogg0m404-sq30-20221123-0040-d00.fits.fz opaque ogg sq30 -56017136 ogg0m404-sq30-20221123-0039-d00.fits.fz opaque ogg sq30 -56017044 ogg0m404-sq30-20221123-0038-d00.fits.fz opaque ogg sq30 -56018344 ogg0m404-sq30-20221123-0067-f00.fits.fz rp ogg sq30 -56018340 ogg0m404-sq30-20221123-0066-f00.fits.fz rp ogg sq30 -56018336 ogg0m404-sq30-20221123-0065-f00.fits.fz rp ogg sq30 -56018331 ogg0m404-sq30-20221123-0064-f00.fits.fz rp ogg sq30 -56018328 ogg0m404-sq30-20221123-0063-f00.fits.fz rp ogg sq30 -56127852 ogg0m404-sq30-20221126-0098-e00.fits.fz rp ogg sq30 +73890113 lsc0m476-sq34-20240809-0144-e00.fits.fz B lsc sq34 +73422225 lsc0m476-sq34-20240723-0408-x00.fits.fz B lsc sq34 +73409803 lsc0m476-sq34-20240723-0057-f00.fits.fz B lsc sq34 +73409794 lsc0m476-sq34-20240723-0056-f00.fits.fz B lsc sq34 +73409788 lsc0m476-sq34-20240723-0055-f00.fits.fz B lsc sq34 +73409785 lsc0m476-sq34-20240723-0054-f00.fits.fz B lsc sq34 +73409786 lsc0m476-sq34-20240723-0053-f00.fits.fz B lsc sq34 +73429574 lsc0m476-sq34-20240723-0548-d00.fits.fz opaque lsc sq34 +73429727 lsc0m476-sq34-20240723-0549-d00.fits.fz opaque lsc sq34 +73429774 lsc0m476-sq34-20240723-0550-d00.fits.fz opaque lsc sq34 +73429865 lsc0m476-sq34-20240723-0551-d00.fits.fz opaque lsc sq34 +73429912 lsc0m476-sq34-20240723-0552-d00.fits.fz opaque lsc sq34 +73429436 lsc0m476-sq34-20240723-0535-b00.fits.fz opaque lsc sq34 +73429432 lsc0m476-sq34-20240723-0536-b00.fits.fz opaque lsc sq34 +73429437 lsc0m476-sq34-20240723-0537-b00.fits.fz opaque lsc sq34 +73429439 lsc0m476-sq34-20240723-0538-b00.fits.fz opaque lsc sq34 +73429443 lsc0m476-sq34-20240723-0539-b00.fits.fz opaque lsc sq34 diff --git a/banzai/tests/data/test_obs_portal_response_lsc_20240723.json b/banzai/tests/data/test_obs_portal_response_lsc_20240723.json new file mode 100644 index 00000000..ada747a8 --- /dev/null +++ b/banzai/tests/data/test_obs_portal_response_lsc_20240723.json @@ -0,0 +1,620 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "created": "2024-07-23T04:05:53.173401Z", + "enclosure": "aqwa", + "end": "2024-07-23T22:36:19.418683Z", + "id": 631156776, + "ipp_value": 1.0, + "modified": "2024-07-23T22:30:04.846999Z", + "name": "LCOGT", + "observation_type": "DIRECT", + "priority": 10, + "proposal": "calibrate", + "request": { + "acceptability_threshold": 90.0, + "configuration_repeats": 1, + "configurations": [ + { + "acquisition_config": { + "extra_params": {}, + "mode": "OFF" + }, + "configuration_status": 812512180, + "constraints": { + "extra_params": {}, + "max_airmass": 20.0, + "max_lunar_phase": 1.0, + "min_lunar_distance": 0.0 + }, + "extra_params": {}, + "guide_camera_name": "", + "guiding_config": { + "exposure_time": null, + "extra_params": {}, + "mode": "OFF", + "optical_elements": {}, + "optional": true + }, + "id": 11542373, + "instrument_configs": [ + { + "exposure_count": 5, + "exposure_time": 1.0, + "extra_params": { + "bin_x": 1, + "bin_y": 1 + }, + "mode": "full_frame", + "optical_elements": { + "filter": "B" + }, + "rois": [], + "rotator_mode": "" + } + ], + "instrument_name": "sq34", + "instrument_type": "0M4-SCICAM-QHY600", + "priority": 3, + "repeat_duration": null, + "state": "COMPLETED", + "summary": { + "end": "2024-07-23T22:22:19Z", + "events": [ + { + "description": "org.lcogt.sequencer.command.mount.HomeCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.EnableTrackingCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.focus.HomeCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.instrument.AbortCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.enclosure.OpenCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.enclosure.OpenCommand", + "state": "DONE", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.focus.HomeCommand", + "state": "DONE", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.EnableTrackingCommand", + "state": "DONE", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.telescopeservices.StowMirrorCoverCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.mount.HomeCommand", + "state": "DONE", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.focus.DeFocusCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.telescopeservices.StowMirrorCoverCommand", + "state": "DONE", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.focus.DeFocusCommand", + "state": "DONE", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.instrument.AbortCommand", + "state": "DONE", + "time": "2024-07-23T22:11:05" + }, + { + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:11:06" + }, + { + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand", + "state": "DONE", + "time": "2024-07-23T22:11:06" + }, + { + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:11:06" + }, + { + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand", + "state": "DONE", + "time": "2024-07-23T22:11:12" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:11:12" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:12:03" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:12:34" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:12:47" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:12:57" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:13:20" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:13:30" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:14:23" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:21:02" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", + "state": "DONE", + "time": "2024-07-23T22:21:03" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:21:03" + }, + { + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:21:04" + }, + { + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", + "state": "DONE", + "time": "2024-07-23T22:21:13" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:21:13" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:21:19" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", + "state": "DONE", + "time": "2024-07-23T22:21:19" + }, + { + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:21:19" + }, + { + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", + "state": "DONE", + "time": "2024-07-23T22:21:29" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:21:29" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", + "state": "DONE", + "time": "2024-07-23T22:21:35" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:21:35" + }, + { + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:21:36" + }, + { + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", + "state": "DONE", + "time": "2024-07-23T22:21:45" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:21:45" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", + "state": "DONE", + "time": "2024-07-23T22:21:51" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:21:51" + }, + { + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:21:53" + }, + { + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", + "state": "DONE", + "time": "2024-07-23T22:22:01" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:22:01" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", + "state": "DONE", + "time": "2024-07-23T22:22:08" + }, + { + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", + "state": "DONE", + "time": "2024-07-23T22:22:08" + }, + { + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:22:08" + }, + { + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", + "state": "DONE", + "time": "2024-07-23T22:22:17" + }, + { + "description": "org.lcogt.sequencer.command.instrument.AbortCommand", + "state": "PROCESSING", + "time": "2024-07-23T22:22:18" + }, + { + "description": "org.lcogt.sequencer.command.instrument.AbortCommand", + "state": "DONE", + "time": "2024-07-23T22:22:19" + } + ], + "id": 4815648, + "reason": "", + "start": "2024-07-23T22:11:04Z", + "state": "COMPLETED", + "time_completed": 0.0 + }, + "target": { + "dec": 0.0, + "epoch": null, + "extra_params": {}, + "hour_angle": 1.0, + "name": "No target", + "parallax": null, + "proper_motion_dec": null, + "proper_motion_ra": null, + "ra": null, + "type": "HOUR_ANGLE" + }, + "type": "SKY_FLAT" + } + ], + "duration": 497, + "extra_params": {}, + "id": 3596511, + "modified": "2024-07-23T22:30:04.801605Z", + "observation_note": "", + "optimization_type": "TIME", + "state": "COMPLETED" + }, + "request_group_id": 2011773, + "site": "lsc", + "start": "2024-07-23T22:04:10.911637Z", + "state": "COMPLETED", + "submitter": "eng", + "telescope": "0m4a" + }, + { + "id": 631287808, + "request": { + "id": 3596764, + "observation_note": "", + "optimization_type": "TIME", + "state": "COMPLETED", + "acceptability_threshold": 90.0, + "configuration_repeats": 1, + "extra_params": {}, + "modified": "2024-07-23T21:20:17.110554Z", + "duration": 3427, + "configurations": [ + { + "id": 11543917, + "instrument_type": "0M4-SCICAM-QHY600", + "type": "BIAS", + "repeat_duration": null, + "extra_params": {}, + "priority": 4, + "instrument_configs": [ + { + "optical_elements": { + "filter": "opaque" + }, + "mode": "full_frame", + "exposure_time": 0.0, + "exposure_count": 5, + "rotator_mode": "", + "extra_params": { + "bin_x": 1, + "bin_y": 1 + }, + "rois": [] + } + ], + "constraints": { + "max_airmass": 20.0, + "min_lunar_distance": 0.0, + "max_lunar_phase": 1.0, + "extra_params": {} + }, + "acquisition_config": { + "mode": "OFF", + "extra_params": {} + }, + "guiding_config": { + "optional": true, + "mode": "OFF", + "optical_elements": {}, + "exposure_time": null, + "extra_params": {} + }, + "target": { + "type": "HOUR_ANGLE", + "name": "No target", + "ra": null, + "dec": 0.0, + "proper_motion_ra": null, + "proper_motion_dec": null, + "parallax": null, + "epoch": null, + "hour_angle": 1.0, + "extra_params": {} + }, + "configuration_status": 812665084, + "state": "COMPLETED", + "instrument_name": "sq34", + "guide_camera_name": "", + "summary": { + "id": 4815555, + "start": "2024-07-23T20:52:00Z", + "end": "2024-07-23T20:54:31Z", + "state": "COMPLETED", + "reason": "", + "time_completed": 0.0, + "events": [ + { + "time": "2024-07-23T20:52:00", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + }, + { + "time": "2024-07-23T20:52:01", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + }, + { + "time": "2024-07-23T20:52:01", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + }, + { + "time": "2024-07-23T20:52:03", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + }, + { + "time": "2024-07-23T20:52:03", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T20:54:30", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T20:54:30", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + }, + { + "time": "2024-07-23T20:54:31", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + } + ] + } + }, + { + "id": 11543918, + "instrument_type": "0M4-SCICAM-QHY600", + "type": "DARK", + "repeat_duration": null, + "extra_params": {}, + "priority": 5, + "instrument_configs": [ + { + "optical_elements": { + "filter": "opaque" + }, + "mode": "full_frame", + "exposure_time": 300.0, + "exposure_count": 5, + "rotator_mode": "", + "extra_params": { + "bin_x": 1, + "bin_y": 1 + }, + "rois": [] + } + ], + "constraints": { + "max_airmass": 20.0, + "min_lunar_distance": 0.0, + "max_lunar_phase": 1.0, + "extra_params": {} + }, + "acquisition_config": { + "mode": "OFF", + "extra_params": {} + }, + "guiding_config": { + "optional": true, + "mode": "OFF", + "optical_elements": {}, + "exposure_time": null, + "extra_params": {} + }, + "target": { + "type": "HOUR_ANGLE", + "name": "No target", + "ra": null, + "dec": 0.0, + "proper_motion_ra": null, + "proper_motion_dec": null, + "parallax": null, + "epoch": null, + "hour_angle": 1.0, + "extra_params": {} + }, + "configuration_status": 812665085, + "state": "COMPLETED", + "instrument_name": "sq34", + "guide_camera_name": "", + "summary": { + "id": 4815574, + "start": "2024-07-23T20:54:34Z", + "end": "2024-07-23T21:20:16Z", + "state": "COMPLETED", + "reason": "", + "time_completed": 1500.0, + "events": [ + { + "time": "2024-07-23T20:54:34", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + }, + { + "time": "2024-07-23T20:54:34", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + }, + { + "time": "2024-07-23T20:54:35", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + }, + { + "time": "2024-07-23T20:54:36", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + }, + { + "time": "2024-07-23T20:54:37", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T21:20:14", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T21:20:15", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + }, + { + "time": "2024-07-23T21:20:16", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + } + ] + } + } + ] + }, + "site": "lsc", + "enclosure": "aqwa", + "telescope": "0m4a", + "start": "2024-07-23T20:25:25.334568Z", + "end": "2024-07-23T21:31:25.334568Z", + "priority": 10, + "state": "COMPLETED", + "proposal": "calibrate", + "submitter": "eng", + "name": "LCOGT", + "ipp_value": 1.0, + "observation_type": "DIRECT", + "request_group_id": 2012021, + "created": "2024-07-23T15:05:56.175376Z", + "modified": "2024-07-23T21:20:17.151237Z" + } + ] +} diff --git a/banzai/tests/data/test_precals.dat b/banzai/tests/data/test_precals.dat index e1ce81ea..dcc2aa71 100644 --- a/banzai/tests/data/test_precals.dat +++ b/banzai/tests/data/test_precals.dat @@ -7,5 +7,5 @@ frameid filename binning mode obstype dateobs site instrument 33347347 tst1m0XX-ep02-20200710-bpm-1.fits.fz '[1, 1]' 1 BPM 2020-07-10 tst ep02 33347348 tst1m0XX-ep03-20200710-bpm-1.fits.fz '[1, 1]' 1 BPM 2020-07-10 tst ep03 33347349 tst1m0XX-ep04-20200710-bpm-1.fits.fz '[1, 1]' 1 BPM 2020-07-10 tst ep04 -58186456 ogg0m404-sq30-20221123-readnoise-default-e2e.fits.fz '[1, 1]' default READNOISE 2022-11-23 ogg sq30 -56389948 ogg0m404-sq30-20221206-bpm-default.fits.fz '[1, 1]' default BPM 2022-12-06 ogg sq30 +69359146 lsc0m409-sq34-20240314-bpm-full_frame.fits.fz '[1, 1]' full_frame BPM 2024-03-14 lsc sq34 +69359157 lsc0m409-sq34-20240314-readnoise-full_frame.fits.fz '[1, 1]' full_frame READNOISE 2024-03-14 lsc sq34 diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 53180e25..e27b2106 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -20,18 +20,16 @@ spec: resources: requests: cpu: 0.1 - memory: 256Mi + memory: 512Mi limits: cpu: 1 memory: 512Mi readinessProbe: exec: command: - - redis-cli - - ping - - | - - grep - - "PONG" + - /bin/sh + - -c + - 'redis-cli ping | grep -q "PONG"' initialDelaySeconds: 5 periodSeconds: 1 - name: banzai-fits-exchange @@ -42,13 +40,14 @@ spec: cpu: 1 memory: 512Mi limits: - cpu: 2 + cpu: 4 memory: 512Mi readinessProbe: exec: command: - - rabbitmqctl - - node_health_check + - rabbitmq-diagnostics + - "-q" + - ping initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 @@ -62,7 +61,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db" + value: "sqlite:////archive/engineering/test.db?timeout=30" - name: RETRY_DELAY value: "0" - name: TASK_HOST @@ -76,7 +75,7 @@ spec: - name: API_ROOT value: "https://archive-api.lco.global/" - name: OMP_NUM_THREADS - value: "2" + value: "1" - name: FITS_EXCHANGE value: "fits_files" - name: OPENTSDB_PYTHON_METRICS_TEST_MODE @@ -93,31 +92,26 @@ spec: - --hostname - "banzai-celery-worker" - --concurrency - - "2" + - "4" - -l - "info" - "-Q" - "$(CELERY_TASK_QUEUE_NAME)" - "-n" - "celery-worker" - readinessProbe: exec: command: - - celery - - -A - - banzai - - status - - | - - grep - - "banzai-celery-worker:.*OK" + - /bin/sh + - -c + - "celery -A banzai status | grep -q '@celery-worker:.*OK'" initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 resources: requests: cpu: 4 - memory: 4Gi + memory: 5Gi limits: cpu: 8 memory: 5Gi @@ -131,7 +125,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db" + value: "sqlite:////archive/engineering/test.db?timeout=30" - name: RETRY_DELAY value: "0" - name: TASK_HOST @@ -172,23 +166,19 @@ spec: readinessProbe: exec: command: - - celery - - -A - - banzai - - status - - | - - grep - - "banzai-celery-worker:.*OK" + - /bin/sh + - -c + - 'celery -A banzai status | grep -q "large-celery-worker:.*OK"' initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 resources: requests: cpu: 2 - memory: 8Gi + memory: 7Gi limits: cpu: 3 - memory: 9Gi + memory: 7Gi - name: banzai-listener image: banzai:test-latest imagePullPolicy: IfNotPresent @@ -199,7 +189,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db" + value: "sqlite:////archive/engineering/test.db?timeout=30" - name: FITS_BROKER value: "localhost" - name: TASK_HOST @@ -222,16 +212,24 @@ spec: value: "e2e_large_task_queue" command: - banzai_run_realtime_pipeline - - "--db-address=sqlite:////archive/engineering/test.db" - - "--fpack" - - "--broker-url=localhost" + - --db-address=$(DB_ADDRESS) + - --fpack + - --broker-url=localhost resources: requests: cpu: 0.1 memory: 1Gi limits: cpu: 1 - memory: 2Gi - + memory: 1Gi + readinessProbe: + exec: + command: + - /bin/sh + - -c + - 'ps -u archive | grep -q "banzai_run_real"' + initialDelaySeconds: 5 + periodSeconds: 1 + timeoutSeconds: 10 dnsPolicy: ClusterFirst restartPolicy: Never diff --git a/banzai/tests/test_bias_level_subtractor.py b/banzai/tests/test_bias_level_subtractor.py index bdc8d37f..734e4bc7 100644 --- a/banzai/tests/test_bias_level_subtractor.py +++ b/banzai/tests/test_bias_level_subtractor.py @@ -52,3 +52,16 @@ def test_bias_master_level_subtraction_is_reasonable(set_random_seed): np.testing.assert_allclose(np.zeros(image.data.shape), image.data, atol=8 * read_noise) np.testing.assert_allclose(image.meta.get('BIASLVL'), input_bias, atol=1.0) + + +def test_multiread_bias_level(): + input_bias = 500.0 + nx, ny = 101, 103 + nreads = 5 + subtractor = BiasMasterLevelSubtractor(None) + image = FakeLCOObservationFrame(hdu_list=[FakeCCDData(image_multiplier=input_bias*nreads, nx=nx, ny=ny)]) + image.n_sub_exposures = nreads + image = subtractor.do_stage(image) + + np.testing.assert_allclose(np.zeros(image.data.shape), image.data, atol=1.0) + np.testing.assert_allclose(image.meta.get('BIASLVL'), input_bias, atol=1.0) diff --git a/banzai/tests/test_bias_maker.py b/banzai/tests/test_bias_maker.py index bcbb5bd1..4b677f0a 100644 --- a/banzai/tests/test_bias_maker.py +++ b/banzai/tests/test_bias_maker.py @@ -71,7 +71,7 @@ def test_makes_a_sensible_master_bias(mock_namer): images = [FakeLCOObservationFrame(hdu_list=[FakeCCDData(data=np.random.normal(0.0, size=(99, 99), scale=expected_readnoise), bias_level=0.0, - read_noise= expected_readnoise, + read_noise=expected_readnoise, meta=Header({'DATE-OBS': '2019-12-04T14:34:00', 'DETSEC': '[1:100,1:100]', 'DATASEC': '[1:100,1:100]', @@ -84,3 +84,29 @@ def test_makes_a_sensible_master_bias(mock_namer): assert np.abs(np.mean(master_bias)) < 0.1 actual_readnoise = np.std(master_bias) assert np.abs(actual_readnoise - expected_readnoise / (nimages ** 0.5)) < 0.2 + + +def test_multiread_bias_maker(): + nimages = 5 + nreads = 20 + images = [] + pattern_scale = 8 + nx = 101 + ny = 105 + bias_level = 100.0 + bias_pattern = np.random.normal(0.0, pattern_scale, size=(ny, nx)) + for i in range(nimages): + data = nreads * bias_pattern + image = FakeLCOObservationFrame(hdu_list=[FakeCCDData(data=data, bias_level=bias_level, + meta=Header({'DATE-OBS': '2019-12-04T14:34:00', + 'DETSEC': f'[1:{nx},1:{ny}]', + 'DATASEC': f'[1:{nx},1:{ny}]', + 'OBSTYPE': 'BIAS', 'RA': 0.0, 'DEC': 0.0}))]) + image.n_sub_exposures = nreads + images.append(image) + maker = BiasMaker(FakeContext()) + stacked_image = maker.do_stage(images) + np.testing.assert_allclose(stacked_image.meta['BIASLVL'], bias_level, atol=0.1) + + # With 20 x 5 reads, we should get down to better than 1 count at a read_noise of 9 + np.testing.assert_allclose(stacked_image.data, bias_pattern, atol=1.0) diff --git a/banzai/tests/test_bias_subtractor.py b/banzai/tests/test_bias_subtractor.py index b81fe5da..4196dc7e 100644 --- a/banzai/tests/test_bias_subtractor.py +++ b/banzai/tests/test_bias_subtractor.py @@ -69,10 +69,30 @@ def test_bias_subtraction_is_reasonable(mock_master_cal_name, mock_master_frame) input_level = 2000.0 nx = 101 ny = 103 - mock_master_frame.return_value = FakeLCOObservationFrame(hdu_list=[FakeCCDData(bias_level=input_bias, - data=np.random.normal(0.0, input_readnoise, size=(ny, nx)))]) + stacked_hdu = FakeCCDData(bias_level=input_bias, data=np.random.normal(0.0, input_readnoise, size=(ny, nx))) + mock_master_frame.return_value = FakeLCOObservationFrame(hdu_list=[stacked_hdu]) subtractor = BiasSubtractor(FakeContext()) image = FakeLCOObservationFrame(hdu_list=[FakeCCDData(image_multiplier=input_level)]) image = subtractor.do_stage(image) assert np.abs(image.meta.get('BIASLVL') - input_bias) < 1.0 assert np.abs(np.mean(image.data) - (input_level - input_bias)) < 1.0 + + +@mock.patch('banzai.lco.LCOFrameFactory.open') +@mock.patch('banzai.calibrations.CalibrationUser.get_calibration_file_info') +def test_multiread_bias_subtraction(mock_super_cal_name, mock_super_frame): + mock_super_cal_name.return_value = {'filename': 'test.fits'} + input_bias = 500.0 + nreads = 5 + nx = 101 + ny = 103 + bias_pattern = np.random.normal(0.0, 15.0, size=(ny, nx)) + stacked_hdu = FakeCCDData(bias_level=input_bias, data=bias_pattern) + mock_super_frame.return_value = FakeLCOObservationFrame(hdu_list=[stacked_hdu]) + data = bias_pattern * nreads + input_bias * nreads + image = FakeLCOObservationFrame(hdu_list=[FakeCCDData(bias_level=input_bias, data=data)]) + image.n_sub_exposures = nreads + subtractor = BiasSubtractor(FakeContext()) + image = subtractor.do_stage(image) + assert np.abs(image.meta.get('BIASLVL') - input_bias) < 1.0 + assert np.abs(np.mean(image.data)) < 1.0 diff --git a/banzai/tests/test_end_to_end.py b/banzai/tests/test_end_to_end.py index b879ca69..0beeddb7 100644 --- a/banzai/tests/test_end_to_end.py +++ b/banzai/tests/test_end_to_end.py @@ -159,6 +159,9 @@ def run_check_if_stacked_calibrations_are_in_db(raw_file_pattern, calibration_ty def observation_portal_side_effect(*args, **kwargs): + # To produce the mock observation portal response, we need to modify the response from + # this type of url + # https://observe.lco.global/api/observations/?enclosure=aqwa&telescope=0m4a&priority=&state=COMPLETED&time_span=&start_after=2024-07-23&start_before=&end_after=&end_before=2024-07-25&modified_after=&created_after=&created_before=&request_id=&request_group_id=&user=&proposal=calibrate&instrument_type=0M4-SCICAM-QHY600&configuration_type=SKY_FLAT&ordering= site = kwargs['params']['site'] start = datetime.strftime(parse(kwargs['params']['start_after']).replace(tzinfo=None).date(), '%Y%m%d') filename = 'test_obs_portal_response_{site}_{start}.json'.format(site=site, start=start) @@ -248,15 +251,18 @@ class TestScienceFileCreation: @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect) def reduce_science_frames(self, mock_observation_portal): run_reduce_individual_frames('e00.fits') + run_reduce_individual_frames('x00.fits') def test_if_science_frames_were_created(self): expected_files = [] created_files = [] for day_obs in DAYS_OBS: - expected_files += [filename.replace('e00', 'e91') - for filename in TEST_FRAMES['filename'] if 'e00.fits' in filename] - created_files += [os.path.basename(filename) for filename in glob(os.path.join(DATA_ROOT, day_obs, - 'processed', '*e91*'))] + for extension in ['e00', 'x00']: + expected_files += [filename.replace(extension, extension.replace('00', '91')) + for filename in TEST_FRAMES['filename'] if f'{extension}.fits' in filename] + created_files += [os.path.basename(filename) + for filename in glob(os.path.join(DATA_ROOT, day_obs, 'processed', + f'*{extension.replace("00", "91")}*'))] assert len(expected_files) > 0 for expected_file in expected_files: assert expected_file in created_files @@ -264,8 +270,9 @@ def test_if_science_frames_were_created(self): def test_that_photometric_calibration_succeeded(self): science_files = [] for day_obs in DAYS_OBS: - science_files += [filepath for filepath in glob(os.path.join(DATA_ROOT, day_obs, - 'processed', '*e91*'))] + for extension in ['e91', 'x91']: + science_files += [filepath for filepath in + glob(os.path.join(DATA_ROOT, day_obs, 'processed', f'*{extension}*'))] zeropoints = [fits.open(filename)['SCI'].header.get('L1ZP') for filename in science_files] # check that at least one of our images contains a zeropoint assert zeropoints.count(None) != len(zeropoints) diff --git a/banzai/tests/test_frames.py b/banzai/tests/test_frames.py index 0317958b..e66ae92f 100644 --- a/banzai/tests/test_frames.py +++ b/banzai/tests/test_frames.py @@ -140,7 +140,9 @@ def test_all_datatypes_wrong(): def test_subtract(): - test_data = FakeCCDData(image_multiplier=4, uncertainty=3) + nx = 101 + ny = 103 + test_data = FakeCCDData(image_multiplier=4, uncertainty=3*np.ones((ny, nx)), nx=nx, ny=ny) test_data -= 1 assert (test_data.data == 3 * np.ones(test_data.data.shape)).all() @@ -179,7 +181,9 @@ def test_trim(): def test_init_poisson_uncertainties(): # Make sure the uncertainties add in quadrature - test_data = FakeCCDData(image_multiplier=16, uncertainty=3) + nx = 101 + ny = 103 + test_data = FakeCCDData(image_multiplier=16, uncertainty=3 * np.ones((ny, nx)), nx=nx, ny=ny) test_data.init_poisson_uncertainties() assert (test_data.uncertainty == 5 * np.ones(test_data.data.shape)).all() diff --git a/banzai/tests/test_munge.py b/banzai/tests/test_munge.py index 5cc2a7b2..14fea328 100644 --- a/banzai/tests/test_munge.py +++ b/banzai/tests/test_munge.py @@ -55,7 +55,9 @@ def test_defaults_do_not_override_header(): def test_image_no_saturate_header_or_default(): with pytest.raises(MissingSaturate): - LCOFrameFactory._init_saturate(FakeLCOObservationFrame(hdu_list=[FakeCCDData(meta={'INSTRUME': 'blah27'})])) + fake_data = FakeCCDData(meta={'INSTRUME': 'blah27'}) + fake_data.meta.pop('SATURATE') + LCOFrameFactory._init_saturate(FakeLCOObservationFrame(hdu_list=[fake_data])) def test_saturate_no_default_but_header(): diff --git a/banzai/tests/utils.py b/banzai/tests/utils.py index a147ee97..a79f2fd3 100644 --- a/banzai/tests/utils.py +++ b/banzai/tests/utils.py @@ -39,11 +39,17 @@ def __init__(self, image_multiplier=1.0, nx=101, ny=103, name='test_image', read self.mask = np.zeros(self.data.shape, dtype=np.uint8) else: self.mask = mask + self.memmap = True if uncertainty is None: self.uncertainty = self.read_noise * np.ones(self.data.shape, dtype=self.data.dtype) else: self.uncertainty = uncertainty - + if 'SATURATE' not in self.meta: + self.meta['SATURATE'] = 65535.0 + if 'GAIN' not in self.meta: + self.meta['GAIN'] = 1.0 + if 'MAXLIN' not in self.meta: + self.meta['MAXLIN'] = 65535.0 for keyword in kwargs: setattr(self, keyword, kwargs[keyword]) @@ -63,7 +69,7 @@ def __init__(self, hdu_list=None, file_path='/tmp/test_image.fits', instrument=N self._file_path = file_path self.is_bad = False self.hdu_order = ['SCI', 'CAT', 'BPM', 'ERR'] - + self.n_sub_exposures = 1 for keyword in kwargs: setattr(self, keyword, kwargs[keyword]) @@ -159,6 +165,7 @@ def __init__(self, id=0, site='', camera='', enclosure='', telescope='', type='' self.nx = 4096 self.ny = 4096 + class FakeCalImage: def __init__(self): self.frameid = 1234 diff --git a/environment.yaml b/environment.yaml index 0ea4596f..d569e58a 100644 --- a/environment.yaml +++ b/environment.yaml @@ -7,282 +7,278 @@ channels: - astropy - defaults dependencies: - - _libgcc_mutex=0.1=conda_forge - _openmp_mutex=4.5=2_gnu - - affine=2.4.0=pyhd8ed1ab_0 - - amqp=2.6.1=pyh9f0ad1d_0 - - anyio=4.0.0=pyhd8ed1ab_0 - - aom=3.6.1=h59595ed_0 - - apscheduler=3.10.4=pyhd8ed1ab_0 - - asdf=2.15.1=pyhd8ed1ab_0 - - asdf-astropy=0.4.0=pyhd8ed1ab_1 - - asdf-coordinates-schemas=0.2.0=pyhd8ed1ab_1 - - asdf-standard=1.0.3=pyhd8ed1ab_0 - - asdf-transform-schemas=0.3.0=pyhd8ed1ab_0 - - asdf-unit-schemas=0.1.0=pyhd8ed1ab_0 - - asdf-wcs-schemas=0.1.1=pyhd8ed1ab_0 - - astropy=5.3.4=py310h1f7b6fc_2 - - asttokens=2.4.0=pyhd8ed1ab_0 - - attrs=23.1.0=pyh71513ae_1 - - backcall=0.2.0=pyh9f0ad1d_0 - - backports=1.0=pyhd8ed1ab_3 - - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 - - backports.zoneinfo=0.2.1=py310hff52083_8 - - billiard=3.6.4.0=py310h5764c6d_3 - - blosc=1.21.5=h0f2a231_0 - - bottleneck=1.3.7=py310h1f7b6fc_1 - - brotli=1.1.0=hd590300_1 - - brotli-bin=1.1.0=hd590300_1 - - brotli-python=1.1.0=py310hc6cd4ac_1 - - brunsli=0.1=h9c3ff4c_0 - - bzip2=1.0.8=h7f98852_4 - - c-ares=1.20.1=hd590300_0 - - c-blosc2=2.10.5=hb4ffafa_0 - - ca-certificates=2023.7.22=hbcca054_0 - - cairo=1.18.0=h3faef2a_0 - - celery=4.4.0=py_0 - - certifi=2023.7.22=pyhd8ed1ab_0 - - cffi=1.16.0=py310h2fee648_0 - - cfitsio=4.3.0=hbdc6101_0 - - charls=2.4.2=h59595ed_0 - - charset-normalizer=3.3.0=pyhd8ed1ab_0 - - click=8.1.7=unix_pyh707e725_0 - - click-plugins=1.1.1=py_0 - - cligj=0.7.2=pyhd8ed1ab_1 - - colorama=0.4.6=pyhd8ed1ab_0 - - contourpy=1.1.1=py310hd41b1e2_1 - - coverage=7.3.2=py310h2372a71_0 - - cryptography=41.0.4=py310h75e40e8_0 - - cycler=0.12.1=pyhd8ed1ab_0 - - cython=0.29.36=py310hc6cd4ac_1 - - dav1d=1.2.1=hd590300_0 - - decorator=5.1.1=pyhd8ed1ab_0 - - dnspython=2.4.2=pyhd8ed1ab_0 - - elasticsearch=5.5.3=pyh9f0ad1d_0 - - emcee=3.1.4=pyhd8ed1ab_0 - - exceptiongroup=1.1.3=pyhd8ed1ab_0 - - executing=1.2.0=pyhd8ed1ab_0 - - expat=2.5.0=hcb278e6_1 - - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 - - font-ttf-inconsolata=3.000=h77eed37_0 - - font-ttf-source-code-pro=2.038=h77eed37_0 - - font-ttf-ubuntu=0.83=hab24e00_0 - - fontconfig=2.14.2=h14ed4e7_0 - - fonts-conda-ecosystem=1=0 - - fonts-conda-forge=1=0 - - fonttools=4.43.1=py310h2372a71_0 - - freetype=2.12.1=h267a509_2 - - freexl=2.0.0=h743c826_0 - - geos=3.12.0=h59595ed_0 - - geotiff=1.7.1=hf074850_14 - - gettext=0.21.1=h27087fc_0 - - giflib=5.2.1=h0b41bf4_3 - - greenlet=3.0.0=py310hc6cd4ac_1 - - gwcs=0.19.0=pyhd8ed1ab_0 - - h11=0.14.0=pyhd8ed1ab_0 - - h2=4.1.0=pyhd8ed1ab_0 - - hdf4=4.2.15=h2a13503_7 - - hdf5=1.14.2=nompi_h4f84152_100 - - hpack=4.0.0=pyh9f0ad1d_0 - - httpcore=1.0.0=pyhd8ed1ab_0 - - hyperframe=6.0.1=pyhd8ed1ab_0 - - hypothesis=6.87.4=pyha770c72_0 - - icu=73.2=h59595ed_0 - - idna=3.4=pyhd8ed1ab_0 - - imagecodecs=2023.9.18=py310h496a806_2 - - imageio=2.31.5=pyh8c1a49c_0 - - importlib-metadata=6.8.0=pyha770c72_0 - - importlib-resources=6.1.0=pyhd8ed1ab_0 - - importlib_resources=6.1.0=pyhd8ed1ab_0 - - iniconfig=2.0.0=pyhd8ed1ab_0 - - intel-openmp=2022.1.0=h9e868ea_3769 - - ipython=8.16.1=pyh0d859eb_0 - - jedi=0.19.1=pyhd8ed1ab_0 - - jmespath=1.0.1=pyhd8ed1ab_0 - - joblib=1.3.2=pyhd8ed1ab_0 - - json-c=0.17=h7ab15ed_0 - - jsonschema=4.19.1=pyhd8ed1ab_0 - - jsonschema-specifications=2023.7.1=pyhd8ed1ab_0 - - jxrlib=1.1=h7f98852_2 - - kealib=1.5.2=hcd42e92_1 - - keyutils=1.6.1=h166bdaf_0 - - kiwisolver=1.4.5=py310hd41b1e2_1 - - krb5=1.21.2=h659d440_0 - - lazy_loader=0.3=pyhd8ed1ab_0 - - lcms2=2.15=hb7c19ff_3 - - ld_impl_linux-64=2.40=h41732ed_0 - - lerc=4.0.0=h27087fc_0 - - libaec=1.1.2=h59595ed_1 - - libarchive=3.7.2=h039dbb9_0 - - libavif16=1.0.1=h87da1f6_2 - - libblas=3.9.0=18_linux64_openblas - - libboost-headers=1.82.0=ha770c72_6 - - libbrotlicommon=1.1.0=hd590300_1 - - libbrotlidec=1.1.0=hd590300_1 - - libbrotlienc=1.1.0=hd590300_1 - - libcblas=3.9.0=18_linux64_openblas - - libcurl=8.4.0=hca28451_0 - - libdeflate=1.19=hd590300_0 - - libedit=3.1.20191231=he28a2e2_2 - - libev=4.33=h516909a_1 - - libexpat=2.5.0=hcb278e6_1 - - libffi=3.4.2=h7f98852_5 - - libgcc-ng=13.2.0=h807b86a_2 - - libgdal=3.7.2=h6f3d308_7 - - libgfortran-ng=13.2.0=h69a702a_2 - - libgfortran5=13.2.0=ha4646dd_2 - - libglib=2.78.0=hebfc3b9_0 - - libgomp=13.2.0=h807b86a_2 - - libiconv=1.17=h166bdaf_0 - - libjpeg-turbo=3.0.0=hd590300_1 - - libkml=1.3.0=h01aab08_1018 - - liblapack=3.9.0=18_linux64_openblas - - libnetcdf=4.9.2=nompi_h80fb2b6_112 - - libnghttp2=1.52.0=h61bc06f_0 - - libnsl=2.0.0=hd590300_1 - - libopenblas=0.3.24=pthreads_h413a1c8_0 - - libpng=1.6.39=h753d276_0 - - libpq=16.0=hfc447b1_1 - - libprotobuf=3.20.3=h3eb15da_0 - - librttopo=1.1.0=hb58d41b_14 - - libspatialite=5.1.0=h090f1da_0 - - libsqlite=3.43.2=h2797004_0 - - libssh2=1.11.0=h0841786_0 - - libstdcxx-ng=13.2.0=h7e041cc_2 - - libtiff=4.6.0=ha9c0a0a_2 - - libuuid=2.38.1=h0b41bf4_0 - - libwebp-base=1.3.2=hd590300_0 - - libxcb=1.15=h0b41bf4_0 - - libxml2=2.11.5=h232c23b_1 - - libzip=1.10.1=h2629f0a_3 - - libzlib=1.2.13=hd590300_5 - - libzopfli=1.0.3=h9c3ff4c_0 - - lz4-c=1.9.4=hcb278e6_0 - - lzo=2.10=h516909a_1000 - - matplotlib-base=3.8.0=py310h62c0568_2 - - matplotlib-inline=0.1.6=pyhd8ed1ab_0 - - minizip=4.0.1=h0ab5242_5 - - mkl=2022.1.0=hc2b9512_224 - - mock=5.1.0=pyhd8ed1ab_0 - - munkres=1.1.4=pyh9f0ad1d_0 - - mysql-common=8.0.33=hf1915f5_5 - - mysql-connector-python=8.0.32=py310h6eefaca_0 - - mysql-libs=8.0.33=hca2cd23_5 - - ncurses=6.4=hcb278e6_0 - - networkx=3.1=pyhd8ed1ab_0 - - ninja=1.11.1=h924138e_0 - - nspr=4.35=h27087fc_0 - - nss=3.94=h1d7d5a4_0 - - numpy=1.23.5=py310h53a5b5f_0 - - openjpeg=2.5.0=h488ebb8_3 - - openssl=3.1.3=hd590300_0 - - packaging=23.2=pyhd8ed1ab_0 - - parso=0.8.3=pyhd8ed1ab_0 - - pcre2=10.40=hc3806b6_0 - - pexpect=4.8.0=pyh1a96a4e_2 - - photutils=1.9.0=py310h2372a71_0 - - pickleshare=0.7.5=py_1003 - - pillow=10.0.1=py310h01dd4db_2 - - pip=23.2.1=pyhd8ed1ab_0 - - pixman=0.42.2=h59595ed_0 - - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 - - pluggy=1.3.0=pyhd8ed1ab_0 - - poppler=23.10.0=h590f24d_0 - - poppler-data=0.4.12=hd8ed1ab_0 - - postgresql=16.0=h8972f4a_1 - - proj=9.3.0=h1d62c97_1 - - prompt-toolkit=3.0.39=pyha770c72_0 - - prompt_toolkit=3.0.39=hd8ed1ab_0 - - protobuf=3.20.3=py310heca2aa9_1 - - psutil=5.9.5=py310h2372a71_1 - - psycopg2=2.9.7=py310h275853b_1 - - psycopg2-binary=2.9.7=pyhd8ed1ab_1 - - pthread-stubs=0.4=h36c2ea0_1001 - - ptyprocess=0.7.0=pyhd3deb0d_0 - - pure_eval=0.2.2=pyhd8ed1ab_0 - - pycparser=2.21=pyhd8ed1ab_0 - - pyerfa=2.0.0.3=py310h1f7b6fc_1 - - pygments=2.16.1=pyhd8ed1ab_0 - - pyparsing=3.1.1=pyhd8ed1ab_0 - - pysocks=1.7.1=pyha2e5f31_6 - - pytest=7.4.2=pyhd8ed1ab_0 - - pytest-arraydiff=0.5.0=pyhd8ed1ab_0 - - pytest-astropy=0.10.0=pyhd8ed1ab_0 - - pytest-astropy-header=0.2.2=pyhd8ed1ab_0 - - pytest-cov=4.1.0=pyhd8ed1ab_0 - - pytest-doctestplus=1.0.0=pyhd8ed1ab_0 - - pytest-filter-subpackage=0.1.2=pyhd8ed1ab_0 - - pytest-mock=3.11.1=pyhd8ed1ab_0 - - pytest-openfiles=0.5.0=py_0 - - pytest-remotedata=0.4.1=pyhd8ed1ab_0 - - python=3.10.12=hd12c33a_0_cpython - - python-dateutil=2.8.2=pyhd8ed1ab_0 - - python_abi=3.10=4_cp310 - - pytorch=1.12.1=cpu_py310h75c9ab6_0 - - pytz=2023.3.post1=pyhd8ed1ab_0 - - pywavelets=1.4.1=py310h1f7b6fc_1 - - pyyaml=6.0.1=py310h2372a71_1 - - rasterio=1.3.8=py310h6a913dc_4 - - rav1e=0.6.6=he8a937b_2 - - readline=8.2=h8228510_1 - - referencing=0.30.2=pyhd8ed1ab_0 - - requests=2.31.0=pyhd8ed1ab_0 - - rpds-py=0.10.6=py310hcb5633a_0 - - scikit-image=0.22.0=py310hcc13569_2 - - scikit-learn=1.3.1=py310h1fdf081_1 - - scipy=1.11.3=py310hb13e2d6_1 - - semantic_version=2.10.0=pyhd8ed1ab_0 - - sep=1.2.1=py310h1f7b6fc_2 - - setuptools=68.2.2=pyhd8ed1ab_0 - - shapely=2.0.2=py310h7dcad9a_0 - - simplejson=3.19.2=py310h2372a71_0 - - six=1.16.0=pyh6c4a22f_0 - - sleef=3.5.1=h9b69904_2 - - snappy=1.1.10=h9fff704_0 - - sniffio=1.3.0=pyhd8ed1ab_0 - - snuggs=1.4.7=py_0 - - sortedcontainers=2.4.0=pyhd8ed1ab_0 - - sqlalchemy=2.0.22=py310h2372a71_0 - - sqlite=3.43.2=h2c6b66d_0 - - stack_data=0.6.2=pyhd8ed1ab_0 - - svt-av1=1.7.0=h59595ed_0 - - tenacity=8.2.3=pyhd8ed1ab_0 - - threadpoolctl=3.2.0=pyha21a80b_0 - - tifffile=2023.9.26=pyhd8ed1ab_0 - - tiledb=2.16.3=hf0b6e87_3 - - tk=8.6.13=h2797004_0 - - toml=0.10.2=pyhd8ed1ab_0 - - tomli=2.0.1=pyhd8ed1ab_0 - - tqdm=4.66.1=pyhd8ed1ab_0 - - traitlets=5.11.2=pyhd8ed1ab_0 - - typing-extensions=4.8.0=hd8ed1ab_0 - - typing_extensions=4.8.0=pyha770c72_0 - - tzcode=2023c=h0b41bf4_0 - - tzdata=2023c=h71feb2d_0 - - tzlocal=5.1=py310hff52083_0 - - unicodedata2=15.1.0=py310h2372a71_0 - - uriparser=0.9.7=hcb278e6_1 - - wcwidth=0.2.8=pyhd8ed1ab_0 - - wheel=0.41.2=pyhd8ed1ab_0 - - xerces-c=3.2.4=hac6953d_3 - - xorg-kbproto=1.0.7=h7f98852_1002 - - xorg-libice=1.1.1=hd590300_0 - - xorg-libsm=1.2.4=h7391055_0 - - xorg-libx11=1.8.7=h8ee46fc_0 - - xorg-libxau=1.0.11=hd590300_0 - - xorg-libxdmcp=1.1.3=h7f98852_0 - - xorg-libxext=1.3.4=h0b41bf4_2 - - xorg-libxrender=0.9.11=hd590300_0 - - xorg-renderproto=0.11.1=h7f98852_1002 - - xorg-xextproto=7.3.0=h0b41bf4_1003 - - xorg-xproto=7.0.31=h7f98852_1007 - - xz=5.2.6=h166bdaf_0 - - yaml=0.2.5=h7f98852_2 - - zfp=1.0.0=h59595ed_4 - - zipp=3.17.0=pyhd8ed1ab_0 - - zlib=1.2.13=hd590300_5 - - zlib-ng=2.0.7=h0b41bf4_0 - - zstd=1.5.5=hfc55251_0 + - affine=2.4.0 + - amqp=2.6.1 + - anyio=4.0.0 + - aom=3.6.1 + - apscheduler=3.10.4 + - asdf=2.15.1 + - asdf-astropy=0.4.0 + - asdf-coordinates-schemas=0.2.0 + - asdf-standard=1.0.3 + - asdf-transform-schemas=0.3.0 + - asdf-unit-schemas=0.1.0 + - asdf-wcs-schemas=0.1.1 + - astropy=5.3.4 + - asttokens=2.4.0 + - attrs=23.1.0 + - backcall=0.2.0 + - backports=1.0 + - backports.functools_lru_cache=1.6.5 + - backports.zoneinfo=0.2.1 + - billiard=3.6.4.0 + - blosc=1.21.5 + - bottleneck=1.3.7 + - brotli=1.1.0 + - brotli-bin=1.1.0 + - brotli-python=1.1.0 + - brunsli=0.1 + - bzip2=1.0.8 + - c-ares=1.20.1 + - c-blosc2=2.10.5 + - ca-certificates=2023.7.22 + - cairo=1.18.0 + - celery=4.4.0 + - certifi=2023.7.22 + - cffi=1.16.0 + - cfitsio=4.3.0 + - charls=2.4.2 + - charset-normalizer=3.3.0 + - click=8.1.7 + - click-plugins=1.1.1 + - cligj=0.7.2 + - colorama=0.4.6 + - contourpy=1.1.1 + - coverage=7.3.2 + - cryptography=41.0.4 + - cycler=0.12.1 + - cython=0.29.36 + - dav1d=1.2.1 + - decorator=5.1.1 + - dnspython=2.4.2 + - elasticsearch=5.5.3 + - emcee=3.1.4 + - exceptiongroup=1.1.3 + - executing=1.2.0 + - expat=2.5.0 + - font-ttf-dejavu-sans-mono=2.37 + - font-ttf-inconsolata=3.000 + - font-ttf-source-code-pro=2.038 + - font-ttf-ubuntu=0.83 + - fontconfig=2.14.2 + - fonts-conda-ecosystem=1 + - fonts-conda-forge=1 + - fonttools=4.43.1 + - freetype=2.12.1 + - freexl=2.0.0 + - geos=3.12.0 + - geotiff=1.7.1 + - gettext=0.21.1 + - giflib=5.2.1 + - greenlet=3.0.0 + - gwcs=0.19.0 + - h11=0.14.0 + - h2=4.1.0 + - hdf4=4.2.15 + - hdf5=1.14.2 + - hpack=4.0.0 + - httpcore=1.0.0 + - hyperframe=6.0.1 + - hypothesis=6.87.4 + - icu=73.2 + - idna=3.4 + - imagecodecs=2023.9.18 + - imageio=2.31.5 + - importlib-metadata=6.8.0 + - importlib-resources=6.1.0 + - importlib_resources=6.1.0 + - iniconfig=2.0.0 + - ipython=8.16.1 + - jedi=0.19.1 + - jmespath=1.0.1 + - joblib=1.3.2 + - json-c=0.17 + - jsonschema=4.19.1 + - jsonschema-specifications=2023.7.1 + - jxrlib=1.1 + - kealib=1.5.2 + - keyutils=1.6.1 + - kiwisolver=1.4.5 + - krb5=1.21.2 + - lazy_loader=0.3 + - lcms2=2.15 + - ld_impl_linux-64=2.40 + - lerc=4.0.0 + - libaec=1.1.2 + - libarchive=3.7.2 + - libavif16=1.0.1 + - libblas=3.9.0 + - libboost-headers=1.82.0 + - libbrotlicommon=1.1.0 + - libbrotlidec=1.1.0 + - libbrotlienc=1.1.0 + - libcblas=3.9.0 + - libcurl=8.4.0 + - libdeflate=1.19 + - libedit=3.1.20191231 + - libev=4.33 + - libexpat=2.5.0 + - libffi=3.4.2 + - libgcc-ng=13.2.0 + - libgdal=3.7.2 + - libgfortran-ng=13.2.0 + - libgfortran5=13.2.0 + - libglib=2.78.0 + - libgomp=13.2.0 + - libiconv=1.17 + - libjpeg-turbo=3.0.0 + - libkml=1.3.0 + - liblapack=3.9.0 + - libnetcdf=4.9.2 + - libnghttp2=1.52.0 + - libnsl=2.0.0 + - libopenblas=0.3.24 + - libpng=1.6.39 + - libpq=16.0 + - libprotobuf=3.20.3 + - librttopo=1.1.0 + - libspatialite=5.1.0 + - libsqlite=3.43.2 + - libssh2=1.11.0 + - libstdcxx-ng=13.2.0 + - libtiff=4.6.0 + - libuuid=2.38.1 + - libwebp-base=1.3.2 + - libxcb=1.15 + - libxml2=2.11.5 + - libzip=1.10.1 + - libzlib=1.2.13 + - libzopfli=1.0.3 + - lz4-c=1.9.4 + - lzo=2.10 + - matplotlib-base=3.8.0 + - matplotlib-inline=0.1.6 + - minizip=4.0.1 + - mock=5.1.0 + - munkres=1.1.4 + - mysql-common=8.0.33 + - mysql-connector-python=8.0.32 + - mysql-libs=8.0.33 + - ncurses=6.4 + - networkx=3.1 + - ninja=1.11.1 + - nspr=4.35 + - nss=3.94 + - numpy=1.23.5 + - openjpeg=2.5.0 + - openssl=3.1.3 + - packaging=23.2 + - parso=0.8.3 + - pcre2=10.40 + - pexpect=4.8.0 + - photutils=1.9.0 + - pickleshare=0.7.5 + - pillow=10.0.1 + - pip=23.2.1 + - pixman=0.42.2 + - pkgutil-resolve-name=1.3.10 + - pluggy=1.5.0 + - poppler=23.10.0 + - poppler-data=0.4.12 + - postgresql=16.0 + - proj=9.3.0 + - prompt-toolkit=3.0.39 + - prompt_toolkit=3.0.39 + - protobuf=3.20.3 + - psutil=5.9.5 + - psycopg2=2.9.7 + - psycopg2-binary=2.9.7 + - pthread-stubs=0.4 + - ptyprocess=0.7.0 + - pure_eval=0.2.2 + - pycparser=2.21 + - pyerfa=2.0.0.3 + - pygments=2.16.1 + - pyparsing=3.1.1 + - pysocks=1.7.1 + - pytest=8.0.0 + - pytest-arraydiff=0.5.0 + - pytest-astropy=0.10.0 + - pytest-astropy-header=0.2.2 + - pytest-cov=4.1.0 + - pytest-doctestplus=1.0.0 + - pytest-filter-subpackage=0.1.2 + - pytest-mock=3.11.1 + - pytest-openfiles=0.5.0 + - pytest-remotedata=0.4.1 + - python=3.10.12 + - python-dateutil=2.8.2 + - python_abi=3.10 + - pytorch=1.12.1 + - pytz=2023.3.post1 + - pywavelets=1.4.1 + - pyyaml=6.0.1 + - rasterio=1.3.8 + - rav1e=0.6.6 + - readline=8.2 + - referencing=0.30.2 + - requests=2.31.0 + - rpds-py=0.10.6 + - scikit-image=0.22.0 + - scikit-learn=1.3.1 + - scipy=1.11.3 + - semantic_version=2.10.0 + - setuptools=68.2.2 + - shapely=2.0.2 + - simplejson=3.19.2 + - six=1.16.0 + - sleef=3.5.1 + - snappy=1.1.10 + - sniffio=1.3.0 + - snuggs=1.4.7 + - sortedcontainers=2.4.0 + - sqlalchemy=2.0.22 + - sqlite=3.43.2 + - stack_data=0.6.2 + - svt-av1=1.7.0 + - tenacity=8.2.3 + - threadpoolctl=3.2.0 + - tifffile=2023.9.26 + - tiledb=2.16.3 + - tk=8.6.13 + - toml=0.10.2 + - tomli=2.0.1 + - tqdm=4.66.1 + - traitlets=5.11.2 + - typing-extensions=4.8.0 + - typing_extensions=4.8.0 + - tzcode=2023c + - tzdata=2023c + - tzlocal=5.1 + - unicodedata2=15.1.0 + - uriparser=0.9.7 + - wcwidth=0.2.8 + - wheel=0.41.2 + - xerces-c=3.2.4 + - xorg-kbproto=1.0.7 + - xorg-libice=1.1.1 + - xorg-libsm=1.2.4 + - xorg-libx11=1.8.7 + - xorg-libxau=1.0.11 + - xorg-libxdmcp=1.1.3 + - xorg-libxext=1.3.4 + - xorg-libxrender=0.9.11 + - xorg-renderproto=0.11.1 + - xorg-xextproto=7.3.0 + - xorg-xproto=7.0.31 + - xz=5.2.6 + - yaml=0.2.5 + - zfp=1.0.0 + - zipp=3.17.0 + - zlib=1.2.13 + - zlib-ng=2.0.7 + - zstd=1.5.5 - pip: - asciitree==0.3.3 - astropy-healpix==1.0.0 @@ -292,6 +288,7 @@ dependencies: - cloudpickle==2.2.1 - cosmic-conn==0.4.1 - dask==2023.9.3 + - extension_helpers==1.1.1 - fasteners==0.19 - fsspec==2023.9.2 - kombu==4.6.11 @@ -309,6 +306,7 @@ dependencies: - redis==5.0.1 - reproject==0.12.0 - s3transfer==0.7.0 + - sep==1.2.1 - toolz==0.12.0 - urllib3==1.26.17 - vine==1.3.0 diff --git a/pytest.ini b/pytest.ini index b30fc1bd..e68b30c2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -26,6 +26,7 @@ markers = dark_comparer dark_maker dark_normalizer + dark_subtractor date_utils dbs fits_utils @@ -48,6 +49,7 @@ markers = pattern_noise_qc pointing quick_select + read_noise runtime_context saturation_qc saving_qc diff --git a/setup.cfg b/setup.cfg index 8e7c6b29..28c222b0 100755 --- a/setup.cfg +++ b/setup.cfg @@ -61,7 +61,7 @@ install_requires = pytest>=4.0 pyyaml psycopg2-binary - celery[redis]>=4.3.1,<5 + celery[redis]>5 apscheduler python-dateutil ocs_ingester>=3.0.4,<4.0.0