From 4489497a7b51f7abd36ddd311481bdaa584e8bba Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 24 Jul 2024 10:27:10 -0400 Subject: [PATCH 01/14] Added support for stacked images. --- .github/workflows/e2e.yaml | 14 ++++++++++- banzai/bias.py | 6 ++--- banzai/data.py | 6 ++++- banzai/lco.py | 2 +- banzai/readnoise.py | 3 ++- banzai/settings.py | 4 +-- banzai/tests/data/test_data.dat | 43 +++++++++++++-------------------- 7 files changed, 43 insertions(+), 35 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d43efe84..3fc9086c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -43,10 +43,22 @@ jobs: # Wait for banzai to be ready kubectl wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test - - name: Test Master Bias Creation + - name: Test Super Bias Creation run: | kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias + - name: Test Super Dark Creation + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark + + - name: Test Super Flat Creation + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat + + - name: Test Science Frame Creation + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files + - name: Cleanup run: | kubectl delete pod banzai-e2e-test diff --git a/banzai/bias.py b/banzai/bias.py index 19429727..316f166f 100755 --- a/banzai/bias.py +++ b/banzai/bias.py @@ -31,9 +31,9 @@ def calibration_type(self): return 'bias' def apply_master_calibration(self, image, master_calibration_image): - image -= master_calibration_image.bias_level + image -= master_calibration_image.bias_level * image.n_sub_exposures image.meta['BIASLVL'] = master_calibration_image.bias_level, 'Bias level that was removed after overscan' - image -= master_calibration_image + image -= master_calibration_image * image.n_sub_exposures image.meta['L1IDBIAS'] = master_calibration_image.filename, 'ID of bias frame' image.meta['L1STATBI'] = 1, "Status flag for bias frame correction" return image @@ -65,7 +65,7 @@ def __init__(self, runtime_context): def do_stage(self, image): bias_level = stats.sigma_clipped_mean(image.data, 3.5, mask=image.mask) image -= bias_level - image.meta['BIASLVL'] = bias_level, 'Bias level that was removed after overscan' + image.meta['BIASLVL'] = bias_level / image.n_sub_exposures, 'Bias level that was removed after overscan' return image diff --git a/banzai/data.py b/banzai/data.py index 4adf7322..2cda4f7d 100644 --- a/banzai/data.py +++ b/banzai/data.py @@ -132,7 +132,7 @@ def __init__(self, data: Union[np.array, Table], meta: fits.Header, mask: np.array = None, name: str = '', uncertainty: np.array = None, memmap=True): super().__init__(data=data, meta=meta, mask=mask, name=name, memmap=memmap) if uncertainty is None: - uncertainty = self.read_noise * np.ones(data.shape, dtype=data.dtype) / self.gain + uncertainty = self.read_noise * self.n_sub_exposures * np.ones(data.shape, dtype=data.dtype) / self.gain self.uncertainty = self._init_array(uncertainty) self._detector_section = Section.parse_region_keyword(self.meta.get('DETSEC')) self._data_section = Section.parse_region_keyword(self.meta.get('DATASEC')) @@ -302,6 +302,10 @@ def data_section(self, section): self.meta['DATASEC'] = section.to_region_keyword() self._data_section = section + @property + def n_sub_exposures(self): + return self.meta.get('NSUBREAD', 1) + def rebin(self, binning): # TODO: Implement me return self diff --git a/banzai/lco.py b/banzai/lco.py index 6f801318..67447d57 100644 --- a/banzai/lco.py +++ b/banzai/lco.py @@ -450,7 +450,7 @@ def open(self, file_info, runtime_context) -> Optional[ObservationFrame]: # update datasec/trimsec for fs01 if hdu.header.get('INSTRUME') == 'fs01': self._update_fs01_sections(hdu) - if hdu.data.dtype == np.uint16: + if hdu.data.dtype == np.uint16 or hdu.data.dtype == np.uint32: hdu.data = hdu.data.astype(np.float64) # check if we need to propagate any header keywords from the primary header if primary_hdu is not None: diff --git a/banzai/readnoise.py b/banzai/readnoise.py index ba11ab69..08950775 100644 --- a/banzai/readnoise.py +++ b/banzai/readnoise.py @@ -1,5 +1,6 @@ from banzai.calibrations import CalibrationUser from banzai.logs import format_exception, get_logger +import numpy as np logger = get_logger() @@ -8,7 +9,7 @@ class ReadNoiseLoader(CalibrationUser): def apply_master_calibration(self, image, master_calibration_image): try: for image_extension, readnoise_extension in zip(image.ccd_hdus, master_calibration_image.ccd_hdus): - image_extension.add_uncertainty(readnoise_extension.data) + image_extension.add_uncertainty(readnoise_extension.data * np.sqrt(image_extension.n_sub_exposures)) except: logger.error(f"Can't add READNOISE to image, stopping reduction: {format_exception()}", image=image) return None diff --git a/banzai/settings.py b/banzai/settings.py index fdfb98b5..9c511253 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -54,8 +54,8 @@ EXTRA_STAGES = {'BIAS': ['banzai.bias.BiasMasterLevelSubtractor', 'banzai.bias.BiasComparer'], 'DARK': ['banzai.dark.DarkNormalizer', 'banzai.dark.DarkTemperatureChecker', 'banzai.dark.DarkComparer'], - 'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer', 'banzai.qc.PatternNoiseDetector', - 'banzai.flats.FlatComparer'], + 'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer', + 'banzai.qc.PatternNoiseDetector', 'banzai.flats.FlatComparer'], 'STANDARD': None, 'EXPOSE': None, 'EXPERIMENTAL': None} diff --git a/banzai/tests/data/test_data.dat b/banzai/tests/data/test_data.dat index f7bc38f7..91a6b880 100644 --- a/banzai/tests/data/test_data.dat +++ b/banzai/tests/data/test_data.dat @@ -304,29 +304,20 @@ frameid filename filter site instrument 33347343 tst1m0XX-ep04-20200324-0018-d00.fits.fz ip*diffuser tst ep04 33347344 tst1m0XX-ep04-20200324-0019-d00.fits.fz ip*diffuser tst ep04 33347345 tst1m0XX-ep04-20200324-0020-d00.fits.fz ip*diffuser tst ep04 -56016937 ogg0m404-sq30-20221123-0037-b00.fits.fz opaque ogg sq30 -56016931 ogg0m404-sq30-20221123-0036-b00.fits.fz opaque ogg sq30 -56016928 ogg0m404-sq30-20221123-0035-b00.fits.fz opaque ogg sq30 -56016925 ogg0m404-sq30-20221123-0034-b00.fits.fz opaque ogg sq30 -56016920 ogg0m404-sq30-20221123-0033-b00.fits.fz opaque ogg sq30 -56016916 ogg0m404-sq30-20221123-0032-b00.fits.fz opaque ogg sq30 -56016912 ogg0m404-sq30-20221123-0031-b00.fits.fz opaque ogg sq30 -56016909 ogg0m404-sq30-20221123-0030-b00.fits.fz opaque ogg sq30 -56016906 ogg0m404-sq30-20221123-0029-b00.fits.fz opaque ogg sq30 -56016903 ogg0m404-sq30-20221123-0028-b00.fits.fz opaque ogg sq30 -56016899 ogg0m404-sq30-20221123-0027-b00.fits.fz opaque ogg sq30 -56016898 ogg0m404-sq30-20221123-0026-b00.fits.fz opaque ogg sq30 -56016896 ogg0m404-sq30-20221123-0025-b00.fits.fz opaque ogg sq30 -56016891 ogg0m404-sq30-20221123-0024-b00.fits.fz opaque ogg sq30 -56016889 ogg0m404-sq30-20221123-0023-b00.fits.fz opaque ogg sq30 -56017384 ogg0m404-sq30-20221123-0042-d00.fits.fz opaque ogg sq30 -56017294 ogg0m404-sq30-20221123-0041-d00.fits.fz opaque ogg sq30 -56017205 ogg0m404-sq30-20221123-0040-d00.fits.fz opaque ogg sq30 -56017136 ogg0m404-sq30-20221123-0039-d00.fits.fz opaque ogg sq30 -56017044 ogg0m404-sq30-20221123-0038-d00.fits.fz opaque ogg sq30 -56018344 ogg0m404-sq30-20221123-0067-f00.fits.fz rp ogg sq30 -56018340 ogg0m404-sq30-20221123-0066-f00.fits.fz rp ogg sq30 -56018336 ogg0m404-sq30-20221123-0065-f00.fits.fz rp ogg sq30 -56018331 ogg0m404-sq30-20221123-0064-f00.fits.fz rp ogg sq30 -56018328 ogg0m404-sq30-20221123-0063-f00.fits.fz rp ogg sq30 -56127852 ogg0m404-sq30-20221126-0098-e00.fits.fz rp ogg sq30 +73414350 lsc0m476-sq34-20240723-0301-e00.fits.fz B lsc sq34 +73422225 lsc0m476-sq34-20240723-0408-x00.fits.fz B lsc sq34 +73409803 lsc0m476-sq34-20240723-0057-f00.fits.fz B lsc sq34 +73409794 lsc0m476-sq34-20240723-0056-f00.fits.fz B lsc sq34 +73409788 lsc0m476-sq34-20240723-0055-f00.fits.fz B lsc sq34 +73409785 lsc0m476-sq34-20240723-0054-f00.fits.fz B lsc sq34 +73215904 lsc0m476-sq34-20240723-0053-f00.fits.fz B lsc sq34 +73429574 lsc0m476-sq34-20240723-0548-d00.fits.fz opaque lsc sq34 +73429727 lsc0m476-sq34-20240723-0549-d00.fits.fz opaque lsc sq34 +73429774 lsc0m476-sq34-20240723-0550-d00.fits.fz opaque lsc sq34 +73429865 lsc0m476-sq34-20240723-0551-d00.fits.fz opaque lsc sq34 +73429912 lsc0m476-sq34-20240723-0552-d00.fits.fz opaque lsc sq34 +73429436 lsc0m476-sq34-20240723-0535-b00.fits.fz opaque lsc sq34 +73429432 lsc0m476-sq34-20240723-0536-b00.fits.fz opaque lsc sq34 +73429437 lsc0m476-sq34-20240723-0537-b00.fits.fz opaque lsc sq34 +73429439 lsc0m476-sq34-20240723-0538-b00.fits.fz opaque lsc sq34 +73429443 lsc0m476-sq34-20240723-0539-b00.fits.fz opaque lsc sq34 From 00dd08a68be71f5aee09a6253baa4c892cf6b689 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 24 Jul 2024 11:05:17 -0400 Subject: [PATCH 02/14] Fixes to tests. --- banzai/data.py | 9 +++++++++ banzai/frames.py | 11 ++++++++++- banzai/tests/test_munge.py | 4 +++- banzai/tests/utils.py | 11 +++++++++-- setup.cfg | 2 +- 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/banzai/data.py b/banzai/data.py index 2cda4f7d..eb43c6c5 100644 --- a/banzai/data.py +++ b/banzai/data.py @@ -155,6 +155,15 @@ def __imul__(self, value): self.meta['MAXLIN'] *= value return self + def __mul__(self, value): + output = CCDData(self.data * value, meta=self.meta.copy(), + name=self.name, uncertainty=self.uncertainty * value, + mask=self.mask.copy(), memmap=self.memmap) + output.meta['SATURATE'] *= value + output.meta['GAIN'] /= value + output.meta['MAXLIN'] *= value + return output + def __itruediv__(self, value): if isinstance(value, CCDData): self.uncertainty = np.abs(self.data / value.data) * \ diff --git a/banzai/frames.py b/banzai/frames.py index 093a5734..ea4bbfbb 100644 --- a/banzai/frames.py +++ b/banzai/frames.py @@ -195,7 +195,10 @@ def binning(self): return [min(x_binnings), min(y_binnings)] def __sub__(self, other): - return self.primary_hdu - other.primary_hdu + if isinstance(other, ObservationFrame): + return self.primary_hdu - other.primary_hdu + else: + return self.primary_hdu - other def __isub__(self, other): if isinstance(other, ObservationFrame): @@ -204,6 +207,12 @@ def __isub__(self, other): self.primary_hdu.__isub__(other) return self + def __mul__(self, other): + if isinstance(other, ObservationFrame): + return self.primary_hdu.__mul__(other.primary_hdu) + else: + return self.primary_hdu.__mul__(other) + def __imul__(self, other): if isinstance(other, ObservationFrame): self.primary_hdu.__imul__(other.primary_hdu) diff --git a/banzai/tests/test_munge.py b/banzai/tests/test_munge.py index 5cc2a7b2..14fea328 100644 --- a/banzai/tests/test_munge.py +++ b/banzai/tests/test_munge.py @@ -55,7 +55,9 @@ def test_defaults_do_not_override_header(): def test_image_no_saturate_header_or_default(): with pytest.raises(MissingSaturate): - LCOFrameFactory._init_saturate(FakeLCOObservationFrame(hdu_list=[FakeCCDData(meta={'INSTRUME': 'blah27'})])) + fake_data = FakeCCDData(meta={'INSTRUME': 'blah27'}) + fake_data.meta.pop('SATURATE') + LCOFrameFactory._init_saturate(FakeLCOObservationFrame(hdu_list=[fake_data])) def test_saturate_no_default_but_header(): diff --git a/banzai/tests/utils.py b/banzai/tests/utils.py index a147ee97..afd3aa61 100644 --- a/banzai/tests/utils.py +++ b/banzai/tests/utils.py @@ -43,7 +43,13 @@ def __init__(self, image_multiplier=1.0, nx=101, ny=103, name='test_image', read self.uncertainty = self.read_noise * np.ones(self.data.shape, dtype=self.data.dtype) else: self.uncertainty = uncertainty - + self.memmap = True + if 'SATURATE' not in self.meta: + self.meta['SATURATE'] = 65535.0 + if 'GAIN' not in self.meta: + self.meta['GAIN'] = 1.0 + if 'MAXLIN' not in self.meta: + self.meta['MAXLIN'] = 65535.0 for keyword in kwargs: setattr(self, keyword, kwargs[keyword]) @@ -63,7 +69,7 @@ def __init__(self, hdu_list=None, file_path='/tmp/test_image.fits', instrument=N self._file_path = file_path self.is_bad = False self.hdu_order = ['SCI', 'CAT', 'BPM', 'ERR'] - + self.n_sub_exposures = 1 for keyword in kwargs: setattr(self, keyword, kwargs[keyword]) @@ -159,6 +165,7 @@ def __init__(self, id=0, site='', camera='', enclosure='', telescope='', type='' self.nx = 4096 self.ny = 4096 + class FakeCalImage: def __init__(self): self.frameid = 1234 diff --git a/setup.cfg b/setup.cfg index 8e7c6b29..28c222b0 100755 --- a/setup.cfg +++ b/setup.cfg @@ -61,7 +61,7 @@ install_requires = pytest>=4.0 pyyaml psycopg2-binary - celery[redis]>=4.3.1,<5 + celery[redis]>5 apscheduler python-dateutil ocs_ingester>=3.0.4,<4.0.0 From c43dec349f3c5572bfecc7881f46d4830aa51586 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Mon, 12 Aug 2024 16:07:11 -0400 Subject: [PATCH 03/14] Fixes to tests, especially e2e testing infrastructure --- CHANGES.md | 12 +- banzai/data.py | 5 +- banzai/frames.py | 4 + banzai/tests/data/configdb_example.json | 374 ++++++++++- ...test_obs_portal_response_lsc_20240723.json | 605 ++++++++++++++++++ banzai/tests/data/test_precals.dat | 4 +- banzai/tests/e2e-k8s.yaml | 9 +- banzai/tests/test_end_to_end.py | 3 + environment.yaml | 547 ++++++++-------- 9 files changed, 1276 insertions(+), 287 deletions(-) create mode 100644 banzai/tests/data/test_obs_portal_response_lsc_20240723.json diff --git a/CHANGES.md b/CHANGES.md index 48faa0bf..68106182 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,18 +1,22 @@ -1.17.0 (2023-04-24) +1.18.0 (2024-08-09) +- Added support for frames that are composed of sub-exposures that + are stacked at site + +1.17.0 (2024-04-24) ------------------- - We now omit sources in the photometry stage that have an area larger than 1000 pixels as they lead to long processing times and are almost invariably spurious. -1.16.1 (2023-04-23) +1.16.1 (2024-04-23) ------------------- - Correction to aperture photometry. We were incorrectly using the radius instead of the diameter -1.16.0 (2023-04-18) +1.16.0 (2024-04-18) ------------------- - Calibration frames are now associated with output data products rather than frames so that we have more than one calibration data product produced per frame. -1.15.2 (2023-04-12) +1.15.2 (2024-04-12) ------------------- - Fix to fpacking data when the image data array is None diff --git a/banzai/data.py b/banzai/data.py index eb43c6c5..31ed60e6 100644 --- a/banzai/data.py +++ b/banzai/data.py @@ -313,7 +313,10 @@ def data_section(self, section): @property def n_sub_exposures(self): - return self.meta.get('NSUBREAD', 1) + n_exposures = self.meta.get('NSUBREAD', 1) + if str(n_exposures).lower() in ['n/a', 'unknown', 'none', '']: + n_exposures = 1 + return n_exposures def rebin(self, binning): # TODO: Implement me diff --git a/banzai/frames.py b/banzai/frames.py index ea4bbfbb..3c77c947 100644 --- a/banzai/frames.py +++ b/banzai/frames.py @@ -78,6 +78,10 @@ def background(self): def background(self, value): self.primary_hdu.background = value + @property + def n_sub_exposures(self): + return self.primary_hdu.n_sub_exposures + @abc.abstractmethod def save_processing_metadata(self, context): pass diff --git a/banzai/tests/data/configdb_example.json b/banzai/tests/data/configdb_example.json index 250900f7..b6ef51f2 100644 --- a/banzai/tests/data/configdb_example.json +++ b/banzai/tests/data/configdb_example.json @@ -2174,7 +2174,379 @@ "horizon": 15.0, "ha_limit_pos": 4.4666667, "ha_limit_neg": -4.533333, - "instrument_set": [], + "instrument_set": [{ + "id": 1496, + "code": "sq34", + "state": "SCHEDULABLE", + "telescope": "http://configdb.lco.gtn/telescopes/13/", + "autoguider_camera": { + "id": 1270, + "code": "sq34", + "camera_type": { + "id": 189, + "size": "117x78", + "pscale": 0.734, + "name": "0m4 QHY600", + "code": "0m4-SciCam-QHY600", + "pixels_x": 9576, + "pixels_y": 6388, + "max_rois": 0 + }, + "orientation": 0.0, + "optical_elements": { + "filters": "OIII,SII,Astrodon-Exo,w,opaque,up,rp,ip,gp,zs,V,B,H-Alpha" + }, + "optical_element_groups": [ + { + "name": "DeltaRho QHY600 default wheel", + "type": "filters", + "optical_elements": [ + { + "name": "OIII", + "code": "OIII", + "schedulable": true + }, + { + "name": "S[II]", + "code": "SII", + "schedulable": true + }, + { + "name": "Astrodon-Exo", + "code": "Astrodon-Exo", + "schedulable": true + }, + { + "name": "PanSTARRS-w", + "code": "w", + "schedulable": true + }, + { + "name": "Opaque", + "code": "opaque", + "schedulable": false + }, + { + "name": "SDSS-up", + "code": "up", + "schedulable": true + }, + { + "name": "SDSS-rp", + "code": "rp", + "schedulable": true + }, + { + "name": "SDSS-ip", + "code": "ip", + "schedulable": true + }, + { + "name": "SDSS-gp", + "code": "gp", + "schedulable": true + }, + { + "name": "PanSTARRS-Z", + "code": "zs", + "schedulable": true + }, + { + "name": "Bessell-V", + "code": "V", + "schedulable": true + }, + { + "name": "Bessell-B", + "code": "B", + "schedulable": true + }, + { + "name": "H Alpha", + "code": "H-Alpha", + "schedulable": true + } + ], + "element_change_overhead": 2.0, + "default": "" + } + ], + "host": "inst.0m4a.aqwa.lsc.lco.gtn" + }, + "science_cameras": [ + { + "id": 1270, + "code": "sq34", + "camera_type": { + "id": 189, + "size": "117x78", + "pscale": 0.734, + "name": "0m4 QHY600", + "code": "0m4-SciCam-QHY600", + "pixels_x": 9576, + "pixels_y": 6388, + "max_rois": 0 + }, + "orientation": 0.0, + "optical_elements": { + "filters": "OIII,SII,Astrodon-Exo,w,opaque,up,rp,ip,gp,zs,V,B,H-Alpha" + }, + "optical_element_groups": [ + { + "name": "DeltaRho QHY600 default wheel", + "type": "filters", + "optical_elements": [ + { + "name": "OIII", + "code": "OIII", + "schedulable": true + }, + { + "name": "S[II]", + "code": "SII", + "schedulable": true + }, + { + "name": "Astrodon-Exo", + "code": "Astrodon-Exo", + "schedulable": true + }, + { + "name": "PanSTARRS-w", + "code": "w", + "schedulable": true + }, + { + "name": "Opaque", + "code": "opaque", + "schedulable": false + }, + { + "name": "SDSS-up", + "code": "up", + "schedulable": true + }, + { + "name": "SDSS-rp", + "code": "rp", + "schedulable": true + }, + { + "name": "SDSS-ip", + "code": "ip", + "schedulable": true + }, + { + "name": "SDSS-gp", + "code": "gp", + "schedulable": true + }, + { + "name": "PanSTARRS-Z", + "code": "zs", + "schedulable": true + }, + { + "name": "Bessell-V", + "code": "V", + "schedulable": true + }, + { + "name": "Bessell-B", + "code": "B", + "schedulable": true + }, + { + "name": "H Alpha", + "code": "H-Alpha", + "schedulable": true + } + ], + "element_change_overhead": 2.0, + "default": "" + } + ], + "host": "inst.0m4a.aqwa.lsc.lco.gtn" + } + ], + "instrument_type": { + "id": 100, + "name": "0m4 SCICAM QHY600", + "code": "0m4-SciCam-QHY600", + "fixed_overhead_per_exposure": 1.0, + "instrument_category": "IMAGE", + "observation_front_padding": 90.0, + "acquire_exposure_time": 0.0, + "default_configuration_type": "EXPOSE", + "mode_types": [ + { + "type": "guiding", + "modes": [ + { + "name": "Guide Off", + "overhead": 0.0, + "code": "OFF", + "schedulable": true, + "validation_schema": {} + }, + { + "name": "Guide On", + "overhead": 0.0, + "code": "ON", + "schedulable": true, + "validation_schema": {} + } + ], + "default": "ON" + }, + { + "type": "readout", + "modes": [ + { + "name": "QHY600 Central 30x30 arcmin", + "overhead": 4.0, + "code": "central30x30", + "schedulable": true, + "validation_schema": {} + }, + { + "name": "QHY600 Full Frame Readout", + "overhead": 9.0, + "code": "full_frame", + "schedulable": true, + "validation_schema": { + "extra_params": { + "type": "dict", + "schema": { + "bin_x": { + "type": "integer", + "allowed": [ + 1 + ], + "default": 1 + }, + "bin_y": { + "type": "integer", + "allowed": [ + 1 + ], + "default": 1 + } + }, + "default": {}, + "required": true + }, + "exposure_count": { + "max": 50 + } + } + } + ], + "default": "central30x30" + }, + { + "type": "acquisition", + "modes": [ + { + "name": "Acquire Off", + "overhead": 0.0, + "code": "OFF", + "schedulable": true, + "validation_schema": {} + } + ], + "default": "OFF" + } + ], + "default_acceptability_threshold": 90.0, + "config_front_padding": 5.0, + "allow_self_guiding": true, + "configuration_types": [ + { + "name": "Exposure", + "code": "EXPOSE", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": false, + "requires_optical_elements": true, + "validation_schema": {} + }, + { + "name": "Exposure Sequence", + "code": "REPEAT_EXPOSE", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": false, + "requires_optical_elements": true, + "validation_schema": {} + }, + { + "name": "AUTO_FOCUS", + "code": "AUTO_FOCUS", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": false, + "requires_optical_elements": true, + "validation_schema": {} + }, + { + "name": "SKY_FLAT", + "code": "SKY_FLAT", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": false, + "requires_optical_elements": true, + "validation_schema": { + "exposure_time": { + "type": "float", + "default": 1 + } + } + }, + { + "name": "BIAS", + "code": "BIAS", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": true, + "requires_optical_elements": false, + "validation_schema": {} + }, + { + "name": "DARK", + "code": "DARK", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": true, + "requires_optical_elements": false, + "validation_schema": {} + }, + { + "name": "STANDARD", + "code": "STANDARD", + "config_change_overhead": 0.0, + "schedulable": true, + "force_acquisition_off": false, + "requires_optical_elements": true, + "validation_schema": {} + } + ], + "validation_schema": { + "extra_params": { + "type": "dict", + "schema": { + "defocus": { + "max": 5.0, + "min": -5.0, + "type": "float" + } + } + } + } + }, + "__str__": "lsc.aqwa.0m4a.sq34" + } + ], "__str__": "lsc.aqwa.0m4a" } ], diff --git a/banzai/tests/data/test_obs_portal_response_lsc_20240723.json b/banzai/tests/data/test_obs_portal_response_lsc_20240723.json new file mode 100644 index 00000000..77343778 --- /dev/null +++ b/banzai/tests/data/test_obs_portal_response_lsc_20240723.json @@ -0,0 +1,605 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 631366169, + "request": { + "id": 3597084, + "observation_note": "", + "optimization_type": "TIME", + "state": "COMPLETED", + "acceptability_threshold": 90.0, + "configuration_repeats": 1, + "extra_params": {}, + "modified": "2024-07-24T05:21:36.018338Z", + "duration": 472, + "configurations": [ + { + "id": 11542373, + "instrument_type": "0M4-SCICAM-QHY600", + "type": "SKY_FLAT", + "repeat_duration": null, + "extra_params": {}, + "priority": 3, + "instrument_configs": [ + { + "optical_elements": { + "filter": "B" + }, + "mode": "full_frame", + "exposure_time": 1.0, + "exposure_count": 5, + "rotator_mode": "", + "extra_params": { + "bin_x": 1, + "bin_y": 1 + }, + "rois": [] + } + ], + "constraints": { + "max_airmass": 20.0, + "min_lunar_distance": 0.0, + "max_lunar_phase": 1.0, + "extra_params": {} + }, + "acquisition_config": { + "mode": "OFF", + "extra_params": {} + }, + "guiding_config": { + "optional": true, + "mode": "OFF", + "optical_elements": {}, + "exposure_time": null, + "extra_params": {} + }, + "target": { + "type": "HOUR_ANGLE", + "name": "No target", + "ra": null, + "dec": 0.0, + "proper_motion_ra": null, + "proper_motion_dec": null, + "parallax": null, + "epoch": null, + "hour_angle": 1.0, + "extra_params": {} + }, + "configuration_status": 812512180, + "state": "COMPLETED", + "instrument_name": "sq34", + "guide_camera_name": "", + "summary": { + "id": 4815648, + "start": "2024-07-23T22:11:04Z", + "end": "2024-07-23T22:22:19Z", + "state": "COMPLETED", + "reason": "", + "time_completed": 0.0, + "events": [ + { + "time": "2024-07-23T22:11:05", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.mount.HomeCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.astrometric.EnableTrackingCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.focus.HomeCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.enclosure.OpenCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "DONE", + "description": "org.lcogt.sequencer.command.enclosure.OpenCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "DONE", + "description": "org.lcogt.sequencer.command.focus.HomeCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.EnableTrackingCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.telescopeservices.StowMirrorCoverCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "DONE", + "description": "org.lcogt.sequencer.command.mount.HomeCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.focus.DeFocusCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "DONE", + "description": "org.lcogt.sequencer.command.telescopeservices.StowMirrorCoverCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "DONE", + "description": "org.lcogt.sequencer.command.focus.DeFocusCommand" + }, + { + "time": "2024-07-23T22:11:05", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + }, + { + "time": "2024-07-23T22:11:06", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + }, + { + "time": "2024-07-23T22:11:06", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + }, + { + "time": "2024-07-23T22:11:06", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + }, + { + "time": "2024-07-23T22:11:12", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + }, + { + "time": "2024-07-23T22:11:12", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:12:03", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:12:34", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:12:47", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:12:57", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:13:20", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:13:30", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:14:23", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:21:02", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + }, + { + "time": "2024-07-23T22:21:03", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + }, + { + "time": "2024-07-23T22:21:03", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:21:04", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T22:21:13", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T22:21:13", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + }, + { + "time": "2024-07-23T22:21:19", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:21:19", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + }, + { + "time": "2024-07-23T22:21:19", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T22:21:29", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T22:21:29", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + }, + { + "time": "2024-07-23T22:21:35", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + }, + { + "time": "2024-07-23T22:21:35", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:21:36", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T22:21:45", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T22:21:45", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + }, + { + "time": "2024-07-23T22:21:51", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + }, + { + "time": "2024-07-23T22:21:51", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:21:53", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T22:22:01", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T22:22:01", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + }, + { + "time": "2024-07-23T22:22:08", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + }, + { + "time": "2024-07-23T22:22:08", + "state": "DONE", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + }, + { + "time": "2024-07-23T22:22:08", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T22:22:17", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-23T22:22:18", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + }, + { + "time": "2024-07-23T22:22:19", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + } + ] + } + } + ] + } + }, +{ + "id": 631577495, + "request": { + "id": 3597581, + "observation_note": "", + "optimization_type": "TIME", + "state": "COMPLETED", + "acceptability_threshold": 90.0, + "configuration_repeats": 1, + "extra_params": {}, + "modified": "2024-07-24T21:21:09.473360Z", + "duration": 3427, + "configurations": [ + { + "id": 11549305, + "instrument_type": "0M4-SCICAM-QHY600", + "type": "BIAS", + "repeat_duration": null, + "extra_params": {}, + "priority": 4, + "instrument_configs": [ + { + "optical_elements": { + "filter": "opaque" + }, + "mode": "full_frame", + "exposure_time": 0.0, + "exposure_count": 5, + "rotator_mode": "", + "extra_params": { + "bin_x": 1, + "bin_y": 1 + }, + "rois": [] + } + ], + "constraints": { + "max_airmass": 20.0, + "min_lunar_distance": 0.0, + "max_lunar_phase": 1.0, + "extra_params": {} + }, + "acquisition_config": { + "mode": "OFF", + "extra_params": {} + }, + "guiding_config": { + "optional": true, + "mode": "OFF", + "optical_elements": {}, + "exposure_time": null, + "extra_params": {} + }, + "target": { + "type": "HOUR_ANGLE", + "name": "No target", + "ra": null, + "dec": 0.0, + "proper_motion_ra": null, + "proper_motion_dec": null, + "parallax": null, + "epoch": null, + "hour_angle": 1.0, + "extra_params": {} + }, + "configuration_status": 813028208, + "state": "COMPLETED", + "instrument_name": "sq34", + "guide_camera_name": "", + "summary": { + "id": 4818287, + "start": "2024-07-24T20:52:40Z", + "end": "2024-07-24T20:55:16Z", + "state": "COMPLETED", + "reason": "", + "time_completed": 0.0, + "events": [ + { + "time": "2024-07-24T20:52:40", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + }, + { + "time": "2024-07-24T20:52:41", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + }, + { + "time": "2024-07-24T20:52:41", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + }, + { + "time": "2024-07-24T20:52:42", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + }, + { + "time": "2024-07-24T20:52:43", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-24T20:55:14", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-24T20:55:15", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + }, + { + "time": "2024-07-24T20:55:16", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + } + ] + } + }, + { + "id": 11549306, + "instrument_type": "0M4-SCICAM-QHY600", + "type": "DARK", + "repeat_duration": null, + "extra_params": {}, + "priority": 5, + "instrument_configs": [ + { + "optical_elements": { + "filter": "opaque" + }, + "mode": "full_frame", + "exposure_time": 300.0, + "exposure_count": 5, + "rotator_mode": "", + "extra_params": { + "bin_x": 1, + "bin_y": 1 + }, + "rois": [] + } + ], + "constraints": { + "max_airmass": 20.0, + "min_lunar_distance": 0.0, + "max_lunar_phase": 1.0, + "extra_params": {} + }, + "acquisition_config": { + "mode": "OFF", + "extra_params": {} + }, + "guiding_config": { + "optional": true, + "mode": "OFF", + "optical_elements": {}, + "exposure_time": null, + "extra_params": {} + }, + "target": { + "type": "HOUR_ANGLE", + "name": "No target", + "ra": null, + "dec": 0.0, + "proper_motion_ra": null, + "proper_motion_dec": null, + "parallax": null, + "epoch": null, + "hour_angle": 1.0, + "extra_params": {} + }, + "configuration_status": 813028209, + "state": "COMPLETED", + "instrument_name": "sq34", + "guide_camera_name": "", + "summary": { + "id": 4818307, + "start": "2024-07-24T20:55:19Z", + "end": "2024-07-24T21:21:09Z", + "state": "COMPLETED", + "reason": "", + "time_completed": 1500.0, + "events": [ + { + "time": "2024-07-24T20:55:19", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + }, + { + "time": "2024-07-24T20:55:20", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + }, + { + "time": "2024-07-24T20:55:20", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + }, + { + "time": "2024-07-24T20:55:22", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + }, + { + "time": "2024-07-24T20:55:23", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-24T21:21:07", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + }, + { + "time": "2024-07-24T21:21:07", + "state": "PROCESSING", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + }, + { + "time": "2024-07-24T21:21:08", + "state": "DONE", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + } + ] + } + } + ] + }, + "site": "lsc", + "enclosure": "aqwa", + "telescope": "0m4a", + "start": "2024-07-24T20:26:04.169574Z", + "end": "2024-07-24T21:32:04.169574Z", + "priority": 10, + "state": "COMPLETED", + "proposal": "calibrate", + "submitter": "eng", + "name": "LCOGT", + "ipp_value": 1.0, + "observation_type": "DIRECT", + "request_group_id": 2012784, + "created": "2024-07-24T15:05:49.265713Z", + "modified": "2024-07-24T21:21:09.532093Z" + } + ] +} \ No newline at end of file diff --git a/banzai/tests/data/test_precals.dat b/banzai/tests/data/test_precals.dat index e1ce81ea..aa5ccb00 100644 --- a/banzai/tests/data/test_precals.dat +++ b/banzai/tests/data/test_precals.dat @@ -7,5 +7,5 @@ frameid filename binning mode obstype dateobs site instrument 33347347 tst1m0XX-ep02-20200710-bpm-1.fits.fz '[1, 1]' 1 BPM 2020-07-10 tst ep02 33347348 tst1m0XX-ep03-20200710-bpm-1.fits.fz '[1, 1]' 1 BPM 2020-07-10 tst ep03 33347349 tst1m0XX-ep04-20200710-bpm-1.fits.fz '[1, 1]' 1 BPM 2020-07-10 tst ep04 -58186456 ogg0m404-sq30-20221123-readnoise-default-e2e.fits.fz '[1, 1]' default READNOISE 2022-11-23 ogg sq30 -56389948 ogg0m404-sq30-20221206-bpm-default.fits.fz '[1, 1]' default BPM 2022-12-06 ogg sq30 +69359146 lsc0m409-sq34-20240314-bpm-full_frame.fits.fz '[1, 1]' full_frame BPM 2024-03-14 lsc sq34 +69359157 lsc0m409-sq34-20240314-readnoise-full_frame.fits.fz '[1, 1]' full_frame 2024-03-14 lsc sq34 diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 53180e25..9fb20c50 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -42,13 +42,14 @@ spec: cpu: 1 memory: 512Mi limits: - cpu: 2 + cpu: 4 memory: 512Mi readinessProbe: exec: command: - - rabbitmqctl - - node_health_check + - rabbitmq-diagnostics + - "-q" + - ping initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 @@ -93,7 +94,7 @@ spec: - --hostname - "banzai-celery-worker" - --concurrency - - "2" + - "4" - -l - "info" - "-Q" diff --git a/banzai/tests/test_end_to_end.py b/banzai/tests/test_end_to_end.py index b879ca69..63e9cbf6 100644 --- a/banzai/tests/test_end_to_end.py +++ b/banzai/tests/test_end_to_end.py @@ -159,6 +159,9 @@ def run_check_if_stacked_calibrations_are_in_db(raw_file_pattern, calibration_ty def observation_portal_side_effect(*args, **kwargs): + # To produce the mock observation portal resonse, we need to modify the response from + # this type of url + # https://observe.lco.global/api/observations/?enclosure=aqwa&telescope=0m4a&priority=&state=COMPLETED&time_span=&start_after=2024-07-23&start_before=&end_after=&end_before=2024-07-25&modified_after=&created_after=&created_before=&request_id=&request_group_id=&user=&proposal=calibrate&instrument_type=0M4-SCICAM-QHY600&configuration_type=SKY_FLAT&ordering= site = kwargs['params']['site'] start = datetime.strftime(parse(kwargs['params']['start_after']).replace(tzinfo=None).date(), '%Y%m%d') filename = 'test_obs_portal_response_{site}_{start}.json'.format(site=site, start=start) diff --git a/environment.yaml b/environment.yaml index 0ea4596f..fceea57e 100644 --- a/environment.yaml +++ b/environment.yaml @@ -7,282 +7,278 @@ channels: - astropy - defaults dependencies: - - _libgcc_mutex=0.1=conda_forge - _openmp_mutex=4.5=2_gnu - - affine=2.4.0=pyhd8ed1ab_0 - - amqp=2.6.1=pyh9f0ad1d_0 - - anyio=4.0.0=pyhd8ed1ab_0 - - aom=3.6.1=h59595ed_0 - - apscheduler=3.10.4=pyhd8ed1ab_0 - - asdf=2.15.1=pyhd8ed1ab_0 - - asdf-astropy=0.4.0=pyhd8ed1ab_1 - - asdf-coordinates-schemas=0.2.0=pyhd8ed1ab_1 - - asdf-standard=1.0.3=pyhd8ed1ab_0 - - asdf-transform-schemas=0.3.0=pyhd8ed1ab_0 - - asdf-unit-schemas=0.1.0=pyhd8ed1ab_0 - - asdf-wcs-schemas=0.1.1=pyhd8ed1ab_0 - - astropy=5.3.4=py310h1f7b6fc_2 - - asttokens=2.4.0=pyhd8ed1ab_0 - - attrs=23.1.0=pyh71513ae_1 - - backcall=0.2.0=pyh9f0ad1d_0 - - backports=1.0=pyhd8ed1ab_3 - - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 - - backports.zoneinfo=0.2.1=py310hff52083_8 - - billiard=3.6.4.0=py310h5764c6d_3 - - blosc=1.21.5=h0f2a231_0 - - bottleneck=1.3.7=py310h1f7b6fc_1 - - brotli=1.1.0=hd590300_1 - - brotli-bin=1.1.0=hd590300_1 - - brotli-python=1.1.0=py310hc6cd4ac_1 - - brunsli=0.1=h9c3ff4c_0 - - bzip2=1.0.8=h7f98852_4 - - c-ares=1.20.1=hd590300_0 - - c-blosc2=2.10.5=hb4ffafa_0 - - ca-certificates=2023.7.22=hbcca054_0 - - cairo=1.18.0=h3faef2a_0 - - celery=4.4.0=py_0 - - certifi=2023.7.22=pyhd8ed1ab_0 - - cffi=1.16.0=py310h2fee648_0 - - cfitsio=4.3.0=hbdc6101_0 - - charls=2.4.2=h59595ed_0 - - charset-normalizer=3.3.0=pyhd8ed1ab_0 - - click=8.1.7=unix_pyh707e725_0 - - click-plugins=1.1.1=py_0 - - cligj=0.7.2=pyhd8ed1ab_1 - - colorama=0.4.6=pyhd8ed1ab_0 - - contourpy=1.1.1=py310hd41b1e2_1 - - coverage=7.3.2=py310h2372a71_0 - - cryptography=41.0.4=py310h75e40e8_0 - - cycler=0.12.1=pyhd8ed1ab_0 - - cython=0.29.36=py310hc6cd4ac_1 - - dav1d=1.2.1=hd590300_0 - - decorator=5.1.1=pyhd8ed1ab_0 - - dnspython=2.4.2=pyhd8ed1ab_0 - - elasticsearch=5.5.3=pyh9f0ad1d_0 - - emcee=3.1.4=pyhd8ed1ab_0 - - exceptiongroup=1.1.3=pyhd8ed1ab_0 - - executing=1.2.0=pyhd8ed1ab_0 - - expat=2.5.0=hcb278e6_1 - - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 - - font-ttf-inconsolata=3.000=h77eed37_0 - - font-ttf-source-code-pro=2.038=h77eed37_0 - - font-ttf-ubuntu=0.83=hab24e00_0 - - fontconfig=2.14.2=h14ed4e7_0 - - fonts-conda-ecosystem=1=0 - - fonts-conda-forge=1=0 - - fonttools=4.43.1=py310h2372a71_0 - - freetype=2.12.1=h267a509_2 - - freexl=2.0.0=h743c826_0 - - geos=3.12.0=h59595ed_0 - - geotiff=1.7.1=hf074850_14 - - gettext=0.21.1=h27087fc_0 - - giflib=5.2.1=h0b41bf4_3 - - greenlet=3.0.0=py310hc6cd4ac_1 - - gwcs=0.19.0=pyhd8ed1ab_0 - - h11=0.14.0=pyhd8ed1ab_0 - - h2=4.1.0=pyhd8ed1ab_0 - - hdf4=4.2.15=h2a13503_7 - - hdf5=1.14.2=nompi_h4f84152_100 - - hpack=4.0.0=pyh9f0ad1d_0 - - httpcore=1.0.0=pyhd8ed1ab_0 - - hyperframe=6.0.1=pyhd8ed1ab_0 - - hypothesis=6.87.4=pyha770c72_0 - - icu=73.2=h59595ed_0 - - idna=3.4=pyhd8ed1ab_0 - - imagecodecs=2023.9.18=py310h496a806_2 - - imageio=2.31.5=pyh8c1a49c_0 - - importlib-metadata=6.8.0=pyha770c72_0 - - importlib-resources=6.1.0=pyhd8ed1ab_0 - - importlib_resources=6.1.0=pyhd8ed1ab_0 - - iniconfig=2.0.0=pyhd8ed1ab_0 - - intel-openmp=2022.1.0=h9e868ea_3769 - - ipython=8.16.1=pyh0d859eb_0 - - jedi=0.19.1=pyhd8ed1ab_0 - - jmespath=1.0.1=pyhd8ed1ab_0 - - joblib=1.3.2=pyhd8ed1ab_0 - - json-c=0.17=h7ab15ed_0 - - jsonschema=4.19.1=pyhd8ed1ab_0 - - jsonschema-specifications=2023.7.1=pyhd8ed1ab_0 - - jxrlib=1.1=h7f98852_2 - - kealib=1.5.2=hcd42e92_1 - - keyutils=1.6.1=h166bdaf_0 - - kiwisolver=1.4.5=py310hd41b1e2_1 - - krb5=1.21.2=h659d440_0 - - lazy_loader=0.3=pyhd8ed1ab_0 - - lcms2=2.15=hb7c19ff_3 - - ld_impl_linux-64=2.40=h41732ed_0 - - lerc=4.0.0=h27087fc_0 - - libaec=1.1.2=h59595ed_1 - - libarchive=3.7.2=h039dbb9_0 - - libavif16=1.0.1=h87da1f6_2 - - libblas=3.9.0=18_linux64_openblas - - libboost-headers=1.82.0=ha770c72_6 - - libbrotlicommon=1.1.0=hd590300_1 - - libbrotlidec=1.1.0=hd590300_1 - - libbrotlienc=1.1.0=hd590300_1 - - libcblas=3.9.0=18_linux64_openblas - - libcurl=8.4.0=hca28451_0 - - libdeflate=1.19=hd590300_0 - - libedit=3.1.20191231=he28a2e2_2 - - libev=4.33=h516909a_1 - - libexpat=2.5.0=hcb278e6_1 - - libffi=3.4.2=h7f98852_5 - - libgcc-ng=13.2.0=h807b86a_2 - - libgdal=3.7.2=h6f3d308_7 - - libgfortran-ng=13.2.0=h69a702a_2 - - libgfortran5=13.2.0=ha4646dd_2 - - libglib=2.78.0=hebfc3b9_0 - - libgomp=13.2.0=h807b86a_2 - - libiconv=1.17=h166bdaf_0 - - libjpeg-turbo=3.0.0=hd590300_1 - - libkml=1.3.0=h01aab08_1018 - - liblapack=3.9.0=18_linux64_openblas - - libnetcdf=4.9.2=nompi_h80fb2b6_112 - - libnghttp2=1.52.0=h61bc06f_0 - - libnsl=2.0.0=hd590300_1 - - libopenblas=0.3.24=pthreads_h413a1c8_0 - - libpng=1.6.39=h753d276_0 - - libpq=16.0=hfc447b1_1 - - libprotobuf=3.20.3=h3eb15da_0 - - librttopo=1.1.0=hb58d41b_14 - - libspatialite=5.1.0=h090f1da_0 - - libsqlite=3.43.2=h2797004_0 - - libssh2=1.11.0=h0841786_0 - - libstdcxx-ng=13.2.0=h7e041cc_2 - - libtiff=4.6.0=ha9c0a0a_2 - - libuuid=2.38.1=h0b41bf4_0 - - libwebp-base=1.3.2=hd590300_0 - - libxcb=1.15=h0b41bf4_0 - - libxml2=2.11.5=h232c23b_1 - - libzip=1.10.1=h2629f0a_3 - - libzlib=1.2.13=hd590300_5 - - libzopfli=1.0.3=h9c3ff4c_0 - - lz4-c=1.9.4=hcb278e6_0 - - lzo=2.10=h516909a_1000 - - matplotlib-base=3.8.0=py310h62c0568_2 - - matplotlib-inline=0.1.6=pyhd8ed1ab_0 - - minizip=4.0.1=h0ab5242_5 - - mkl=2022.1.0=hc2b9512_224 - - mock=5.1.0=pyhd8ed1ab_0 - - munkres=1.1.4=pyh9f0ad1d_0 - - mysql-common=8.0.33=hf1915f5_5 - - mysql-connector-python=8.0.32=py310h6eefaca_0 - - mysql-libs=8.0.33=hca2cd23_5 - - ncurses=6.4=hcb278e6_0 - - networkx=3.1=pyhd8ed1ab_0 - - ninja=1.11.1=h924138e_0 - - nspr=4.35=h27087fc_0 - - nss=3.94=h1d7d5a4_0 - - numpy=1.23.5=py310h53a5b5f_0 - - openjpeg=2.5.0=h488ebb8_3 - - openssl=3.1.3=hd590300_0 - - packaging=23.2=pyhd8ed1ab_0 - - parso=0.8.3=pyhd8ed1ab_0 - - pcre2=10.40=hc3806b6_0 - - pexpect=4.8.0=pyh1a96a4e_2 - - photutils=1.9.0=py310h2372a71_0 - - pickleshare=0.7.5=py_1003 - - pillow=10.0.1=py310h01dd4db_2 - - pip=23.2.1=pyhd8ed1ab_0 - - pixman=0.42.2=h59595ed_0 - - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 - - pluggy=1.3.0=pyhd8ed1ab_0 - - poppler=23.10.0=h590f24d_0 - - poppler-data=0.4.12=hd8ed1ab_0 - - postgresql=16.0=h8972f4a_1 - - proj=9.3.0=h1d62c97_1 - - prompt-toolkit=3.0.39=pyha770c72_0 - - prompt_toolkit=3.0.39=hd8ed1ab_0 - - protobuf=3.20.3=py310heca2aa9_1 - - psutil=5.9.5=py310h2372a71_1 - - psycopg2=2.9.7=py310h275853b_1 - - psycopg2-binary=2.9.7=pyhd8ed1ab_1 - - pthread-stubs=0.4=h36c2ea0_1001 - - ptyprocess=0.7.0=pyhd3deb0d_0 - - pure_eval=0.2.2=pyhd8ed1ab_0 - - pycparser=2.21=pyhd8ed1ab_0 - - pyerfa=2.0.0.3=py310h1f7b6fc_1 - - pygments=2.16.1=pyhd8ed1ab_0 - - pyparsing=3.1.1=pyhd8ed1ab_0 - - pysocks=1.7.1=pyha2e5f31_6 - - pytest=7.4.2=pyhd8ed1ab_0 - - pytest-arraydiff=0.5.0=pyhd8ed1ab_0 - - pytest-astropy=0.10.0=pyhd8ed1ab_0 - - pytest-astropy-header=0.2.2=pyhd8ed1ab_0 - - pytest-cov=4.1.0=pyhd8ed1ab_0 - - pytest-doctestplus=1.0.0=pyhd8ed1ab_0 - - pytest-filter-subpackage=0.1.2=pyhd8ed1ab_0 - - pytest-mock=3.11.1=pyhd8ed1ab_0 - - pytest-openfiles=0.5.0=py_0 - - pytest-remotedata=0.4.1=pyhd8ed1ab_0 - - python=3.10.12=hd12c33a_0_cpython - - python-dateutil=2.8.2=pyhd8ed1ab_0 - - python_abi=3.10=4_cp310 - - pytorch=1.12.1=cpu_py310h75c9ab6_0 - - pytz=2023.3.post1=pyhd8ed1ab_0 - - pywavelets=1.4.1=py310h1f7b6fc_1 - - pyyaml=6.0.1=py310h2372a71_1 - - rasterio=1.3.8=py310h6a913dc_4 - - rav1e=0.6.6=he8a937b_2 - - readline=8.2=h8228510_1 - - referencing=0.30.2=pyhd8ed1ab_0 - - requests=2.31.0=pyhd8ed1ab_0 - - rpds-py=0.10.6=py310hcb5633a_0 - - scikit-image=0.22.0=py310hcc13569_2 - - scikit-learn=1.3.1=py310h1fdf081_1 - - scipy=1.11.3=py310hb13e2d6_1 - - semantic_version=2.10.0=pyhd8ed1ab_0 - - sep=1.2.1=py310h1f7b6fc_2 - - setuptools=68.2.2=pyhd8ed1ab_0 - - shapely=2.0.2=py310h7dcad9a_0 - - simplejson=3.19.2=py310h2372a71_0 - - six=1.16.0=pyh6c4a22f_0 - - sleef=3.5.1=h9b69904_2 - - snappy=1.1.10=h9fff704_0 - - sniffio=1.3.0=pyhd8ed1ab_0 - - snuggs=1.4.7=py_0 - - sortedcontainers=2.4.0=pyhd8ed1ab_0 - - sqlalchemy=2.0.22=py310h2372a71_0 - - sqlite=3.43.2=h2c6b66d_0 - - stack_data=0.6.2=pyhd8ed1ab_0 - - svt-av1=1.7.0=h59595ed_0 - - tenacity=8.2.3=pyhd8ed1ab_0 - - threadpoolctl=3.2.0=pyha21a80b_0 - - tifffile=2023.9.26=pyhd8ed1ab_0 - - tiledb=2.16.3=hf0b6e87_3 - - tk=8.6.13=h2797004_0 - - toml=0.10.2=pyhd8ed1ab_0 - - tomli=2.0.1=pyhd8ed1ab_0 - - tqdm=4.66.1=pyhd8ed1ab_0 - - traitlets=5.11.2=pyhd8ed1ab_0 - - typing-extensions=4.8.0=hd8ed1ab_0 - - typing_extensions=4.8.0=pyha770c72_0 - - tzcode=2023c=h0b41bf4_0 - - tzdata=2023c=h71feb2d_0 - - tzlocal=5.1=py310hff52083_0 - - unicodedata2=15.1.0=py310h2372a71_0 - - uriparser=0.9.7=hcb278e6_1 - - wcwidth=0.2.8=pyhd8ed1ab_0 - - wheel=0.41.2=pyhd8ed1ab_0 - - xerces-c=3.2.4=hac6953d_3 - - xorg-kbproto=1.0.7=h7f98852_1002 - - xorg-libice=1.1.1=hd590300_0 - - xorg-libsm=1.2.4=h7391055_0 - - xorg-libx11=1.8.7=h8ee46fc_0 - - xorg-libxau=1.0.11=hd590300_0 - - xorg-libxdmcp=1.1.3=h7f98852_0 - - xorg-libxext=1.3.4=h0b41bf4_2 - - xorg-libxrender=0.9.11=hd590300_0 - - xorg-renderproto=0.11.1=h7f98852_1002 - - xorg-xextproto=7.3.0=h0b41bf4_1003 - - xorg-xproto=7.0.31=h7f98852_1007 - - xz=5.2.6=h166bdaf_0 - - yaml=0.2.5=h7f98852_2 - - zfp=1.0.0=h59595ed_4 - - zipp=3.17.0=pyhd8ed1ab_0 - - zlib=1.2.13=hd590300_5 - - zlib-ng=2.0.7=h0b41bf4_0 - - zstd=1.5.5=hfc55251_0 + - affine=2.4.0 + - amqp=2.6.1 + - anyio=4.0.0 + - aom=3.6.1 + - apscheduler=3.10.4 + - asdf=2.15.1 + - asdf-astropy=0.4.0 + - asdf-coordinates-schemas=0.2.0 + - asdf-standard=1.0.3 + - asdf-transform-schemas=0.3.0 + - asdf-unit-schemas=0.1.0 + - asdf-wcs-schemas=0.1.1 + - astropy=5.3.4 + - asttokens=2.4.0 + - attrs=23.1.0 + - backcall=0.2.0 + - backports=1.0 + - backports.functools_lru_cache=1.6.5 + - backports.zoneinfo=0.2.1 + - billiard=3.6.4.0 + - blosc=1.21.5 + - bottleneck=1.3.7 + - brotli=1.1.0 + - brotli-bin=1.1.0 + - brotli-python=1.1.0 + - brunsli=0.1 + - bzip2=1.0.8 + - c-ares=1.20.1 + - c-blosc2=2.10.5 + - ca-certificates=2023.7.22 + - cairo=1.18.0 + - celery=4.4.0 + - certifi=2023.7.22 + - cffi=1.16.0 + - cfitsio=4.3.0 + - charls=2.4.2 + - charset-normalizer=3.3.0 + - click=8.1.7 + - click-plugins=1.1.1 + - cligj=0.7.2 + - colorama=0.4.6 + - contourpy=1.1.1 + - coverage=7.3.2 + - cryptography=41.0.4 + - cycler=0.12.1 + - cython=0.29.36 + - dav1d=1.2.1 + - decorator=5.1.1 + - dnspython=2.4.2 + - elasticsearch=5.5.3 + - emcee=3.1.4 + - exceptiongroup=1.1.3 + - executing=1.2.0 + - expat=2.5.0 + - font-ttf-dejavu-sans-mono=2.37 + - font-ttf-inconsolata=3.000 + - font-ttf-source-code-pro=2.038 + - font-ttf-ubuntu=0.83 + - fontconfig=2.14.2 + - fonts-conda-ecosystem=1 + - fonts-conda-forge=1 + - fonttools=4.43.1 + - freetype=2.12.1 + - freexl=2.0.0 + - geos=3.12.0 + - geotiff=1.7.1 + - gettext=0.21.1 + - giflib=5.2.1 + - greenlet=3.0.0 + - gwcs=0.19.0 + - h11=0.14.0 + - h2=4.1.0 + - hdf4=4.2.15 + - hdf5=1.14.2 + - hpack=4.0.0 + - httpcore=1.0.0 + - hyperframe=6.0.1 + - hypothesis=6.87.4 + - icu=73.2 + - idna=3.4 + - imagecodecs=2023.9.18 + - imageio=2.31.5 + - importlib-metadata=6.8.0 + - importlib-resources=6.1.0 + - importlib_resources=6.1.0 + - iniconfig=2.0.0 + - ipython=8.16.1 + - jedi=0.19.1 + - jmespath=1.0.1 + - joblib=1.3.2 + - json-c=0.17 + - jsonschema=4.19.1 + - jsonschema-specifications=2023.7.1 + - jxrlib=1.1 + - kealib=1.5.2 + - keyutils=1.6.1 + - kiwisolver=1.4.5 + - krb5=1.21.2 + - lazy_loader=0.3 + - lcms2=2.15 + - ld_impl_linux-64=2.40 + - lerc=4.0.0 + - libaec=1.1.2 + - libarchive=3.7.2 + - libavif16=1.0.1 + - libblas=3.9.0 + - libboost-headers=1.82.0 + - libbrotlicommon=1.1.0 + - libbrotlidec=1.1.0 + - libbrotlienc=1.1.0 + - libcblas=3.9.0 + - libcurl=8.4.0 + - libdeflate=1.19 + - libedit=3.1.20191231 + - libev=4.33 + - libexpat=2.5.0 + - libffi=3.4.2 + - libgcc-ng=13.2.0 + - libgdal=3.7.2 + - libgfortran-ng=13.2.0 + - libgfortran5=13.2.0 + - libglib=2.78.0 + - libgomp=13.2.0 + - libiconv=1.17 + - libjpeg-turbo=3.0.0 + - libkml=1.3.0 + - liblapack=3.9.0 + - libnetcdf=4.9.2 + - libnghttp2=1.52.0 + - libnsl=2.0.0 + - libopenblas=0.3.24 + - libpng=1.6.39 + - libpq=16.0 + - libprotobuf=3.20.3 + - librttopo=1.1.0 + - libspatialite=5.1.0 + - libsqlite=3.43.2 + - libssh2=1.11.0 + - libstdcxx-ng=13.2.0 + - libtiff=4.6.0 + - libuuid=2.38.1 + - libwebp-base=1.3.2 + - libxcb=1.15 + - libxml2=2.11.5 + - libzip=1.10.1 + - libzlib=1.2.13 + - libzopfli=1.0.3 + - lz4-c=1.9.4 + - lzo=2.10 + - matplotlib-base=3.8.0 + - matplotlib-inline=0.1.6 + - minizip=4.0.1 + - mock=5.1.0 + - munkres=1.1.4 + - mysql-common=8.0.33 + - mysql-connector-python=8.0.32 + - mysql-libs=8.0.33 + - ncurses=6.4 + - networkx=3.1 + - ninja=1.11.1 + - nspr=4.35 + - nss=3.94 + - numpy=1.23.5 + - openjpeg=2.5.0 + - openssl=3.1.3 + - packaging=23.2 + - parso=0.8.3 + - pcre2=10.40 + - pexpect=4.8.0 + - photutils=1.9.0 + - pickleshare=0.7.5 + - pillow=10.0.1 + - pip=23.2.1 + - pixman=0.42.2 + - pkgutil-resolve-name=1.3.10 + - pluggy=1.3.0 + - poppler=23.10.0 + - poppler-data=0.4.12 + - postgresql=16.0 + - proj=9.3.0 + - prompt-toolkit=3.0.39 + - prompt_toolkit=3.0.39 + - protobuf=3.20.3 + - psutil=5.9.5 + - psycopg2=2.9.7 + - psycopg2-binary=2.9.7 + - pthread-stubs=0.4 + - ptyprocess=0.7.0 + - pure_eval=0.2.2 + - pycparser=2.21 + - pyerfa=2.0.0.3 + - pygments=2.16.1 + - pyparsing=3.1.1 + - pysocks=1.7.1 + - pytest=7.4.2 + - pytest-arraydiff=0.5.0 + - pytest-astropy=0.10.0 + - pytest-astropy-header=0.2.2 + - pytest-cov=4.1.0 + - pytest-doctestplus=1.0.0 + - pytest-filter-subpackage=0.1.2 + - pytest-mock=3.11.1 + - pytest-openfiles=0.5.0 + - pytest-remotedata=0.4.1 + - python=3.10.12 + - python-dateutil=2.8.2 + - python_abi=3.10 + - pytorch=1.12.1 + - pytz=2023.3.post1 + - pywavelets=1.4.1 + - pyyaml=6.0.1 + - rasterio=1.3.8 + - rav1e=0.6.6 + - readline=8.2 + - referencing=0.30.2 + - requests=2.31.0 + - rpds-py=0.10.6 + - scikit-image=0.22.0 + - scikit-learn=1.3.1 + - scipy=1.11.3 + - semantic_version=2.10.0 + - setuptools=68.2.2 + - shapely=2.0.2 + - simplejson=3.19.2 + - six=1.16.0 + - sleef=3.5.1 + - snappy=1.1.10 + - sniffio=1.3.0 + - snuggs=1.4.7 + - sortedcontainers=2.4.0 + - sqlalchemy=2.0.22 + - sqlite=3.43.2 + - stack_data=0.6.2 + - svt-av1=1.7.0 + - tenacity=8.2.3 + - threadpoolctl=3.2.0 + - tifffile=2023.9.26 + - tiledb=2.16.3 + - tk=8.6.13 + - toml=0.10.2 + - tomli=2.0.1 + - tqdm=4.66.1 + - traitlets=5.11.2 + - typing-extensions=4.8.0 + - typing_extensions=4.8.0 + - tzcode=2023c + - tzdata=2023c + - tzlocal=5.1 + - unicodedata2=15.1.0 + - uriparser=0.9.7 + - wcwidth=0.2.8 + - wheel=0.41.2 + - xerces-c=3.2.4 + - xorg-kbproto=1.0.7 + - xorg-libice=1.1.1 + - xorg-libsm=1.2.4 + - xorg-libx11=1.8.7 + - xorg-libxau=1.0.11 + - xorg-libxdmcp=1.1.3 + - xorg-libxext=1.3.4 + - xorg-libxrender=0.9.11 + - xorg-renderproto=0.11.1 + - xorg-xextproto=7.3.0 + - xorg-xproto=7.0.31 + - xz=5.2.6 + - yaml=0.2.5 + - zfp=1.0.0 + - zipp=3.17.0 + - zlib=1.2.13 + - zlib-ng=2.0.7 + - zstd=1.5.5 - pip: - asciitree==0.3.3 - astropy-healpix==1.0.0 @@ -309,6 +305,7 @@ dependencies: - redis==5.0.1 - reproject==0.12.0 - s3transfer==0.7.0 + - sep==1.2.1 - toolz==0.12.0 - urllib3==1.26.17 - vine==1.3.0 From 2c563afeb80d7bc263972f7e3ab3d6a1a14698b4 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Mon, 12 Aug 2024 19:02:22 -0400 Subject: [PATCH 04/14] Fix to readnoise image record --- banzai/tests/data/test_precals.dat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/banzai/tests/data/test_precals.dat b/banzai/tests/data/test_precals.dat index aa5ccb00..dcc2aa71 100644 --- a/banzai/tests/data/test_precals.dat +++ b/banzai/tests/data/test_precals.dat @@ -8,4 +8,4 @@ frameid filename binning mode obstype dateobs site instrument 33347348 tst1m0XX-ep03-20200710-bpm-1.fits.fz '[1, 1]' 1 BPM 2020-07-10 tst ep03 33347349 tst1m0XX-ep04-20200710-bpm-1.fits.fz '[1, 1]' 1 BPM 2020-07-10 tst ep04 69359146 lsc0m409-sq34-20240314-bpm-full_frame.fits.fz '[1, 1]' full_frame BPM 2024-03-14 lsc sq34 -69359157 lsc0m409-sq34-20240314-readnoise-full_frame.fits.fz '[1, 1]' full_frame 2024-03-14 lsc sq34 +69359157 lsc0m409-sq34-20240314-readnoise-full_frame.fits.fz '[1, 1]' full_frame READNOISE 2024-03-14 lsc sq34 From d8aea06a3edc391e38913ef04dab0bab107db493 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Tue, 13 Aug 2024 09:46:30 -0400 Subject: [PATCH 05/14] Attempt to fix the mock observation portal call for the sq data --- ...test_obs_portal_response_lsc_20240723.json | 355 +++++++++--------- 1 file changed, 185 insertions(+), 170 deletions(-) diff --git a/banzai/tests/data/test_obs_portal_response_lsc_20240723.json b/banzai/tests/data/test_obs_portal_response_lsc_20240723.json index 77343778..130e4863 100644 --- a/banzai/tests/data/test_obs_portal_response_lsc_20240723.json +++ b/banzai/tests/data/test_obs_portal_response_lsc_20240723.json @@ -4,359 +4,374 @@ "previous": null, "results": [ { - "id": 631366169, + "created": "2024-07-23T04:05:53.173401Z", + "enclosure": "aqwa", + "end": "2024-07-23T22:36:19.418683Z", + "id": 631156776, + "ipp_value": 1.0, + "modified": "2024-07-23T22:30:04.846999Z", + "name": "LCOGT", + "observation_type": "DIRECT", + "priority": 10, + "proposal": "calibrate", "request": { - "id": 3597084, - "observation_note": "", - "optimization_type": "TIME", - "state": "COMPLETED", "acceptability_threshold": 90.0, "configuration_repeats": 1, - "extra_params": {}, - "modified": "2024-07-24T05:21:36.018338Z", - "duration": 472, "configurations": [ { - "id": 11542373, - "instrument_type": "0M4-SCICAM-QHY600", - "type": "SKY_FLAT", - "repeat_duration": null, + "acquisition_config": { + "extra_params": {}, + "mode": "OFF" + }, + "configuration_status": 812512180, + "constraints": { + "extra_params": {}, + "max_airmass": 20.0, + "max_lunar_phase": 1.0, + "min_lunar_distance": 0.0 + }, "extra_params": {}, - "priority": 3, + "guide_camera_name": "", + "guiding_config": { + "exposure_time": None, + "extra_params": {}, + "mode": "OFF", + "optical_elements": {}, + "optional": True + }, + "id": 11542373, "instrument_configs": [ { - "optical_elements": { - "filter": "B" - }, - "mode": "full_frame", - "exposure_time": 1.0, "exposure_count": 5, - "rotator_mode": "", + "exposure_time": 1.0, "extra_params": { "bin_x": 1, "bin_y": 1 }, - "rois": [] + "mode": "full_frame", + "optical_elements": { + "filter": "B" + }, + "rois": [], + "rotator_mode": "" } ], - "constraints": { - "max_airmass": 20.0, - "min_lunar_distance": 0.0, - "max_lunar_phase": 1.0, - "extra_params": {} - }, - "acquisition_config": { - "mode": "OFF", - "extra_params": {} - }, - "guiding_config": { - "optional": true, - "mode": "OFF", - "optical_elements": {}, - "exposure_time": null, - "extra_params": {} - }, - "target": { - "type": "HOUR_ANGLE", - "name": "No target", - "ra": null, - "dec": 0.0, - "proper_motion_ra": null, - "proper_motion_dec": null, - "parallax": null, - "epoch": null, - "hour_angle": 1.0, - "extra_params": {} - }, - "configuration_status": 812512180, - "state": "COMPLETED", "instrument_name": "sq34", - "guide_camera_name": "", + "instrument_type": "0M4-SCICAM-QHY600", + "priority": 3, + "repeat_duration": None, + "state": "COMPLETED", "summary": { - "id": 4815648, - "start": "2024-07-23T22:11:04Z", "end": "2024-07-23T22:22:19Z", - "state": "COMPLETED", - "reason": "", - "time_completed": 0.0, "events": [ { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.mount.HomeCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.mount.HomeCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.astrometric.EnableTrackingCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.astrometric.EnableTrackingCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.focus.HomeCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.focus.HomeCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.enclosure.OpenCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.enclosure.OpenCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.enclosure.OpenCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.enclosure.OpenCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.focus.HomeCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.focus.HomeCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.astrometric.EnableTrackingCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.EnableTrackingCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.telescopeservices.StowMirrorCoverCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.telescopeservices.StowMirrorCoverCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.mount.HomeCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.mount.HomeCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.focus.DeFocusCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.focus.DeFocusCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.telescopeservices.StowMirrorCoverCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.telescopeservices.StowMirrorCoverCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.focus.DeFocusCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.focus.DeFocusCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:05", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + "time": "2024-07-23T22:11:05" }, { - "time": "2024-07-23T22:11:06", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + "time": "2024-07-23T22:11:06" }, { - "time": "2024-07-23T22:11:06", + "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" + "time": "2024-07-23T22:11:06" }, { - "time": "2024-07-23T22:11:06", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + "time": "2024-07-23T22:11:06" }, { - "time": "2024-07-23T22:11:12", + "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" + "time": "2024-07-23T22:11:12" }, { - "time": "2024-07-23T22:11:12", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:11:12" }, { - "time": "2024-07-23T22:12:03", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:12:03" }, { - "time": "2024-07-23T22:12:34", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:12:34" }, { - "time": "2024-07-23T22:12:47", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:12:47" }, { - "time": "2024-07-23T22:12:57", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:12:57" }, { - "time": "2024-07-23T22:13:20", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:13:20" }, { - "time": "2024-07-23T22:13:30", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:13:30" }, { - "time": "2024-07-23T22:14:23", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:14:23" }, { - "time": "2024-07-23T22:21:02", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + "time": "2024-07-23T22:21:02" }, { - "time": "2024-07-23T22:21:03", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + "time": "2024-07-23T22:21:03" }, { - "time": "2024-07-23T22:21:03", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:21:03" }, { - "time": "2024-07-23T22:21:04", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + "time": "2024-07-23T22:21:04" }, { - "time": "2024-07-23T22:21:13", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + "time": "2024-07-23T22:21:13" }, { - "time": "2024-07-23T22:21:13", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + "time": "2024-07-23T22:21:13" }, { - "time": "2024-07-23T22:21:19", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:21:19" }, { - "time": "2024-07-23T22:21:19", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + "time": "2024-07-23T22:21:19" }, { - "time": "2024-07-23T22:21:19", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + "time": "2024-07-23T22:21:19" }, { - "time": "2024-07-23T22:21:29", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + "time": "2024-07-23T22:21:29" }, { - "time": "2024-07-23T22:21:29", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + "time": "2024-07-23T22:21:29" }, { - "time": "2024-07-23T22:21:35", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + "time": "2024-07-23T22:21:35" }, { - "time": "2024-07-23T22:21:35", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:21:35" }, { - "time": "2024-07-23T22:21:36", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + "time": "2024-07-23T22:21:36" }, { - "time": "2024-07-23T22:21:45", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + "time": "2024-07-23T22:21:45" }, { - "time": "2024-07-23T22:21:45", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + "time": "2024-07-23T22:21:45" }, { - "time": "2024-07-23T22:21:51", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + "time": "2024-07-23T22:21:51" }, { - "time": "2024-07-23T22:21:51", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:21:51" }, { - "time": "2024-07-23T22:21:53", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + "time": "2024-07-23T22:21:53" }, { - "time": "2024-07-23T22:22:01", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + "time": "2024-07-23T22:22:01" }, { - "time": "2024-07-23T22:22:01", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + "time": "2024-07-23T22:22:01" }, { - "time": "2024-07-23T22:22:08", + "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.OffsetCommand" + "time": "2024-07-23T22:22:08" }, { - "time": "2024-07-23T22:22:08", + "description": "org.lcogt.sequencer.command.astrometric.FlatCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.astrometric.FlatCommand" + "time": "2024-07-23T22:22:08" }, { - "time": "2024-07-23T22:22:08", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + "time": "2024-07-23T22:22:08" }, { - "time": "2024-07-23T22:22:17", + "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" + "time": "2024-07-23T22:22:17" }, { - "time": "2024-07-23T22:22:18", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand", "state": "PROCESSING", - "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + "time": "2024-07-23T22:22:18" }, { - "time": "2024-07-23T22:22:19", + "description": "org.lcogt.sequencer.command.instrument.AbortCommand", "state": "DONE", - "description": "org.lcogt.sequencer.command.instrument.AbortCommand" + "time": "2024-07-23T22:22:19" } - ] - } + ], + "id": 4815648, + "reason": "", + "start": "2024-07-23T22:11:04Z", + "state": "COMPLETED", + "time_completed": 0.0 + }, + "target": { + "dec": 0.0, + "epoch": None, + "extra_params": {}, + "hour_angle": 1.0, + "name": "No target", + "parallax": None, + "proper_motion_dec": None, + "proper_motion_ra": None, + "ra": None, + "type": "HOUR_ANGLE" + }, + "type": "SKY_FLAT" } - ] - } + ], + "duration": 497, + "extra_params": {}, + "id": 3596511, + "modified": "2024-07-23T22:30:04.801605Z", + "observation_note": "", + "optimization_type": "TIME", + "state": "COMPLETED" + }, + "request_group_id": 2011773, + "site": "lsc", + "start": "2024-07-23T22:04:10.911637Z", + "state": "COMPLETED", + "submitter": "eng", + "telescope": "0m4a" }, -{ + { "id": 631577495, "request": { "id": 3597581, From 305638c28d56319c78373ea29c3b94e2fd646590 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 14 Aug 2024 14:03:18 -0400 Subject: [PATCH 06/14] Fixes to the deployment of the e2e tests. --- banzai/tests/e2e-k8s.yaml | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 9fb20c50..4fa0effb 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -27,11 +27,9 @@ spec: readinessProbe: exec: command: - - redis-cli - - ping - - | - - grep - - "PONG" + - /bin/sh + - -c + - 'redis-cli ping | grep -q "PONG"' initialDelaySeconds: 5 periodSeconds: 1 - name: banzai-fits-exchange @@ -63,7 +61,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db" + value: "sqlite:////archive/engineering/test.db?timeout=10" - name: RETRY_DELAY value: "0" - name: TASK_HOST @@ -105,13 +103,9 @@ spec: readinessProbe: exec: command: - - celery - - -A - - banzai - - status - - | - - grep - - "banzai-celery-worker:.*OK" + - /bin/sh + - -c + - 'celery -A banzai status | grep -q @banzai-celery-worker:.*OK"' initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 @@ -132,7 +126,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db" + value: "sqlite:////archive/engineering/test.db?timeout=10" - name: RETRY_DELAY value: "0" - name: TASK_HOST @@ -173,13 +167,9 @@ spec: readinessProbe: exec: command: - - celery - - -A - - banzai - - status - - | - - grep - - "banzai-celery-worker:.*OK" + - /bin/sh + - -c + - 'celery -A banzai status | grep -q "large-celery-worker:.*OK"' initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 @@ -200,7 +190,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db" + value: "sqlite:////archive/engineering/test.db?timeout=10" - name: FITS_BROKER value: "localhost" - name: TASK_HOST @@ -223,7 +213,7 @@ spec: value: "e2e_large_task_queue" command: - banzai_run_realtime_pipeline - - "--db-address=sqlite:////archive/engineering/test.db" + - "--db-address='sqlite:////archive/engineering/test.db?timeout=10'" - "--fpack" - "--broker-url=localhost" resources: From 05d66b60a670da8cb4607507686f72a20ccbc13f Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 14 Aug 2024 14:11:02 -0400 Subject: [PATCH 07/14] Typo fix --- banzai/tests/e2e-k8s.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 4fa0effb..d8c9d87a 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -99,13 +99,12 @@ spec: - "$(CELERY_TASK_QUEUE_NAME)" - "-n" - "celery-worker" - readinessProbe: exec: command: - /bin/sh - -c - - 'celery -A banzai status | grep -q @banzai-celery-worker:.*OK"' + - 'celery -A banzai status | grep -q "@banzai-celery-worker:.*OK"' initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 From 7ae8bc257792a8ada0e746d120831f1660c15b01 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 14 Aug 2024 17:35:33 -0400 Subject: [PATCH 08/14] More fixes to e2e deployment. --- ...test_obs_portal_response_lsc_20240723.json | 2 +- banzai/tests/e2e-k8s.yaml | 22 +++++++++++++------ environment.yaml | 4 ++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/banzai/tests/data/test_obs_portal_response_lsc_20240723.json b/banzai/tests/data/test_obs_portal_response_lsc_20240723.json index 130e4863..cceaeb0e 100644 --- a/banzai/tests/data/test_obs_portal_response_lsc_20240723.json +++ b/banzai/tests/data/test_obs_portal_response_lsc_20240723.json @@ -617,4 +617,4 @@ "modified": "2024-07-24T21:21:09.532093Z" } ] -} \ No newline at end of file +} diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index d8c9d87a..a9ed5145 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -20,7 +20,7 @@ spec: resources: requests: cpu: 0.1 - memory: 256Mi + memory: 512Mi limits: cpu: 1 memory: 512Mi @@ -104,14 +104,14 @@ spec: command: - /bin/sh - -c - - 'celery -A banzai status | grep -q "@banzai-celery-worker:.*OK"' + - "celery -A banzai status | grep -q '@celery-worker:.*OK'" initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 resources: requests: cpu: 4 - memory: 4Gi + memory: 5Gi limits: cpu: 8 memory: 5Gi @@ -175,10 +175,10 @@ spec: resources: requests: cpu: 2 - memory: 8Gi + memory: 5Gi limits: cpu: 3 - memory: 9Gi + memory: 5Gi - name: banzai-listener image: banzai:test-latest imagePullPolicy: IfNotPresent @@ -221,7 +221,15 @@ spec: memory: 1Gi limits: cpu: 1 - memory: 2Gi - + memory: 1Gi + readinessProbe: + exec: + command: + - /bin/sh + - -c + - 'ps -u archive | grep -q "banzai_run_real"' + initialDelaySeconds: 5 + periodSeconds: 1 + timeoutSeconds: 10 dnsPolicy: ClusterFirst restartPolicy: Never diff --git a/environment.yaml b/environment.yaml index fceea57e..fc2e7d12 100644 --- a/environment.yaml +++ b/environment.yaml @@ -183,7 +183,7 @@ dependencies: - pip=23.2.1 - pixman=0.42.2 - pkgutil-resolve-name=1.3.10 - - pluggy=1.3.0 + - pluggy=1.5.0 - poppler=23.10.0 - poppler-data=0.4.12 - postgresql=16.0 @@ -202,7 +202,7 @@ dependencies: - pygments=2.16.1 - pyparsing=3.1.1 - pysocks=1.7.1 - - pytest=7.4.2 + - pytest=8.3.2 - pytest-arraydiff=0.5.0 - pytest-astropy=0.10.0 - pytest-astropy-header=0.2.2 From f2ef342e941d3b0460faf7be5115772d6ea9cafa Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 16 Aug 2024 08:38:03 -0400 Subject: [PATCH 09/14] More fixes to e2e tests. --- .github/workflows/e2e.yaml | 8 +- Dockerfile | 2 + banzai/settings.py | 2 +- banzai/tests/data/test_data.dat | 4 +- ...test_obs_portal_response_lsc_20240723.json | 84 +++++++++---------- banzai/tests/e2e-k8s.yaml | 18 ++-- environment.yaml | 3 +- pytest.ini | 2 + 8 files changed, 64 insertions(+), 59 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3fc9086c..443da05b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -45,19 +45,19 @@ jobs: - name: Test Super Bias Creation run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias - name: Test Super Dark Creation run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark - name: Test Super Flat Creation run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat + kubectl exec banzai-e2e-test -c banzai-listener -- pytest-s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat - name: Test Science Frame Creation run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files - name: Cleanup run: | diff --git a/Dockerfile b/Dockerfile index 2f7d5146..f9318952 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,3 +24,5 @@ COPY --chown=10087:10000 . /lco/banzai ENV PATH /home/archive/envs/banzai/bin:$PATH RUN /home/archive/envs/banzai/bin/pip install --no-cache-dir /lco/banzai/ + +RUN cp /lco/banzai/pytest.ini /home/archive/pytest.ini diff --git a/banzai/settings.py b/banzai/settings.py index 9c511253..b5dd54e8 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -80,7 +80,7 @@ 'elp': {'minute': 0, 'hour': 23}, 'ogg': {'minute': 0, 'hour': 3}} -ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', ' ') +ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', 'http://astrometry.lco.gtn/') CALIBRATION_FILENAME_FUNCTIONS = {'BIAS': ('banzai.utils.file_utils.config_to_filename', 'banzai.utils.file_utils.ccdsum_to_filename'), diff --git a/banzai/tests/data/test_data.dat b/banzai/tests/data/test_data.dat index 91a6b880..42c74f5d 100644 --- a/banzai/tests/data/test_data.dat +++ b/banzai/tests/data/test_data.dat @@ -304,13 +304,13 @@ frameid filename filter site instrument 33347343 tst1m0XX-ep04-20200324-0018-d00.fits.fz ip*diffuser tst ep04 33347344 tst1m0XX-ep04-20200324-0019-d00.fits.fz ip*diffuser tst ep04 33347345 tst1m0XX-ep04-20200324-0020-d00.fits.fz ip*diffuser tst ep04 -73414350 lsc0m476-sq34-20240723-0301-e00.fits.fz B lsc sq34 +73890113 lsc0m476-sq34-20240809-0144-e00.fits.fz B lsc sq34 73422225 lsc0m476-sq34-20240723-0408-x00.fits.fz B lsc sq34 73409803 lsc0m476-sq34-20240723-0057-f00.fits.fz B lsc sq34 73409794 lsc0m476-sq34-20240723-0056-f00.fits.fz B lsc sq34 73409788 lsc0m476-sq34-20240723-0055-f00.fits.fz B lsc sq34 73409785 lsc0m476-sq34-20240723-0054-f00.fits.fz B lsc sq34 -73215904 lsc0m476-sq34-20240723-0053-f00.fits.fz B lsc sq34 +73409786 lsc0m476-sq34-20240723-0053-f00.fits.fz B lsc sq34 73429574 lsc0m476-sq34-20240723-0548-d00.fits.fz opaque lsc sq34 73429727 lsc0m476-sq34-20240723-0549-d00.fits.fz opaque lsc sq34 73429774 lsc0m476-sq34-20240723-0550-d00.fits.fz opaque lsc sq34 diff --git a/banzai/tests/data/test_obs_portal_response_lsc_20240723.json b/banzai/tests/data/test_obs_portal_response_lsc_20240723.json index cceaeb0e..ada747a8 100644 --- a/banzai/tests/data/test_obs_portal_response_lsc_20240723.json +++ b/banzai/tests/data/test_obs_portal_response_lsc_20240723.json @@ -33,11 +33,11 @@ "extra_params": {}, "guide_camera_name": "", "guiding_config": { - "exposure_time": None, + "exposure_time": null, "extra_params": {}, "mode": "OFF", "optical_elements": {}, - "optional": True + "optional": true }, "id": 11542373, "instrument_configs": [ @@ -59,7 +59,7 @@ "instrument_name": "sq34", "instrument_type": "0M4-SCICAM-QHY600", "priority": 3, - "repeat_duration": None, + "repeat_duration": null, "state": "COMPLETED", "summary": { "end": "2024-07-23T22:22:19Z", @@ -343,14 +343,14 @@ }, "target": { "dec": 0.0, - "epoch": None, + "epoch": null, "extra_params": {}, "hour_angle": 1.0, "name": "No target", - "parallax": None, - "proper_motion_dec": None, - "proper_motion_ra": None, - "ra": None, + "parallax": null, + "proper_motion_dec": null, + "proper_motion_ra": null, + "ra": null, "type": "HOUR_ANGLE" }, "type": "SKY_FLAT" @@ -372,20 +372,20 @@ "telescope": "0m4a" }, { - "id": 631577495, + "id": 631287808, "request": { - "id": 3597581, + "id": 3596764, "observation_note": "", "optimization_type": "TIME", "state": "COMPLETED", "acceptability_threshold": 90.0, "configuration_repeats": 1, "extra_params": {}, - "modified": "2024-07-24T21:21:09.473360Z", + "modified": "2024-07-23T21:20:17.110554Z", "duration": 3427, "configurations": [ { - "id": 11549305, + "id": 11543917, "instrument_type": "0M4-SCICAM-QHY600", "type": "BIAS", "repeat_duration": null, @@ -436,55 +436,55 @@ "hour_angle": 1.0, "extra_params": {} }, - "configuration_status": 813028208, + "configuration_status": 812665084, "state": "COMPLETED", "instrument_name": "sq34", "guide_camera_name": "", "summary": { - "id": 4818287, - "start": "2024-07-24T20:52:40Z", - "end": "2024-07-24T20:55:16Z", + "id": 4815555, + "start": "2024-07-23T20:52:00Z", + "end": "2024-07-23T20:54:31Z", "state": "COMPLETED", "reason": "", "time_completed": 0.0, "events": [ { - "time": "2024-07-24T20:52:40", + "time": "2024-07-23T20:52:00", "state": "PROCESSING", "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" }, { - "time": "2024-07-24T20:52:41", + "time": "2024-07-23T20:52:01", "state": "DONE", "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" }, { - "time": "2024-07-24T20:52:41", + "time": "2024-07-23T20:52:01", "state": "PROCESSING", "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" }, { - "time": "2024-07-24T20:52:42", + "time": "2024-07-23T20:52:03", "state": "DONE", "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" }, { - "time": "2024-07-24T20:52:43", + "time": "2024-07-23T20:52:03", "state": "PROCESSING", "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" }, { - "time": "2024-07-24T20:55:14", + "time": "2024-07-23T20:54:30", "state": "DONE", "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" }, { - "time": "2024-07-24T20:55:15", + "time": "2024-07-23T20:54:30", "state": "PROCESSING", "description": "org.lcogt.sequencer.command.instrument.AbortCommand" }, { - "time": "2024-07-24T20:55:16", + "time": "2024-07-23T20:54:31", "state": "DONE", "description": "org.lcogt.sequencer.command.instrument.AbortCommand" } @@ -492,7 +492,7 @@ } }, { - "id": 11549306, + "id": 11543918, "instrument_type": "0M4-SCICAM-QHY600", "type": "DARK", "repeat_duration": null, @@ -543,55 +543,55 @@ "hour_angle": 1.0, "extra_params": {} }, - "configuration_status": 813028209, + "configuration_status": 812665085, "state": "COMPLETED", "instrument_name": "sq34", "guide_camera_name": "", "summary": { - "id": 4818307, - "start": "2024-07-24T20:55:19Z", - "end": "2024-07-24T21:21:09Z", + "id": 4815574, + "start": "2024-07-23T20:54:34Z", + "end": "2024-07-23T21:20:16Z", "state": "COMPLETED", "reason": "", "time_completed": 1500.0, "events": [ { - "time": "2024-07-24T20:55:19", + "time": "2024-07-23T20:54:34", "state": "PROCESSING", "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" }, { - "time": "2024-07-24T20:55:20", + "time": "2024-07-23T20:54:34", "state": "DONE", "description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand" }, { - "time": "2024-07-24T20:55:20", + "time": "2024-07-23T20:54:35", "state": "PROCESSING", "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" }, { - "time": "2024-07-24T20:55:22", + "time": "2024-07-23T20:54:36", "state": "DONE", "description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand" }, { - "time": "2024-07-24T20:55:23", + "time": "2024-07-23T20:54:37", "state": "PROCESSING", "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" }, { - "time": "2024-07-24T21:21:07", + "time": "2024-07-23T21:20:14", "state": "DONE", "description": "org.lcogt.sequencer.command.instrument.StartExposureCommand" }, { - "time": "2024-07-24T21:21:07", + "time": "2024-07-23T21:20:15", "state": "PROCESSING", "description": "org.lcogt.sequencer.command.instrument.AbortCommand" }, { - "time": "2024-07-24T21:21:08", + "time": "2024-07-23T21:20:16", "state": "DONE", "description": "org.lcogt.sequencer.command.instrument.AbortCommand" } @@ -603,8 +603,8 @@ "site": "lsc", "enclosure": "aqwa", "telescope": "0m4a", - "start": "2024-07-24T20:26:04.169574Z", - "end": "2024-07-24T21:32:04.169574Z", + "start": "2024-07-23T20:25:25.334568Z", + "end": "2024-07-23T21:31:25.334568Z", "priority": 10, "state": "COMPLETED", "proposal": "calibrate", @@ -612,9 +612,9 @@ "name": "LCOGT", "ipp_value": 1.0, "observation_type": "DIRECT", - "request_group_id": 2012784, - "created": "2024-07-24T15:05:49.265713Z", - "modified": "2024-07-24T21:21:09.532093Z" + "request_group_id": 2012021, + "created": "2024-07-23T15:05:56.175376Z", + "modified": "2024-07-23T21:20:17.151237Z" } ] } diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index a9ed5145..46c2c858 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -61,7 +61,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db?timeout=10" + value: "sqlite:////archive/engineering/test.db?timeout=15" - name: RETRY_DELAY value: "0" - name: TASK_HOST @@ -75,7 +75,7 @@ spec: - name: API_ROOT value: "https://archive-api.lco.global/" - name: OMP_NUM_THREADS - value: "2" + value: "1" - name: FITS_EXCHANGE value: "fits_files" - name: OPENTSDB_PYTHON_METRICS_TEST_MODE @@ -125,7 +125,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db?timeout=10" + value: "sqlite:////archive/engineering/test.db?timeout=15" - name: RETRY_DELAY value: "0" - name: TASK_HOST @@ -175,10 +175,10 @@ spec: resources: requests: cpu: 2 - memory: 5Gi + memory: 7Gi limits: cpu: 3 - memory: 5Gi + memory: 7Gi - name: banzai-listener image: banzai:test-latest imagePullPolicy: IfNotPresent @@ -189,7 +189,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db?timeout=10" + value: "sqlite:////archive/engineering/test.db?timeout=15" - name: FITS_BROKER value: "localhost" - name: TASK_HOST @@ -212,9 +212,9 @@ spec: value: "e2e_large_task_queue" command: - banzai_run_realtime_pipeline - - "--db-address='sqlite:////archive/engineering/test.db?timeout=10'" - - "--fpack" - - "--broker-url=localhost" + - --db-address=$(DB_ADDRESS) + - --fpack + - --broker-url=localhost resources: requests: cpu: 0.1 diff --git a/environment.yaml b/environment.yaml index fc2e7d12..4fa41269 100644 --- a/environment.yaml +++ b/environment.yaml @@ -202,7 +202,7 @@ dependencies: - pygments=2.16.1 - pyparsing=3.1.1 - pysocks=1.7.1 - - pytest=8.3.2 + - pytest=8.0.0 - pytest-arraydiff=0.5.0 - pytest-astropy=0.10.0 - pytest-astropy-header=0.2.2 @@ -288,6 +288,7 @@ dependencies: - cloudpickle==2.2.1 - cosmic-conn==0.4.1 - dask==2023.9.3 + - extenion_helpers==1.1.1 - fasteners==0.19 - fsspec==2023.9.2 - kombu==4.6.11 diff --git a/pytest.ini b/pytest.ini index b30fc1bd..e68b30c2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -26,6 +26,7 @@ markers = dark_comparer dark_maker dark_normalizer + dark_subtractor date_utils dbs fits_utils @@ -48,6 +49,7 @@ markers = pattern_noise_qc pointing quick_select + read_noise runtime_context saturation_qc saving_qc From 856c8ba0e9e8de0059b63dae52700c8aa39f51d4 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 16 Aug 2024 08:46:06 -0400 Subject: [PATCH 10/14] Typo fix --- environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yaml b/environment.yaml index 4fa41269..d569e58a 100644 --- a/environment.yaml +++ b/environment.yaml @@ -288,7 +288,7 @@ dependencies: - cloudpickle==2.2.1 - cosmic-conn==0.4.1 - dask==2023.9.3 - - extenion_helpers==1.1.1 + - extension_helpers==1.1.1 - fasteners==0.19 - fsspec==2023.9.2 - kombu==4.6.11 From 4f3776abe498cd3b3e7d6f9dfdd280558a30d699 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 16 Aug 2024 09:56:10 -0400 Subject: [PATCH 11/14] Fixes to e2e tests for science frames. --- banzai/settings.py | 2 +- banzai/tests/test_end_to_end.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/banzai/settings.py b/banzai/settings.py index b5dd54e8..7123c5b9 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -80,7 +80,7 @@ 'elp': {'minute': 0, 'hour': 23}, 'ogg': {'minute': 0, 'hour': 3}} -ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', 'http://astrometry.lco.gtn/') +ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', 'http://astrometry.lco.gtn/catalog') CALIBRATION_FILENAME_FUNCTIONS = {'BIAS': ('banzai.utils.file_utils.config_to_filename', 'banzai.utils.file_utils.ccdsum_to_filename'), diff --git a/banzai/tests/test_end_to_end.py b/banzai/tests/test_end_to_end.py index 63e9cbf6..3bd0e52a 100644 --- a/banzai/tests/test_end_to_end.py +++ b/banzai/tests/test_end_to_end.py @@ -256,10 +256,12 @@ def test_if_science_frames_were_created(self): expected_files = [] created_files = [] for day_obs in DAYS_OBS: - expected_files += [filename.replace('e00', 'e91') - for filename in TEST_FRAMES['filename'] if 'e00.fits' in filename] - created_files += [os.path.basename(filename) for filename in glob(os.path.join(DATA_ROOT, day_obs, - 'processed', '*e91*'))] + for extension in ['e00', 'x00']: + expected_files += [filename.replace(extension, extension.replace('00', '91')) + for filename in TEST_FRAMES['filename'] if f'{extension}.fits' in filename] + created_files += [os.path.basename(filename) + for filename in glob(os.path.join(DATA_ROOT, day_obs, 'processed', + f'*{extension.replace("00", "91")}*'))] assert len(expected_files) > 0 for expected_file in expected_files: assert expected_file in created_files @@ -267,8 +269,9 @@ def test_if_science_frames_were_created(self): def test_that_photometric_calibration_succeeded(self): science_files = [] for day_obs in DAYS_OBS: - science_files += [filepath for filepath in glob(os.path.join(DATA_ROOT, day_obs, - 'processed', '*e91*'))] + for extension in ['e91', 'x91']: + science_files += [filepath for filepath in + glob(os.path.join(DATA_ROOT, day_obs, 'processed', f'*{extension}*'))] zeropoints = [fits.open(filename)['SCI'].header.get('L1ZP') for filename in science_files] # check that at least one of our images contains a zeropoint assert zeropoints.count(None) != len(zeropoints) From 5c679baa6d08ebcb70f67dbda4fc1f3bea50f2a6 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 16 Aug 2024 12:07:17 -0400 Subject: [PATCH 12/14] More fixes to e2e tests. --- banzai/tests/e2e-k8s.yaml | 6 +++--- banzai/tests/test_end_to_end.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 46c2c858..e27b2106 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -61,7 +61,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db?timeout=15" + value: "sqlite:////archive/engineering/test.db?timeout=30" - name: RETRY_DELAY value: "0" - name: TASK_HOST @@ -125,7 +125,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db?timeout=15" + value: "sqlite:////archive/engineering/test.db?timeout=30" - name: RETRY_DELAY value: "0" - name: TASK_HOST @@ -189,7 +189,7 @@ spec: readOnly: false env: - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db?timeout=15" + value: "sqlite:////archive/engineering/test.db?timeout=30" - name: FITS_BROKER value: "localhost" - name: TASK_HOST diff --git a/banzai/tests/test_end_to_end.py b/banzai/tests/test_end_to_end.py index 3bd0e52a..afd7a9d2 100644 --- a/banzai/tests/test_end_to_end.py +++ b/banzai/tests/test_end_to_end.py @@ -251,6 +251,7 @@ class TestScienceFileCreation: @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect) def reduce_science_frames(self, mock_observation_portal): run_reduce_individual_frames('e00.fits') + run_reduce_individual_frames('x00.fits') def test_if_science_frames_were_created(self): expected_files = [] From 38da0e0614f70410edd49cd7b0e03eebe96857f4 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 16 Aug 2024 13:49:01 -0400 Subject: [PATCH 13/14] Fixes to unit tests. --- .github/workflows/e2e.yaml | 2 +- banzai/tests/utils.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 443da05b..4ea2d474 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -53,7 +53,7 @@ jobs: - name: Test Super Flat Creation run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest-s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat - name: Test Science Frame Creation run: | diff --git a/banzai/tests/utils.py b/banzai/tests/utils.py index afd3aa61..2a48aa95 100644 --- a/banzai/tests/utils.py +++ b/banzai/tests/utils.py @@ -69,7 +69,10 @@ def __init__(self, hdu_list=None, file_path='/tmp/test_image.fits', instrument=N self._file_path = file_path self.is_bad = False self.hdu_order = ['SCI', 'CAT', 'BPM', 'ERR'] - self.n_sub_exposures = 1 + + @property + def n_sub_exposures(self): + return 1 for keyword in kwargs: setattr(self, keyword, kwargs[keyword]) From ba54178dc48defbd302559e8a53e33e3efbc1aa6 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Tue, 27 Aug 2024 17:01:13 -0400 Subject: [PATCH 14/14] Fixes based on review comments. --- .github/workflows/unit-tests.yml | 5 ---- banzai/bias.py | 4 ++++ banzai/data.py | 20 ++++++++++++---- banzai/frames.py | 4 ++++ banzai/readnoise.py | 2 +- banzai/tests/test_bias_level_subtractor.py | 13 ++++++++++ banzai/tests/test_bias_maker.py | 28 +++++++++++++++++++++- banzai/tests/test_bias_subtractor.py | 24 +++++++++++++++++-- banzai/tests/test_end_to_end.py | 2 +- banzai/tests/test_frames.py | 8 +++++-- banzai/tests/utils.py | 7 ++---- 11 files changed, 95 insertions(+), 22 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3c8e4e77..02bce075 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -18,11 +18,6 @@ jobs: python-version: 3.9 toxenv: build_docs - - name: Python 3.9 with developer version of astropy and numpy - os: ubuntu-latest - python-version: 3.9 - toxenv: py39-test-devdeps - - name: Python 3.8 with minimal dependencies os: ubuntu-latest python-version: '3.8' diff --git a/banzai/bias.py b/banzai/bias.py index 316f166f..c49e5330 100755 --- a/banzai/bias.py +++ b/banzai/bias.py @@ -17,6 +17,8 @@ def calibration_type(self): return 'BIAS' def make_master_calibration_frame(self, images): + for image in images: + image /= image.n_sub_exposures master_image = super(BiasMaker, self).make_master_calibration_frame(images) master_image.bias_level = np.mean([image.bias_level for image in images if image.bias_level is not None]) return master_image @@ -65,6 +67,8 @@ def __init__(self, runtime_context): def do_stage(self, image): bias_level = stats.sigma_clipped_mean(image.data, 3.5, mask=image.mask) image -= bias_level + # This is only run for bias frames and n_sub_exposures should always be 1 + # but we divide it here for consistency to cover pathological uses cases image.meta['BIASLVL'] = bias_level / image.n_sub_exposures, 'Bias level that was removed after overscan' return image diff --git a/banzai/data.py b/banzai/data.py index 31ed60e6..19f76c3d 100644 --- a/banzai/data.py +++ b/banzai/data.py @@ -132,7 +132,8 @@ def __init__(self, data: Union[np.array, Table], meta: fits.Header, mask: np.array = None, name: str = '', uncertainty: np.array = None, memmap=True): super().__init__(data=data, meta=meta, mask=mask, name=name, memmap=memmap) if uncertainty is None: - uncertainty = self.read_noise * self.n_sub_exposures * np.ones(data.shape, dtype=data.dtype) / self.gain + uncertainty = self.read_noise * np.sqrt(self.n_sub_exposures) * np.ones(data.shape, dtype=data.dtype) + uncertainty /= self.gain self.uncertainty = self._init_array(uncertainty) self._detector_section = Section.parse_region_keyword(self.meta.get('DETSEC')) self._data_section = Section.parse_region_keyword(self.meta.get('DATASEC')) @@ -185,7 +186,7 @@ def to_fits(self, context): def __del__(self): super().__del__() - del self.uncertainty + del self._uncertainty def __isub__(self, value): if isinstance(value, CCDData): @@ -201,9 +202,14 @@ def __sub__(self, other): return type(self)(data=self.data - other.data, meta=self.meta, mask=self.mask | other.mask, uncertainty=uncertainty) - def add_uncertainty(self, readnoise: np.array): - self._validate_array(readnoise) - self.uncertainty = self._init_array(readnoise) + @property + def uncertainty(self): + return self._uncertainty + + @uncertainty.setter + def uncertainty(self, value: np.array): + self._validate_array(value) + self._uncertainty = self._init_array(value) def signal_to_noise(self): return np.abs(self.data) / self.uncertainty @@ -318,6 +324,10 @@ def n_sub_exposures(self): n_exposures = 1 return n_exposures + @n_sub_exposures.setter + def n_sub_exposures(self, value): + self.meta['NSUBREAD'] = value + def rebin(self, binning): # TODO: Implement me return self diff --git a/banzai/frames.py b/banzai/frames.py index 3c77c947..47543aa5 100644 --- a/banzai/frames.py +++ b/banzai/frames.py @@ -82,6 +82,10 @@ def background(self, value): def n_sub_exposures(self): return self.primary_hdu.n_sub_exposures + @n_sub_exposures.setter + def n_sub_exposures(self, value): + self.primary_hdu.n_sub_exposures = value + @abc.abstractmethod def save_processing_metadata(self, context): pass diff --git a/banzai/readnoise.py b/banzai/readnoise.py index 08950775..00acd440 100644 --- a/banzai/readnoise.py +++ b/banzai/readnoise.py @@ -9,7 +9,7 @@ class ReadNoiseLoader(CalibrationUser): def apply_master_calibration(self, image, master_calibration_image): try: for image_extension, readnoise_extension in zip(image.ccd_hdus, master_calibration_image.ccd_hdus): - image_extension.add_uncertainty(readnoise_extension.data * np.sqrt(image_extension.n_sub_exposures)) + image_extension.uncertainty = readnoise_extension.data * np.sqrt(image_extension.n_sub_exposures) except: logger.error(f"Can't add READNOISE to image, stopping reduction: {format_exception()}", image=image) return None diff --git a/banzai/tests/test_bias_level_subtractor.py b/banzai/tests/test_bias_level_subtractor.py index bdc8d37f..734e4bc7 100644 --- a/banzai/tests/test_bias_level_subtractor.py +++ b/banzai/tests/test_bias_level_subtractor.py @@ -52,3 +52,16 @@ def test_bias_master_level_subtraction_is_reasonable(set_random_seed): np.testing.assert_allclose(np.zeros(image.data.shape), image.data, atol=8 * read_noise) np.testing.assert_allclose(image.meta.get('BIASLVL'), input_bias, atol=1.0) + + +def test_multiread_bias_level(): + input_bias = 500.0 + nx, ny = 101, 103 + nreads = 5 + subtractor = BiasMasterLevelSubtractor(None) + image = FakeLCOObservationFrame(hdu_list=[FakeCCDData(image_multiplier=input_bias*nreads, nx=nx, ny=ny)]) + image.n_sub_exposures = nreads + image = subtractor.do_stage(image) + + np.testing.assert_allclose(np.zeros(image.data.shape), image.data, atol=1.0) + np.testing.assert_allclose(image.meta.get('BIASLVL'), input_bias, atol=1.0) diff --git a/banzai/tests/test_bias_maker.py b/banzai/tests/test_bias_maker.py index bcbb5bd1..4b677f0a 100644 --- a/banzai/tests/test_bias_maker.py +++ b/banzai/tests/test_bias_maker.py @@ -71,7 +71,7 @@ def test_makes_a_sensible_master_bias(mock_namer): images = [FakeLCOObservationFrame(hdu_list=[FakeCCDData(data=np.random.normal(0.0, size=(99, 99), scale=expected_readnoise), bias_level=0.0, - read_noise= expected_readnoise, + read_noise=expected_readnoise, meta=Header({'DATE-OBS': '2019-12-04T14:34:00', 'DETSEC': '[1:100,1:100]', 'DATASEC': '[1:100,1:100]', @@ -84,3 +84,29 @@ def test_makes_a_sensible_master_bias(mock_namer): assert np.abs(np.mean(master_bias)) < 0.1 actual_readnoise = np.std(master_bias) assert np.abs(actual_readnoise - expected_readnoise / (nimages ** 0.5)) < 0.2 + + +def test_multiread_bias_maker(): + nimages = 5 + nreads = 20 + images = [] + pattern_scale = 8 + nx = 101 + ny = 105 + bias_level = 100.0 + bias_pattern = np.random.normal(0.0, pattern_scale, size=(ny, nx)) + for i in range(nimages): + data = nreads * bias_pattern + image = FakeLCOObservationFrame(hdu_list=[FakeCCDData(data=data, bias_level=bias_level, + meta=Header({'DATE-OBS': '2019-12-04T14:34:00', + 'DETSEC': f'[1:{nx},1:{ny}]', + 'DATASEC': f'[1:{nx},1:{ny}]', + 'OBSTYPE': 'BIAS', 'RA': 0.0, 'DEC': 0.0}))]) + image.n_sub_exposures = nreads + images.append(image) + maker = BiasMaker(FakeContext()) + stacked_image = maker.do_stage(images) + np.testing.assert_allclose(stacked_image.meta['BIASLVL'], bias_level, atol=0.1) + + # With 20 x 5 reads, we should get down to better than 1 count at a read_noise of 9 + np.testing.assert_allclose(stacked_image.data, bias_pattern, atol=1.0) diff --git a/banzai/tests/test_bias_subtractor.py b/banzai/tests/test_bias_subtractor.py index b81fe5da..4196dc7e 100644 --- a/banzai/tests/test_bias_subtractor.py +++ b/banzai/tests/test_bias_subtractor.py @@ -69,10 +69,30 @@ def test_bias_subtraction_is_reasonable(mock_master_cal_name, mock_master_frame) input_level = 2000.0 nx = 101 ny = 103 - mock_master_frame.return_value = FakeLCOObservationFrame(hdu_list=[FakeCCDData(bias_level=input_bias, - data=np.random.normal(0.0, input_readnoise, size=(ny, nx)))]) + stacked_hdu = FakeCCDData(bias_level=input_bias, data=np.random.normal(0.0, input_readnoise, size=(ny, nx))) + mock_master_frame.return_value = FakeLCOObservationFrame(hdu_list=[stacked_hdu]) subtractor = BiasSubtractor(FakeContext()) image = FakeLCOObservationFrame(hdu_list=[FakeCCDData(image_multiplier=input_level)]) image = subtractor.do_stage(image) assert np.abs(image.meta.get('BIASLVL') - input_bias) < 1.0 assert np.abs(np.mean(image.data) - (input_level - input_bias)) < 1.0 + + +@mock.patch('banzai.lco.LCOFrameFactory.open') +@mock.patch('banzai.calibrations.CalibrationUser.get_calibration_file_info') +def test_multiread_bias_subtraction(mock_super_cal_name, mock_super_frame): + mock_super_cal_name.return_value = {'filename': 'test.fits'} + input_bias = 500.0 + nreads = 5 + nx = 101 + ny = 103 + bias_pattern = np.random.normal(0.0, 15.0, size=(ny, nx)) + stacked_hdu = FakeCCDData(bias_level=input_bias, data=bias_pattern) + mock_super_frame.return_value = FakeLCOObservationFrame(hdu_list=[stacked_hdu]) + data = bias_pattern * nreads + input_bias * nreads + image = FakeLCOObservationFrame(hdu_list=[FakeCCDData(bias_level=input_bias, data=data)]) + image.n_sub_exposures = nreads + subtractor = BiasSubtractor(FakeContext()) + image = subtractor.do_stage(image) + assert np.abs(image.meta.get('BIASLVL') - input_bias) < 1.0 + assert np.abs(np.mean(image.data)) < 1.0 diff --git a/banzai/tests/test_end_to_end.py b/banzai/tests/test_end_to_end.py index afd7a9d2..0beeddb7 100644 --- a/banzai/tests/test_end_to_end.py +++ b/banzai/tests/test_end_to_end.py @@ -159,7 +159,7 @@ def run_check_if_stacked_calibrations_are_in_db(raw_file_pattern, calibration_ty def observation_portal_side_effect(*args, **kwargs): - # To produce the mock observation portal resonse, we need to modify the response from + # To produce the mock observation portal response, we need to modify the response from # this type of url # https://observe.lco.global/api/observations/?enclosure=aqwa&telescope=0m4a&priority=&state=COMPLETED&time_span=&start_after=2024-07-23&start_before=&end_after=&end_before=2024-07-25&modified_after=&created_after=&created_before=&request_id=&request_group_id=&user=&proposal=calibrate&instrument_type=0M4-SCICAM-QHY600&configuration_type=SKY_FLAT&ordering= site = kwargs['params']['site'] diff --git a/banzai/tests/test_frames.py b/banzai/tests/test_frames.py index 0317958b..e66ae92f 100644 --- a/banzai/tests/test_frames.py +++ b/banzai/tests/test_frames.py @@ -140,7 +140,9 @@ def test_all_datatypes_wrong(): def test_subtract(): - test_data = FakeCCDData(image_multiplier=4, uncertainty=3) + nx = 101 + ny = 103 + test_data = FakeCCDData(image_multiplier=4, uncertainty=3*np.ones((ny, nx)), nx=nx, ny=ny) test_data -= 1 assert (test_data.data == 3 * np.ones(test_data.data.shape)).all() @@ -179,7 +181,9 @@ def test_trim(): def test_init_poisson_uncertainties(): # Make sure the uncertainties add in quadrature - test_data = FakeCCDData(image_multiplier=16, uncertainty=3) + nx = 101 + ny = 103 + test_data = FakeCCDData(image_multiplier=16, uncertainty=3 * np.ones((ny, nx)), nx=nx, ny=ny) test_data.init_poisson_uncertainties() assert (test_data.uncertainty == 5 * np.ones(test_data.data.shape)).all() diff --git a/banzai/tests/utils.py b/banzai/tests/utils.py index 2a48aa95..a79f2fd3 100644 --- a/banzai/tests/utils.py +++ b/banzai/tests/utils.py @@ -39,11 +39,11 @@ def __init__(self, image_multiplier=1.0, nx=101, ny=103, name='test_image', read self.mask = np.zeros(self.data.shape, dtype=np.uint8) else: self.mask = mask + self.memmap = True if uncertainty is None: self.uncertainty = self.read_noise * np.ones(self.data.shape, dtype=self.data.dtype) else: self.uncertainty = uncertainty - self.memmap = True if 'SATURATE' not in self.meta: self.meta['SATURATE'] = 65535.0 if 'GAIN' not in self.meta: @@ -69,10 +69,7 @@ def __init__(self, hdu_list=None, file_path='/tmp/test_image.fits', instrument=N self._file_path = file_path self.is_bad = False self.hdu_order = ['SCI', 'CAT', 'BPM', 'ERR'] - - @property - def n_sub_exposures(self): - return 1 + self.n_sub_exposures = 1 for keyword in kwargs: setattr(self, keyword, kwargs[keyword])