From c34d4400a9b8f94176532adffd8d65b41e7069bd Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Thu, 30 May 2024 09:56:11 -0400 Subject: [PATCH 1/4] removed feature step from production deployment --- .github/workflows/production_ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/production_ci.yaml b/.github/workflows/production_ci.yaml index a7866916e..609d28bb4 100644 --- a/.github/workflows/production_ci.yaml +++ b/.github/workflows/production_ci.yaml @@ -281,7 +281,7 @@ jobs: poetry run python main.py deploy add-package correction-step --chart=correction-step --values=correction_step-helm-values --chart-folder=correction_step poetry run python main.py deploy add-package early-classifier-step --chart=early-classifier --values=early_classification_step-helm-values --chart-folder=early_classification_step # feature step from main - poetry run python main.py deploy add-package feature-step --chart=feature-step --values=feature_step-helm-values --chart-folder=feature_step + # poetry run python main.py deploy add-package feature-step --chart=feature-step --values=feature_step-helm-values --chart-folder=feature_step # feature step freezed 23.x version poetry run python main.py deploy add-package feature-step-23.x --chart=feature-step --values=feature_step-helm-values:13 --chart-folder=feature_step poetry run python main.py deploy add-package lc-classifier-step --chart=lc-classifier-step --values=lc_classification_step_ztf-helm-values --chart-folder=lc_classification_step From 45a2faac6423a1df76610cc82f4be3b45750288e Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Thu, 30 May 2024 10:14:28 -0400 Subject: [PATCH 2/4] added a filter for the forced --- .../prv_candidates_step/core/strategy/ztf.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py index ee0d9f1db..9c6215ae4 100644 --- a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py +++ b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py @@ -175,6 +175,24 @@ def extract_detections_and_non_detections(alert: dict) -> dict: prv_candidates = pickle.loads(prv_candidates) if prv_candidates else [] forced_photometries = alert["extra_fields"].pop("fp_hists", []) + + # filter forced photometry by value + # potencial bugs from ztf + def filter_fp(fp): + forcediffimflux_bad_values = [ + None, 0 + ] + # if the value is not close to -99999 return true + good_fp = ( + fp["forcediffimflux"] not in forcediffimflux_bad_values + and + not np.isclose(fp["forcediffimflux"], -99999) + ) + return good_fp + + # use the filter funcion to remove bad fp + forced_photometries = filter(filter_fp, forced_photometries) + forced_photometries = ( pickle.loads(forced_photometries) if forced_photometries else [] ) From 7073725dada6e770770e79f8872327181a4680fc Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Thu, 30 May 2024 10:24:35 -0400 Subject: [PATCH 3/4] fixed the parsing of the fp avro --- .../prv_candidates_step/core/strategy/ztf.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py index 9c6215ae4..7f695cd34 100644 --- a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py +++ b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py @@ -175,10 +175,13 @@ def extract_detections_and_non_detections(alert: dict) -> dict: prv_candidates = pickle.loads(prv_candidates) if prv_candidates else [] forced_photometries = alert["extra_fields"].pop("fp_hists", []) + forced_photometries = ( + pickle.loads(forced_photometries) if forced_photometries else [] + ) # filter forced photometry by value # potencial bugs from ztf - def filter_fp(fp): + def filter_fp(fp): forcediffimflux_bad_values = [ None, 0 ] @@ -191,11 +194,7 @@ def filter_fp(fp): return good_fp # use the filter funcion to remove bad fp - forced_photometries = filter(filter_fp, forced_photometries) - - forced_photometries = ( - pickle.loads(forced_photometries) if forced_photometries else [] - ) + forced_photometries = list(filter(filter_fp, forced_photometries)) acopy = copy.deepcopy(alert) detections = [acopy] From 1bca0fbbaf4d361226688a3d6f21b9d9030a944a Mon Sep 17 00:00:00 2001 From: Alex Alvarez Toledo Date: Thu, 30 May 2024 10:25:12 -0400 Subject: [PATCH 4/4] run black --- .../prv_candidates_step/core/strategy/ztf.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py index 7f695cd34..b397c0d4e 100644 --- a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py +++ b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py @@ -181,15 +181,13 @@ def extract_detections_and_non_detections(alert: dict) -> dict: # filter forced photometry by value # potencial bugs from ztf - def filter_fp(fp): - forcediffimflux_bad_values = [ - None, 0 - ] + def filter_fp(fp): + forcediffimflux_bad_values = [None, 0] # if the value is not close to -99999 return true - good_fp = ( - fp["forcediffimflux"] not in forcediffimflux_bad_values - and - not np.isclose(fp["forcediffimflux"], -99999) + good_fp = fp[ + "forcediffimflux" + ] not in forcediffimflux_bad_values and not np.isclose( + fp["forcediffimflux"], -99999 ) return good_fp