From beeeb7a02181941071487f55e6367b7999c1ae3c Mon Sep 17 00:00:00 2001 From: ale-munozarancibia Date: Wed, 6 Dec 2023 11:44:32 -0300 Subject: [PATCH 01/20] Fix mag and e_mag values when forcediffimflux and/or forcediffimfluxunc is -99999 --- .../prv_candidates_step/core/strategy/ztf.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py index 3a009afd0..d5ff9761f 100644 --- a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py +++ b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py @@ -79,11 +79,21 @@ def __calculate_mag(cls, data): magzpsci = data["magzpsci"] flux2uJy = 10.0 ** ((8.9 - magzpsci) / 2.5) * 1.0e6 - forcediffimflux = data["forcediffimflux"] * flux2uJy - forcediffimfluxunc = data["forcediffimfluxunc"] * flux2uJy + if np.isclose(data["forcediffimflux"], -99999.): + mag = np.nan + else: + forcediffimflux = data["forcediffimflux"] * flux2uJy + mag = -2.5 * np.log10(np.abs(forcediffimflux)) + 23.9 + + if np.isclose(data["forcediffimfluxunc"], -99999.): + e_mag = np.nan + else: + if np.isclose(data["forcediffimflux"], -99999.): + e_mag = np.nan + else: + forcediffimfluxunc = data["forcediffimfluxunc"] * flux2uJy + e_mag = 1.0857 * forcediffimfluxunc / np.abs(forcediffimflux) - mag = -2.5 * np.log10(np.abs(forcediffimflux)) + 23.9 - e_mag = 1.0857 * forcediffimfluxunc / np.abs(forcediffimflux) return mag, e_mag @classmethod From c5dac92891ac313afa4e4e566b03e6b872b49ab4 Mon Sep 17 00:00:00 2001 From: ale-munozarancibia Date: Wed, 6 Dec 2023 12:51:13 -0300 Subject: [PATCH 02/20] Simplified expressions --- .../prv_candidates_step/core/strategy/ztf.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py index d5ff9761f..6e86ca2a0 100644 --- a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py +++ b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py @@ -79,20 +79,18 @@ def __calculate_mag(cls, data): magzpsci = data["magzpsci"] flux2uJy = 10.0 ** ((8.9 - magzpsci) / 2.5) * 1.0e6 + forcediffimflux = data["forcediffimflux"] * flux2uJy + forcediffimfluxunc = data["forcediffimfluxunc"] * flux2uJy + + mag = -2.5 * np.log10(np.abs(forcediffimflux)) + 23.9 + e_mag = 1.0857 * forcediffimfluxunc / np.abs(forcediffimflux) + if np.isclose(data["forcediffimflux"], -99999.): mag = np.nan - else: - forcediffimflux = data["forcediffimflux"] * flux2uJy - mag = -2.5 * np.log10(np.abs(forcediffimflux)) + 23.9 - if np.isclose(data["forcediffimfluxunc"], -99999.): + if np.isclose(data["forcediffimflux"], -99999.) \ + or np.isclose(data["forcediffimfluxunc"], -99999.): e_mag = np.nan - else: - if np.isclose(data["forcediffimflux"], -99999.): - e_mag = np.nan - else: - forcediffimfluxunc = data["forcediffimfluxunc"] * flux2uJy - e_mag = 1.0857 * forcediffimfluxunc / np.abs(forcediffimflux) return mag, e_mag From 1d7b0c4e957750b39d28113a94dbb6d555ae0532 Mon Sep 17 00:00:00 2001 From: ale-munozarancibia Date: Wed, 6 Dec 2023 12:58:11 -0300 Subject: [PATCH 03/20] Fixed if condition --- prv_candidates_step/prv_candidates_step/core/strategy/ztf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py index 6e86ca2a0..f2d3150a9 100644 --- a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py +++ b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py @@ -87,9 +87,9 @@ def __calculate_mag(cls, data): if np.isclose(data["forcediffimflux"], -99999.): mag = np.nan + e_mag = np.nan - if np.isclose(data["forcediffimflux"], -99999.) \ - or np.isclose(data["forcediffimfluxunc"], -99999.): + if np.isclose(data["forcediffimfluxunc"], -99999.): e_mag = np.nan return mag, e_mag From 4bce93e28d3437d98a06f72b1fd32b8f8abb9047 Mon Sep 17 00:00:00 2001 From: ale-munozarancibia Date: Wed, 6 Dec 2023 14:09:31 -0300 Subject: [PATCH 04/20] Fixed black error --- prv_candidates_step/prv_candidates_step/core/strategy/ztf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py index f2d3150a9..d9fae08fc 100644 --- a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py +++ b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py @@ -85,11 +85,11 @@ def __calculate_mag(cls, data): mag = -2.5 * np.log10(np.abs(forcediffimflux)) + 23.9 e_mag = 1.0857 * forcediffimfluxunc / np.abs(forcediffimflux) - if np.isclose(data["forcediffimflux"], -99999.): + if np.isclose(data["forcediffimflux"], -99999.0): mag = np.nan e_mag = np.nan - if np.isclose(data["forcediffimfluxunc"], -99999.): + if np.isclose(data["forcediffimfluxunc"], -99999.0): e_mag = np.nan return mag, e_mag From 356750e9d88355b0983d0a9ffac9710d31a5e631 Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Tue, 5 Dec 2023 14:21:49 -0300 Subject: [PATCH 05/20] chore: update value --- .../production/templates/xmatch_step_helm_values.tftpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/helm_values/production/templates/xmatch_step_helm_values.tftpl b/ci/helm_values/production/templates/xmatch_step_helm_values.tftpl index 3549c2871..a8530f922 100644 --- a/ci/helm_values/production/templates/xmatch_step_helm_values.tftpl +++ b/ci/helm_values/production/templates/xmatch_step_helm_values.tftpl @@ -57,7 +57,7 @@ configYaml: sasl.password: ${kafka_password} SCHEMA_PATH: "/schemas/xmatch_step/output.avsc" SCRIBE_PRODUCER_CONFIG: - CLASS: "apf.producers.KafkaSchemalessProducer" + CLASS: "apf.producers.KafkaProducer" PARAMS: bootstrap.servers: ${kafka_server} sasl.username: ${kafka_username} From ab031192526ead24ec0bdb0f39e2c4e7c58e9eb6 Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Tue, 5 Dec 2023 16:50:54 -0300 Subject: [PATCH 06/20] fix: add early classifier --- ci/helm_values/production/main.tf | 4 ++-- ci/helm_values/production/variables.tf | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ci/helm_values/production/main.tf b/ci/helm_values/production/main.tf index 8064fc34f..f18b68b79 100644 --- a/ci/helm_values/production/main.tf +++ b/ci/helm_values/production/main.tf @@ -213,8 +213,8 @@ resource "aws_ssm_parameter" "early_classification_step" { value = templatefile("templates/early_classification_step_helm_values.tftpl", { kafka_server = data.aws_msk_cluster.msk_production.bootstrap_brokers_sasl_scram kafka_public_server = data.aws_msk_cluster.msk_public.bootstrap_brokers_sasl_scram - kafka_username = var.correction_kafka_username - kafka_password = var.correction_kafka_password + kafka_username = var.early_kafka_username + kafka_password = var.early_kafka_password ghcr_username = var.ghcr_username ghcr_password = var.ghcr_password db_host = data.aws_instance.psql-alerts.private_ip diff --git a/ci/helm_values/production/variables.tf b/ci/helm_values/production/variables.tf index 01d5c3892..dbe269c89 100644 --- a/ci/helm_values/production/variables.tf +++ b/ci/helm_values/production/variables.tf @@ -94,6 +94,12 @@ variable "logstash_kafka_username" { variable "logstash_kafka_password" { type = string } +variable "early_kafka_username" { + type = string +} +variable "early_kafka_password" { + type = string +} variable "ghcr_username" { type = string } From f827328dd3e131569f709290ee75643da1bff83d Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 10:46:42 -0300 Subject: [PATCH 07/20] fix: psql default values --- libs/db-plugins/db_plugins/db/sql/models.py | 22 +++++++++---------- .../sorting_hat_step/utils/database.py | 9 ++++++++ .../sorting_hat_step/utils/wizard.py | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/libs/db-plugins/db_plugins/db/sql/models.py b/libs/db-plugins/db_plugins/db/sql/models.py index e7bae5b0c..a00bc0ae0 100755 --- a/libs/db-plugins/db_plugins/db/sql/models.py +++ b/libs/db-plugins/db_plugins/db/sql/models.py @@ -39,25 +39,25 @@ class Object(Base): __tablename__ = "object" oid = Column(String, primary_key=True) - ndethist = Column(Integer) - ncovhist = Column(Integer) + ndethist = Column(Integer, nullable=False) + ncovhist = Column(Integer, nullable=False) mjdstarthist = Column(Float(precision=53)) mjdendhist = Column(Float(precision=53)) - corrected = Column(Boolean) - stellar = Column(Boolean) - ndet = Column(Integer) + corrected = Column(Boolean, nullable=False, default=False) + stellar = Column(Boolean, nullable=False, default=False) + ndet = Column(Integer, nullable=False, default=1) g_r_max = Column(Float) g_r_max_corr = Column(Float) g_r_mean = Column(Float) g_r_mean_corr = Column(Float) - meanra = Column(Float(precision=53)) - meandec = Column(Float(precision=53)) + meanra = Column(Float(precision=53), nullable=False) + meandec = Column(Float(precision=53), nullable=False) sigmara = Column(Float(precision=53)) sigmadec = Column(Float(precision=53)) - deltajd = Column(Float(precision=53)) - firstmjd = Column(Float(precision=53)) - lastmjd = Column(Float(precision=53)) - step_id_corr = Column(String) + deltajd = Column(Float(precision=53), nullable=False) + firstmjd = Column(Float(precision=53), nullable=False) + lastmjd = Column(Float(precision=53), nullable=False) + step_id_corr = Column(String, nullable=False) diffpos = Column(Boolean) reference_change = Column(Boolean) diff --git a/sorting_hat_step/sorting_hat_step/utils/database.py b/sorting_hat_step/sorting_hat_step/utils/database.py index 43d276fa0..c1fc54916 100644 --- a/sorting_hat_step/sorting_hat_step/utils/database.py +++ b/sorting_hat_step/sorting_hat_step/utils/database.py @@ -5,6 +5,9 @@ from db_plugins.db.sql.models import Object from ..database import MongoConnection, PsqlConnection +import importlib.metadata + +version = importlib.metadata.version("sorting-hat-step") def oid_query(db: MongoConnection, oid: list) -> Union[str, None]: @@ -82,6 +85,12 @@ def format_extra_fields(record): "ncovhist": extra_fields["ncovhist"], "mjdstarthist": extra_fields["jdstarthist"] - 2400000.5, "mjdendhist": extra_fields["jdendhist"] - 2400000.5, + "meanra": record["ra"], + "meandec": record["dec"], + "firstmjd": record["mjd"], + "lastmjd": record["mjd"], + "deltajd": 0, + "step_id_corr": version, } oids = { diff --git a/sorting_hat_step/sorting_hat_step/utils/wizard.py b/sorting_hat_step/sorting_hat_step/utils/wizard.py index 8e6d7029b..64df9a1b9 100644 --- a/sorting_hat_step/sorting_hat_step/utils/wizard.py +++ b/sorting_hat_step/sorting_hat_step/utils/wizard.py @@ -212,7 +212,7 @@ def insert_empty_objects(mongodb: MongoConnection, alerts: pd.DataFrame, psql=No :param db: Connection to the database. :alerts: Dataframe with alerts. """ - objects = alerts[["oid", "aid", "sid", "extra_fields"]] + objects = alerts[["oid", "aid", "sid", "extra_fields", "ra", "dec", "mjd"]] objects = objects.rename(columns={"aid": "_id"}) mongo_objects = objects.groupby("_id").oid.apply(list).reset_index() logger.debug( From d453f2b559f2030999e90fcaf527b4378a7bc4b7 Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 10:54:02 -0300 Subject: [PATCH 08/20] tests: fix mock data --- .../tests/unittest/test_wizard.py | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/sorting_hat_step/tests/unittest/test_wizard.py b/sorting_hat_step/tests/unittest/test_wizard.py index 9760b022b..02529b8cd 100644 --- a/sorting_hat_step/tests/unittest/test_wizard.py +++ b/sorting_hat_step/tests/unittest/test_wizard.py @@ -110,10 +110,42 @@ def test_decode(self): def test_write_object(self, insert_mock): alerts = pd.DataFrame( [ - {"oid": 10, "aid": 0, "sid": "ZTF", "extra_fields": {}}, - {"oid": 20, "aid": 1, "sid": "ZTF", "extra_fields": {}}, - {"oid": 30, "aid": 0, "sid": "ZTF", "extra_fields": {}}, - {"oid": 40, "aid": 2, "sid": "ZTF", "extra_fields": {}}, + { + "oid": 10, + "aid": 0, + "sid": "ZTF", + "extra_fields": {}, + "ra": 0, + "dec": 0, + "mjd": 0, + }, + { + "oid": 20, + "aid": 1, + "sid": "ZTF", + "extra_fields": {}, + "ra": 0, + "dec": 0, + "mjd": 0, + }, + { + "oid": 30, + "aid": 0, + "sid": "ZTF", + "extra_fields": {}, + "ra": 0, + "dec": 0, + "mjd": 0, + }, + { + "oid": 40, + "aid": 2, + "sid": "ZTF", + "extra_fields": {}, + "ra": 0, + "dec": 0, + "mjd": 0, + }, ] ) From bc0d8a1a8af8b0b34815a7826137838e033597b5 Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 11:08:28 -0300 Subject: [PATCH 09/20] tests: fix mock data with non nullable fields --- scribe/tests/integration/test_step_sql.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/scribe/tests/integration/test_step_sql.py b/scribe/tests/integration/test_step_sql.py index 8236ee5d2..c70640e09 100644 --- a/scribe/tests/integration/test_step_sql.py +++ b/scribe/tests/integration/test_step_sql.py @@ -45,7 +45,7 @@ @pytest.mark.usefixtures("psql_service") @pytest.mark.usefixtures("kafka_service") -class MongoIntegrationTest(unittest.TestCase): +class PsqlIntegrationTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.db = PsqlDatabase(DB_CONFIG["PSQL"]) @@ -58,7 +58,7 @@ def setUpClass(cls): session.execute( text( """ - INSERT INTO step(step_id, name, version, comments, date) + INSERT INTO step(step_id, name, version, comments, date) VALUES ('v1', 'version 1', '1', '', current_timestamp) """ ) @@ -80,10 +80,15 @@ def test_insert_objects_into_database(self): "oid": "ZTF02ululeea", "ndet": 1, "firstmjd": 50001, + "lastmjd": 50001, "g_r_max": 1.0, "g_r_mean_corr": 0.9, "meanra": 45, "meandec": 45, + "ndethist": 1, + "ncovhist": 1, + "deltajd": 0, + "step_id_corr": "v1", }, } ) @@ -95,10 +100,15 @@ def test_insert_objects_into_database(self): "oid": "ZTF03ululeea", "ndet": 1, "firstmjd": 50001, + "lastmjd": 50001, "g_r_max": 1.0, "g_r_mean_corr": 0.9, "meanra": 45, "meandec": 45, + "ndethist": 1, + "ncovhist": 1, + "step_id_corr": "v1", + "deltajd": 0, }, } ) @@ -469,7 +479,7 @@ def test_update_object_stats(self): text( """ SELECT object.oid as oid, fid, meanra, meandec, magstat.ndet as ndet - FROM object JOIN magstat on object.oid = magstat.oid + FROM object JOIN magstat on object.oid = magstat.oid """ ) ) @@ -527,7 +537,7 @@ def test_upsert_xmatch(self): result = session.execute( text( """ - SELECT * FROM xmatch WHERE oid = 'ZTF04ululeea' + SELECT * FROM xmatch WHERE oid = 'ZTF04ululeea' """ ) ) @@ -687,7 +697,7 @@ def test_forced_photometry_insertion(self): result = session.execute( text( """ - SELECT * FROM forced_photometry WHERE oid = 'ZTF04ululeea' + SELECT * FROM forced_photometry WHERE oid = 'ZTF04ululeea' """ ) ) From ab2b891079bf23cf40b430fd5fc1510c0e478f2c Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 11:12:26 -0300 Subject: [PATCH 10/20] tests: fix mock data with non nullable fields --- lightcurve-step/tests/integration/conftest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lightcurve-step/tests/integration/conftest.py b/lightcurve-step/tests/integration/conftest.py index 5f3ca122b..dfc359d06 100644 --- a/lightcurve-step/tests/integration/conftest.py +++ b/lightcurve-step/tests/integration/conftest.py @@ -58,7 +58,9 @@ def _produce(topic: str): { "PARAMS": {"bootstrap.servers": "localhost:9092"}, "TOPIC": topic, - "SCHEMA_PATH": os.path.join(os.path.dirname(__file__), "input_schema.avsc"), + "SCHEMA_PATH": os.path.join( + os.path.dirname(__file__), "input_schema.avsc" + ), } ) messages = generate_message(schema, 10) @@ -70,13 +72,15 @@ def _produce(topic: str): return _produce + def populate_sql(conn: PsqlDatabase): with conn.session() as session: session.execute( text( """ - INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec) - VALUES ('ZTF000llmn', 1, 50001, 1.0, 0.9, 45, 45) ON CONFLICT DO NOTHING + INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec, step_id_corr, \ + lastmjd, deltajd, ncovhist, ndethist, corrected, stellar) + VALUES ('ZTF000llmn', 1, 50001, 1.0, 0.9, 45, 45, 'v1', 50001, 0, 1, 1, false, false) ON CONFLICT DO NOTHING """ ) ) From e0bd983f20b6d140167d70aebaa2626380878a48 Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 11:33:53 -0300 Subject: [PATCH 11/20] tests: fix mock data with non nullable fields on the rest of the tests --- scribe/tests/integration/test_step_sql.py | 41 +++++++++++++---------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/scribe/tests/integration/test_step_sql.py b/scribe/tests/integration/test_step_sql.py index c70640e09..559bd05b4 100644 --- a/scribe/tests/integration/test_step_sql.py +++ b/scribe/tests/integration/test_step_sql.py @@ -170,8 +170,9 @@ def test_insert_detections_into_database(self): with self.db.session() as session: session.execute( text( - """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec) - VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45) ON CONFLICT DO NOTHING""" + """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec, step_id_corr, \ + lastmjd, deltajd, ncovhist, ndethist, corrected, stellar) + VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45, 'v1', 50001, 0, 1, 1, false, false) ON CONFLICT DO NOTHING""" ) ) session.commit() @@ -217,8 +218,9 @@ def test_upsert_non_detections(self): with self.db.session() as session: session.execute( text( - """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec) - VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45) ON CONFLICT DO NOTHING""" + """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec, step_id_corr, \ + lastmjd, deltajd, ncovhist, ndethist, corrected, stellar) + VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45, 'v1', 50001, 0, 1, 1, false, false) ON CONFLICT DO NOTHING""" ) ) session.commit() @@ -249,8 +251,9 @@ def test_upsert_features(self): ) session.execute( text( - """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec) - VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45) ON CONFLICT DO NOTHING""" + """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec, step_id_corr, \ + lastmjd, deltajd, ncovhist, ndethist, corrected, stellar) + VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45, 'v1', 50001, 0, 1, 1, false, false) ON CONFLICT DO NOTHING""" ) ) session.commit() @@ -319,8 +322,9 @@ def test_upsert_probabilities(self): with self.db.session() as session: session.execute( text( - """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec) - VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45) ON CONFLICT DO NOTHING""" + """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec, step_id_corr, \ + lastmjd, deltajd, ncovhist, ndethist, corrected, stellar) + VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45, 'v1', 50001, 0, 1, 1, false, false) ON CONFLICT DO NOTHING""" ) ) session.commit() @@ -383,8 +387,9 @@ def test_update_object_stats(self): with self.db.session() as session: session.execute( text( - """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec) - VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45) ON CONFLICT DO NOTHING""" + """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec, step_id_corr, \ + lastmjd, deltajd, ncovhist, ndethist, corrected, stellar) + VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45, 'v1', 50001, 0, 1, 1, false, false) ON CONFLICT DO NOTHING""" ) ) session.commit() @@ -489,9 +494,10 @@ def test_upsert_xmatch(self): with self.db.session() as session: session.execute( text( - """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec) - VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45), - ('ZTF05ululeea', 1, 50001, 1.0, 0.9, 45, 45) ON CONFLICT DO NOTHING""" + """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec, step_id_corr, \ + lastmjd, deltajd, ncovhist, ndethist, corrected, stellar) + VALUES ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45, 'v1', 50001, 0, 1, 1, false, false), + ('ZTF05ululeea', 1, 50001, 1.0, 0.9, 45, 45, 'v1', 50001, 0, 1, 1, false, false) ON CONFLICT DO NOTHING""" ) ) session.commit() @@ -547,10 +553,11 @@ def test_forced_photometry_insertion(self): with self.db.session() as session: session.execute( text( - """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec) - VALUES - ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45), - ('ZTF05ululeea', 1, 50001, 1.0, 0.9, 45, 45) + """INSERT INTO object(oid, ndet, firstmjd, g_r_max, g_r_mean_corr, meanra, meandec, step_id_corr, \ + lastmjd, deltajd, ncovhist, ndethist, corrected, stellar) + VALUES + ('ZTF04ululeea', 1, 50001, 1.0, 0.9, 45, 45, 'v1', 50001, 0, 1, 1, false, false), + ('ZTF05ululeea', 1, 50001, 1.0, 0.9, 45, 45, 'v1', 50001, 0, 1, 1, false, false) ON CONFLICT DO NOTHING""" ) ) From 6929af33f6bb569bcc1ffc3971c19f2b6c5b97eb Mon Sep 17 00:00:00 2001 From: "@alerceadmin" Date: Wed, 6 Dec 2023 14:44:53 +0000 Subject: [PATCH 12/20] chore: update version --- alert_archiving_step/pyproject.toml | 2 +- charts/alert_archiving_step/Chart.yaml | 4 ++-- charts/correction_step/Chart.yaml | 4 ++-- charts/early_classification_step/Chart.yaml | 4 ++-- charts/feature_step/Chart.yaml | 4 ++-- charts/lc_classification_step/Chart.yaml | 4 ++-- charts/lightcurve-step/Chart.yaml | 4 ++-- charts/magstats_step/Chart.yaml | 4 ++-- charts/metadata_step/Chart.yaml | 4 ++-- charts/prv_candidates_step/Chart.yaml | 4 ++-- charts/s3_step/Chart.yaml | 4 ++-- charts/scribe/Chart.yaml | 4 ++-- charts/sorting_hat_step/Chart.yaml | 4 ++-- charts/watchlist_step/Chart.yaml | 4 ++-- charts/xmatch_step/Chart.yaml | 4 ++-- correction_step/pyproject.toml | 2 +- early_classification_step/pyproject.toml | 4 +++- feature_step/pyproject.toml | 2 +- lc_classification_step/pyproject.toml | 2 +- lightcurve-step/pyproject.toml | 2 +- magstats_step/pyproject.toml | 2 +- metadata_step/pyproject.toml | 2 +- prv_candidates_step/pyproject.toml | 2 +- s3_step/pyproject.toml | 2 +- scribe/pyproject.toml | 2 +- sorting_hat_step/pyproject.toml | 2 +- watchlist_step/pyproject.toml | 2 +- xmatch_step/pyproject.toml | 2 +- 28 files changed, 44 insertions(+), 42 deletions(-) diff --git a/alert_archiving_step/pyproject.toml b/alert_archiving_step/pyproject.toml index e7cb3770b..2c4a8b0fd 100644 --- a/alert_archiving_step/pyproject.toml +++ b/alert_archiving_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "alert-archiving-step" -version = "23.12.11a0" +version = "23.12.11a1" description = "Group alerts and upload to S3 by date" authors = ["ALeRCE"] readme = "README.md" diff --git a/charts/alert_archiving_step/Chart.yaml b/charts/alert_archiving_step/Chart.yaml index 71c371443..23e24924d 100644 --- a/charts/alert_archiving_step/Chart.yaml +++ b/charts/alert_archiving_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: A Helm chart for Kubernetes name: alert-archive-step type: application -version: 1.2.194 +version: 1.2.195 diff --git a/charts/correction_step/Chart.yaml b/charts/correction_step/Chart.yaml index f22492c63..a1c304ecb 100644 --- a/charts/correction_step/Chart.yaml +++ b/charts/correction_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: Correction step chart name: correction-step type: application -version: 2.0.128 +version: 2.0.129 diff --git a/charts/early_classification_step/Chart.yaml b/charts/early_classification_step/Chart.yaml index 5da9c8b87..fb7aabaf1 100644 --- a/charts/early_classification_step/Chart.yaml +++ b/charts/early_classification_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: A Helm chart for Kubernetes name: early-classifier type: application -version: 1.2.305 +version: 1.2.306 diff --git a/charts/feature_step/Chart.yaml b/charts/feature_step/Chart.yaml index f0cb386a4..9c627745b 100644 --- a/charts/feature_step/Chart.yaml +++ b/charts/feature_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: Step for feature calculation name: feature-step type: application -version: 2.0.128 +version: 2.0.129 diff --git a/charts/lc_classification_step/Chart.yaml b/charts/lc_classification_step/Chart.yaml index 715dfa266..e78b34d46 100644 --- a/charts/lc_classification_step/Chart.yaml +++ b/charts/lc_classification_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: Lightcurve classifier step name: lc-classifier-step type: application -version: 5.0.128 +version: 5.0.129 diff --git a/charts/lightcurve-step/Chart.yaml b/charts/lightcurve-step/Chart.yaml index 559812f84..99a6c2291 100644 --- a/charts/lightcurve-step/Chart.yaml +++ b/charts/lightcurve-step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: Full lightcurve provider step name: lightcurve-step type: application -version: 2.0.128 +version: 2.0.129 diff --git a/charts/magstats_step/Chart.yaml b/charts/magstats_step/Chart.yaml index 0027013cc..e10a6fe44 100644 --- a/charts/magstats_step/Chart.yaml +++ b/charts/magstats_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: A Helm chart for Kubernetes name: magstats-step type: application -version: 2.0.128 +version: 2.0.129 diff --git a/charts/metadata_step/Chart.yaml b/charts/metadata_step/Chart.yaml index f077dbaaf..0ba621460 100644 --- a/charts/metadata_step/Chart.yaml +++ b/charts/metadata_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: A Helm chart for Kubernetes name: metadata-step type: application -version: 2.0.128 +version: 2.0.129 diff --git a/charts/prv_candidates_step/Chart.yaml b/charts/prv_candidates_step/Chart.yaml index 005cae73d..dc0fd432e 100644 --- a/charts/prv_candidates_step/Chart.yaml +++ b/charts/prv_candidates_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: Previous candidates processor step name: prv-candidates type: application -version: 2.0.128 +version: 2.0.129 diff --git a/charts/s3_step/Chart.yaml b/charts/s3_step/Chart.yaml index f675db413..57becfff9 100644 --- a/charts/s3_step/Chart.yaml +++ b/charts/s3_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: A Helm chart for Kubernetes name: s3-step type: application -version: 1.4.17 +version: 1.4.18 diff --git a/charts/scribe/Chart.yaml b/charts/scribe/Chart.yaml index de7c4f3a7..6c055fcf6 100644 --- a/charts/scribe/Chart.yaml +++ b/charts/scribe/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 2.0.128 +version: 2.0.129 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 diff --git a/charts/sorting_hat_step/Chart.yaml b/charts/sorting_hat_step/Chart.yaml index f7104b3a8..7c72028e8 100644 --- a/charts/sorting_hat_step/Chart.yaml +++ b/charts/sorting_hat_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: Sorting Hat deployment chart name: sorting-hat type: application -version: 3.0.128 +version: 3.0.129 diff --git a/charts/watchlist_step/Chart.yaml b/charts/watchlist_step/Chart.yaml index 6f118cc42..95bed3c8e 100644 --- a/charts/watchlist_step/Chart.yaml +++ b/charts/watchlist_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: A Helm chart for Kubernetes name: watchlist-step type: application -version: 1.2.301 +version: 1.2.302 diff --git a/charts/xmatch_step/Chart.yaml b/charts/xmatch_step/Chart.yaml index 7071495b0..9ecbb4145 100644 --- a/charts/xmatch_step/Chart.yaml +++ b/charts/xmatch_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a0 +appVersion: 23.12.11a1 description: A Helm chart for Kubernetes name: xmatch-step type: application -version: 3.0.128 +version: 3.0.129 diff --git a/correction_step/pyproject.toml b/correction_step/pyproject.toml index 704e677ed..0949c5e28 100644 --- a/correction_step/pyproject.toml +++ b/correction_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "correction" -version = "23.12.11a0" +version = "23.12.11a1" description = "Correction library for ALeRCE pipeline." authors = ["ALeRCE Broker "] readme = "README.md" diff --git a/early_classification_step/pyproject.toml b/early_classification_step/pyproject.toml index 08e147ac3..25d4c3981 100644 --- a/early_classification_step/pyproject.toml +++ b/early_classification_step/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "early_classification_step" -version = "23.12.11a0" +version = "23.12.11a1" description = "Early Classification Step" authors = [] readme = "README.md" @@ -287,6 +287,7 @@ line-length = 88 + [tool.poetry.group.test.dependencies] @@ -559,6 +560,7 @@ pytest-docker = "^1.0.1" + [tool.poetry.group.model.dependencies] diff --git a/feature_step/pyproject.toml b/feature_step/pyproject.toml index 7048b87e7..de29d8392 100644 --- a/feature_step/pyproject.toml +++ b/feature_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "feature-step" -version = "23.12.11a0" +version = "23.12.11a1" description = "" authors = [] readme = "README.md" diff --git a/lc_classification_step/pyproject.toml b/lc_classification_step/pyproject.toml index 029b3e57b..a2bc98f0b 100644 --- a/lc_classification_step/pyproject.toml +++ b/lc_classification_step/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "lc_classification_step" -version = "23.12.11a0" +version = "23.12.11a1" description = "LC Classification Step" authors = [] readme = "README.md" diff --git a/lightcurve-step/pyproject.toml b/lightcurve-step/pyproject.toml index 3a79b3c53..d6e68f616 100644 --- a/lightcurve-step/pyproject.toml +++ b/lightcurve-step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "lightcurve-step" -version = "23.12.11a0" +version = "23.12.11a1" description = "Lightcurve Step" authors = [] readme = "README.md" diff --git a/magstats_step/pyproject.toml b/magstats_step/pyproject.toml index 95bf8e3ff..05f3d00a1 100644 --- a/magstats_step/pyproject.toml +++ b/magstats_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "magstats-step" -version = "23.12.11a0" +version = "23.12.11a1" description = "" authors = ["ASHuenchuleo ", "Pablo Castellanos"] readme = "README.md" diff --git a/metadata_step/pyproject.toml b/metadata_step/pyproject.toml index 77d9ec6c0..18d7043d0 100644 --- a/metadata_step/pyproject.toml +++ b/metadata_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metadata-step" -version = "23.12.11a0" +version = "23.12.11a1" description = "" authors = ["Pedro Gallardo "] readme = "README.md" diff --git a/prv_candidates_step/pyproject.toml b/prv_candidates_step/pyproject.toml index 13dd1cff1..411f1e1aa 100644 --- a/prv_candidates_step/pyproject.toml +++ b/prv_candidates_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "prv-candidates-step" -version = "23.12.11a0" +version = "23.12.11a1" description = "Previous Detections Step" authors = [] readme = "README.md" diff --git a/s3_step/pyproject.toml b/s3_step/pyproject.toml index e9ed7ee3f..620014a5d 100644 --- a/s3_step/pyproject.toml +++ b/s3_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "s3-step" -version = "23.12.11a0" +version = "23.12.11a1" description = "S3 step" authors = [] readme = "README.md" diff --git a/scribe/pyproject.toml b/scribe/pyproject.toml index 724b6702d..68a659a14 100644 --- a/scribe/pyproject.toml +++ b/scribe/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "scribe" -version = "23.12.11a0" +version = "23.12.11a1" description = "Mongo scribe" authors = [] readme = "README.md" diff --git a/sorting_hat_step/pyproject.toml b/sorting_hat_step/pyproject.toml index 7781cd197..5d6a077ed 100644 --- a/sorting_hat_step/pyproject.toml +++ b/sorting_hat_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "sorting-hat-step" -version = "23.12.11a0" +version = "23.12.11a1" description = "Sorting hat step" authors = [] readme = "README.md" diff --git a/watchlist_step/pyproject.toml b/watchlist_step/pyproject.toml index d63a118ab..6714188d3 100644 --- a/watchlist_step/pyproject.toml +++ b/watchlist_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "watchlist-step" -version = "23.12.11a0" +version = "23.12.11a1" description = "Watchlist Step" authors = [] readme = "README.md" diff --git a/xmatch_step/pyproject.toml b/xmatch_step/pyproject.toml index fe6ba5bb0..f84dc2e96 100644 --- a/xmatch_step/pyproject.toml +++ b/xmatch_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "xmatch-step" -version = "23.12.11a0" +version = "23.12.11a1" description = "xmatch step" authors = [] readme = "README.md" From 38ff93f9547163a0280c71650a741910f11ba337 Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 12:40:26 -0300 Subject: [PATCH 13/20] chore: apply formatting --- schemas/ztf/alert.avsc | 50 +++++++---- schemas/ztf/fp_hist.avsc | 190 ++++++++++++++++++++++++++++++++------- 2 files changed, 195 insertions(+), 45 deletions(-) diff --git a/schemas/ztf/alert.avsc b/schemas/ztf/alert.avsc index f468b8c0f..67b852c29 100644 --- a/schemas/ztf/alert.avsc +++ b/schemas/ztf/alert.avsc @@ -4,19 +4,39 @@ "doc": "avro alert schema for ZTF (www.ztf.caltech.edu)", "version": "4.02", "fields": [ - {"name": "schemavsn", "type": "string", "doc": "schema version used"}, - {"name": "publisher", "type": "string", "doc": "origin of alert packet"}, - {"name": "objectId", "type": "string", "doc": "object identifier or name"}, - {"name": "candid", "type": "long"}, - {"name": "candidate", "type": "candidate"}, - {"name": "prv_candidates", "type": ["null", { - "type": "array", - "items": "prv_candidate"}], "default": null}, - {"name": "fp_hists", "type": ["null", { - "type": "array", - "items": "fp_hist"}], "default": null}, - {"name": "cutoutScience", "type": ["null", "cutout"], "default": null}, - {"name": "cutoutTemplate", "type": ["null", "cutout"], "default": null}, - {"name": "cutoutDifference", "type": ["null", "cutout"], "default": null} - ] + { "name": "schemavsn", "type": "string", "doc": "schema version used" }, + { "name": "publisher", "type": "string", "doc": "origin of alert packet" }, + { + "name": "objectId", + "type": "string", + "doc": "object identifier or name" + }, + { "name": "candid", "type": "long" }, + { "name": "candidate", "type": "candidate" }, + { + "name": "prv_candidates", + "type": [ + "null", + { + "type": "array", + "items": "prv_candidate" + } + ], + "default": null + }, + { + "name": "fp_hists", + "type": [ + "null", + { + "type": "array", + "items": "fp_hist" + } + ], + "default": null + }, + { "name": "cutoutScience", "type": ["null", "cutout"], "default": null }, + { "name": "cutoutTemplate", "type": ["null", "cutout"], "default": null }, + { "name": "cutoutDifference", "type": ["null", "cutout"], "default": null } + ] } diff --git a/schemas/ztf/fp_hist.avsc b/schemas/ztf/fp_hist.avsc index 3a4b4b1ea..e44195319 100644 --- a/schemas/ztf/fp_hist.avsc +++ b/schemas/ztf/fp_hist.avsc @@ -4,34 +4,164 @@ "version": "4.02", "type": "record", "fields": [ - {"name": "field", "type": ["null", "int"], "default": null, "doc": "ZTF field ID"}, - {"name": "rcid", "type": ["null", "int"], "default": null, "doc": "Readout channel ID [00 .. 63]"}, - {"name": "fid", "type": "int", "doc": "Filter ID (1=g; 2=R; 3=i)"}, - {"name": "pid", "type": "long", "doc": "Processing ID for image"}, - {"name": "rfid", "type": "long", "doc": "Processing ID for reference image to facilitate archive retrieval"}, - {"name": "sciinpseeing", "type": ["null", "float"], "default": null, "doc": "Effective FWHM of sci image [pixels]"}, - {"name": "scibckgnd", "type": ["null", "float"], "default": null, "doc": "Background level in sci image [DN]"}, - {"name": "scisigpix", "type": ["null", "float"], "default": null, "doc": "Robust sigma per pixel in sci image [DN]"}, - {"name": "magzpsci", "type": ["null", "float"], "default": null, "doc": "Magnitude zero point for photometry estimates [mag]"}, - {"name": "magzpsciunc", "type": ["null", "float"], "default": null, "doc": "Magnitude zero point uncertainty (in magzpsci) [mag]"}, - {"name": "magzpscirms", "type": ["null", "float"], "default": null, "doc": "RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, - {"name": "clrcoeff", "type": ["null", "float"], "default": null, "doc": "Color coefficient from linear fit from photometric calibration of science image"}, - {"name": "clrcounc", "type": ["null", "float"], "default": null, "doc": "Color coefficient uncertainty from linear fit (corresponding to clrcoeff)"}, - {"name": "exptime", "type": ["null", "float"], "default": null, "doc": "Integration time of camera exposure [sec]"}, - {"name": "adpctdif1", "type": ["null", "float"], "default": null, "doc": "Full sci image astrometric RMS along R.A. with respect to Gaia1 [arcsec]"}, - {"name": "adpctdif2", "type": ["null", "float"], "default": null, "doc": "Full sci image astrometric RMS along Dec. with respect to Gaia1 [arcsec]"}, - {"name": "diffmaglim", "type": ["null", "float"], "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, - {"name": "programid", "type": "int", "doc": "Program ID: encodes either public, collab, or caltech mode"}, - {"name": "jd", "type": "double", "doc": "Observation Julian date at start of exposure [days]"}, - {"name": "forcediffimflux", "type": ["null", "float"], "default": null, "doc": "Forced difference image PSF-fit flux [DN]"}, - {"name": "forcediffimfluxunc", "type": ["null", "float"], "default": null, "doc": "1-sigma uncertainty in forcediffimflux [DN]"}, - {"name": "procstatus", "type": ["null", "string"], "default": null, "doc": "Forced photometry processing status codes (0 => no warnings); see documentation"}, - {"name": "distnr", "type": ["null", "float"], "default": null, "doc": "distance to nearest source in reference image PSF-catalog [arcsec]"}, - {"name": "ranr", "type": "double", "doc": "Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg]"}, - {"name": "decnr", "type": "double", "doc": "Declination of nearest source in reference image PSF-catalog; J2000 [deg]"}, - {"name": "magnr", "type": ["null", "float"], "default": null, "doc": "magnitude of nearest source in reference image PSF-catalog [mag]"}, - {"name": "sigmagnr", "type": ["null", "float"], "default": null, "doc": "1-sigma uncertainty in magnr [mag]"}, - {"name": "chinr", "type": ["null", "float"], "default": null, "doc": "DAOPhot chi parameter of nearest source in reference image PSF-catalog"}, - {"name": "sharpnr", "type": ["null", "float"], "default": null, "doc": "DAOPhot sharp parameter of nearest source in reference image PSF-catalog"} - ] + { + "name": "field", + "type": ["null", "int"], + "default": null, + "doc": "ZTF field ID" + }, + { + "name": "rcid", + "type": ["null", "int"], + "default": null, + "doc": "Readout channel ID [00 .. 63]" + }, + { "name": "fid", "type": "int", "doc": "Filter ID (1=g; 2=R; 3=i)" }, + { "name": "pid", "type": "long", "doc": "Processing ID for image" }, + { + "name": "rfid", + "type": "long", + "doc": "Processing ID for reference image to facilitate archive retrieval" + }, + { + "name": "sciinpseeing", + "type": ["null", "float"], + "default": null, + "doc": "Effective FWHM of sci image [pixels]" + }, + { + "name": "scibckgnd", + "type": ["null", "float"], + "default": null, + "doc": "Background level in sci image [DN]" + }, + { + "name": "scisigpix", + "type": ["null", "float"], + "default": null, + "doc": "Robust sigma per pixel in sci image [DN]" + }, + { + "name": "magzpsci", + "type": ["null", "float"], + "default": null, + "doc": "Magnitude zero point for photometry estimates [mag]" + }, + { + "name": "magzpsciunc", + "type": ["null", "float"], + "default": null, + "doc": "Magnitude zero point uncertainty (in magzpsci) [mag]" + }, + { + "name": "magzpscirms", + "type": ["null", "float"], + "default": null, + "doc": "RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]" + }, + { + "name": "clrcoeff", + "type": ["null", "float"], + "default": null, + "doc": "Color coefficient from linear fit from photometric calibration of science image" + }, + { + "name": "clrcounc", + "type": ["null", "float"], + "default": null, + "doc": "Color coefficient uncertainty from linear fit (corresponding to clrcoeff)" + }, + { + "name": "exptime", + "type": ["null", "float"], + "default": null, + "doc": "Integration time of camera exposure [sec]" + }, + { + "name": "adpctdif1", + "type": ["null", "float"], + "default": null, + "doc": "Full sci image astrometric RMS along R.A. with respect to Gaia1 [arcsec]" + }, + { + "name": "adpctdif2", + "type": ["null", "float"], + "default": null, + "doc": "Full sci image astrometric RMS along Dec. with respect to Gaia1 [arcsec]" + }, + { + "name": "diffmaglim", + "type": ["null", "float"], + "default": null, + "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]" + }, + { + "name": "programid", + "type": "int", + "doc": "Program ID: encodes either public, collab, or caltech mode" + }, + { + "name": "jd", + "type": "double", + "doc": "Observation Julian date at start of exposure [days]" + }, + { + "name": "forcediffimflux", + "type": ["null", "float"], + "default": null, + "doc": "Forced difference image PSF-fit flux [DN]" + }, + { + "name": "forcediffimfluxunc", + "type": ["null", "float"], + "default": null, + "doc": "1-sigma uncertainty in forcediffimflux [DN]" + }, + { + "name": "procstatus", + "type": ["null", "string"], + "default": null, + "doc": "Forced photometry processing status codes (0 => no warnings); see documentation" + }, + { + "name": "distnr", + "type": ["null", "float"], + "default": null, + "doc": "distance to nearest source in reference image PSF-catalog [arcsec]" + }, + { + "name": "ranr", + "type": "double", + "doc": "Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg]" + }, + { + "name": "decnr", + "type": "double", + "doc": "Declination of nearest source in reference image PSF-catalog; J2000 [deg]" + }, + { + "name": "magnr", + "type": ["null", "float"], + "default": null, + "doc": "magnitude of nearest source in reference image PSF-catalog [mag]" + }, + { + "name": "sigmagnr", + "type": ["null", "float"], + "default": null, + "doc": "1-sigma uncertainty in magnr [mag]" + }, + { + "name": "chinr", + "type": ["null", "float"], + "default": null, + "doc": "DAOPhot chi parameter of nearest source in reference image PSF-catalog" + }, + { + "name": "sharpnr", + "type": ["null", "float"], + "default": null, + "doc": "DAOPhot sharp parameter of nearest source in reference image PSF-catalog" + } + ] } From 3de57ddf8a53685eff4c9466a757e7bf41da5951 Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 12:40:36 -0300 Subject: [PATCH 14/20] fix: forced photometry extra fields as columns only consider extra_fields from forced photometry and not alert --- libs/db-plugins/db_plugins/db/sql/models.py | 105 +++----------------- 1 file changed, 14 insertions(+), 91 deletions(-) diff --git a/libs/db-plugins/db_plugins/db/sql/models.py b/libs/db-plugins/db_plugins/db/sql/models.py index a00bc0ae0..f7489c69b 100755 --- a/libs/db-plugins/db_plugins/db/sql/models.py +++ b/libs/db-plugins/db_plugins/db/sql/models.py @@ -285,106 +285,29 @@ class ForcedPhotometry(Base): parent_candid = Column(String) has_stamp = Column(Boolean, nullable=False) # extra fields - diffmaglim = Column(Float) - pdiffimfilename = Column(String) - programpi = Column(String) - programid = Column(Integer) - tblid = Column(Integer) - nid = Column(Integer) - rcid = Column(Integer) field = Column(Integer) - xpos = Column(Float) - ypos = Column(Float) - chipsf = Column(Float) - magap = Column(Float) - sigmagap = Column(Float) - distnr = Column(Float) - magnr = Column(Float) - sigmagnr = Column(Float) - chinr = Column(Float) - sharpnr = Column(Float) - sky = Column(Float) - magdiff = Column(Float) - fwhm = Column(Float) - classtar = Column(Float) - mindtoedge = Column(Float) - magfromlim = Column(Float) - seeratio = Column(Float) - aimage = Column(Float) - bimage = Column(Float) - aimagerat = Column(Float) - bimagerat = Column(Float) - elong = Column(Float) - nneg = Column(Integer) - nbad = Column(Integer) - rb = Column(Float) - ssdistnr = Column(Float) - ssmagnr = Column(Float) - ssnamenr = Column(String) - sumrat = Column(Float) - magapbig = Column(Float) - sigmagapbig = Column(Float) - ranr = Column(Float) - decnr = Column(Float) - sgmag1 = Column(Float) - srmag1 = Column(Float) - simag1 = Column(Float) - szmag1 = Column(Float) - sgscore1 = Column(Float) - distpsnr1 = Column(Float) - ndethist = Column(Integer) - ncovhist = Column(Integer) - jdstarthist = Column(Float) - jdendhist = Column(Float) - scorr = Column(Float) - tooflag = Column(Integer) - objectidps1 = Column(Float) - objectidps2 = Column(Float) - sgmag2 = Column(Float) - srmag2 = Column(Float) - simag2 = Column(Float) - szmag2 = Column(Float) - sgscore2 = Column(Float) - distpsnr2 = Column(Float) - objectidps3 = Column(Float) - sgmag3 = Column(Float) - srmag3 = Column(Float) - simag3 = Column(Float) - szmag3 = Column(Float) - sgscore3 = Column(Float) - distpsnr3 = Column(Float) - nmtchps = Column(Integer) - rfid = Column(Integer) - jdstartref = Column(Float) - jdendref = Column(Float) - nframesref = Column(Integer) - rbversion = Column(String) - dsnrms = Column(Float) - ssnrms = Column(Float) - dsdiff = Column(Float) + rcid = Column(Integer) + rfid = Column(BigInteger) + sciinpseeing = Column(Float) + scibckgnd = Column(Float) + scisigpix = Column(Float) magzpsci = Column(Float) magzpsciunc = Column(Float) magzpscirms = Column(Float) - nmatches = Column(Integer) clrcoeff = Column(Float) clrcounc = Column(Float) - zpclrcov = Column(Float) - zpmed = Column(Float) - clrmed = Column(Float) - clrrms = Column(Float) - neargaia = Column(Float) - neargaiabright = Column(Float) - maggaia = Column(Float) - maggaiabright = Column(Float) exptime = Column(Float) - drb = Column(Float) - drbversion = Column(String) - sciinpseeing = Column(Float) - scibckgnd = Column(Float) - scisigpix = Column(Float) - adpctdif1 = Column(Float) adpctdif2 = Column(Float) + diffmaglim = Column(Float) + programid = Column(Integer) procstatus = Column(String) + distnr = Column(Float) + ranr = Column(Float(precision=53)) + decnr = Column(Float(precision=53)) + magnr = Column(Float) + sigmagnr = Column(Float) + chinr = Column(Float) + sharpnr = Column(Float) __table_args__ = ( Index("ix_forced_photometry_oid", "oid", postgresql_using="hash"), From 90373b4f4091bdcdba3ce44566d4e0b18d0cae80 Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 12:41:29 -0300 Subject: [PATCH 15/20] fix: forced_photometry extra fields --- .../prv_candidates_step/core/strategy/ztf.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py index d9fae08fc..242397352 100644 --- a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py +++ b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py @@ -1,7 +1,6 @@ import copy import numpy as np import pickle -from dataclasses import asdict from survey_parser_plugins.core import SurveyParser from survey_parser_plugins.core.generic import GenericNonDetection @@ -122,15 +121,18 @@ def parse_message(cls, forced_photometry: dict, alert: dict) -> dict: model = cls._Model(**generic, stamps=stamps, extra_fields=extra_fields) model = model.to_dict() + # start with the alert extra fields + # then update with the forced photometry extra fields + # so that the forced photometry extra fields take precedence + final_extra_fields = alert["extra_fields"] + final_extra_fields.update(model["extra_fields"]) model.update( { "aid": alert["aid"], - "has_stamp": False, # ? + "has_stamp": False, "forced": True, "parent_candid": alert["candid"], - "extra_fields": { - **alert["extra_fields"], - }, + "extra_fields": final_extra_fields, } ) model.pop("stamps") From 33f6b7623707a5210c4266d064b60a7a4fcd12cb Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 14:52:32 -0300 Subject: [PATCH 16/20] tests: add update case and assertions --- scribe/tests/integration/test_step_sql.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scribe/tests/integration/test_step_sql.py b/scribe/tests/integration/test_step_sql.py index 559bd05b4..e84a2e34f 100644 --- a/scribe/tests/integration/test_step_sql.py +++ b/scribe/tests/integration/test_step_sql.py @@ -30,8 +30,8 @@ "enable.partition.eof": True, "auto.offset.reset": "beginning", }, - "NUM_MESSAGES": 2, - "TIMEOUT": 10, + "NUM_MESSAGES": 3, + "TIMEOUT": 1, } PRODUCER_CONFIG = { @@ -696,8 +696,11 @@ def test_forced_photometry_insertion(self): } self.producer.produce({"payload": json.dumps(command)}) + command["data"]["mag"] = 20 self.producer.produce({"payload": json.dumps(command)}) - self.producer.producer.flush(1) + command["data"]["pid"] = 420 + self.producer.produce({"payload": json.dumps(command)}) + self.producer.producer.flush() self.step.start() with self.db.session() as session: @@ -708,4 +711,7 @@ def test_forced_photometry_insertion(self): """ ) ) - assert True + result = result.fetchall() + assert len(result) == 2 + assert result[0][0] == 420 + assert result[1][0] == 423432 From d1c2233d35784f54b68d6e161d6bd552ad6f411a Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 14:53:51 -0300 Subject: [PATCH 17/20] refactor: use execute multiple values instead of insert with a list As discussed on https://docs.sqlalchemy.org/en/20/tutorial/dbapi_transactions.html#tutorial-multiple-parameters --- scribe/mongo_scribe/sql/command/commands.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/scribe/mongo_scribe/sql/command/commands.py b/scribe/mongo_scribe/sql/command/commands.py index 8cff6054a..72babf9c7 100644 --- a/scribe/mongo_scribe/sql/command/commands.py +++ b/scribe/mongo_scribe/sql/command/commands.py @@ -103,7 +103,9 @@ def _format_data(self, data: Dict): new_data = {k: v for k, v in new_data.items() if k not in exclude} new_data["step_id_corr"] = data.get("step_id_corr", "ALeRCE v3") new_data["parent_candid"] = ( - int(data["parent_candid"]) if data["parent_candid"] != "None" else None + int(data["parent_candid"]) + if data["parent_candid"] != "None" + else None ) new_data["fid"] = fid_map[new_data["fid"]] @@ -152,11 +154,12 @@ def _format_data(self, data: Dict): def db_operation(session: Session, data: List): unique = {(el["pid"], el["oid"]): el for el in data} unique = list(unique.values()) - fp_stmt = insert(ForcedPhotometry).values(unique) + statement = insert(ForcedPhotometry) return session.connection().execute( - fp_stmt.on_conflict_do_update( - constraint="forced_photometry_pkey", set_=fp_stmt.excluded - ) + statement.on_conflict_do_update( + constraint="forced_photometry_pkey", set_=statement.excluded + ), + unique, ) @@ -294,12 +297,15 @@ def _format_data(self, data): ] parsed.sort(key=lambda e: e["probability"], reverse=True) parsed = [{**el, "ranking": i + 1} for i, el in enumerate(parsed)] - return [{**el, "oid": oid} for el in parsed for oid in self.criteria["oid"]] + return [ + {**el, "oid": oid} for el in parsed for oid in self.criteria["oid"] + ] @staticmethod def db_operation(session: Session, data: List): unique = { - (el["oid"], el["classifier_name"], el["class_name"]): el for el in data + (el["oid"], el["classifier_name"], el["class_name"]): el + for el in data } unique = list(unique.values()) insert_stmt = insert(Probability).values(unique) From 9e8ce4f9d66742183512c54eb2e6a0cfbc99634d Mon Sep 17 00:00:00 2001 From: "@alerceadmin" Date: Wed, 6 Dec 2023 18:21:07 +0000 Subject: [PATCH 18/20] chore: update version --- alert_archiving_step/pyproject.toml | 2 +- charts/alert_archiving_step/Chart.yaml | 4 ++-- charts/correction_step/Chart.yaml | 4 ++-- charts/early_classification_step/Chart.yaml | 4 ++-- charts/feature_step/Chart.yaml | 4 ++-- charts/lc_classification_step/Chart.yaml | 4 ++-- charts/lightcurve-step/Chart.yaml | 4 ++-- charts/magstats_step/Chart.yaml | 4 ++-- charts/metadata_step/Chart.yaml | 4 ++-- charts/prv_candidates_step/Chart.yaml | 4 ++-- charts/s3_step/Chart.yaml | 4 ++-- charts/scribe/Chart.yaml | 4 ++-- charts/sorting_hat_step/Chart.yaml | 4 ++-- charts/watchlist_step/Chart.yaml | 4 ++-- charts/xmatch_step/Chart.yaml | 4 ++-- correction_step/pyproject.toml | 2 +- early_classification_step/pyproject.toml | 4 +++- feature_step/pyproject.toml | 2 +- lc_classification_step/pyproject.toml | 2 +- lightcurve-step/pyproject.toml | 2 +- magstats_step/pyproject.toml | 2 +- metadata_step/pyproject.toml | 2 +- prv_candidates_step/pyproject.toml | 2 +- s3_step/pyproject.toml | 2 +- scribe/pyproject.toml | 2 +- sorting_hat_step/pyproject.toml | 2 +- watchlist_step/pyproject.toml | 2 +- xmatch_step/pyproject.toml | 2 +- 28 files changed, 44 insertions(+), 42 deletions(-) diff --git a/alert_archiving_step/pyproject.toml b/alert_archiving_step/pyproject.toml index 2c4a8b0fd..753ee3b18 100644 --- a/alert_archiving_step/pyproject.toml +++ b/alert_archiving_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "alert-archiving-step" -version = "23.12.11a1" +version = "23.12.11a2" description = "Group alerts and upload to S3 by date" authors = ["ALeRCE"] readme = "README.md" diff --git a/charts/alert_archiving_step/Chart.yaml b/charts/alert_archiving_step/Chart.yaml index 23e24924d..0910c58f4 100644 --- a/charts/alert_archiving_step/Chart.yaml +++ b/charts/alert_archiving_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: A Helm chart for Kubernetes name: alert-archive-step type: application -version: 1.2.195 +version: 1.2.196 diff --git a/charts/correction_step/Chart.yaml b/charts/correction_step/Chart.yaml index a1c304ecb..a2c7ae169 100644 --- a/charts/correction_step/Chart.yaml +++ b/charts/correction_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: Correction step chart name: correction-step type: application -version: 2.0.129 +version: 2.0.130 diff --git a/charts/early_classification_step/Chart.yaml b/charts/early_classification_step/Chart.yaml index fb7aabaf1..aae024779 100644 --- a/charts/early_classification_step/Chart.yaml +++ b/charts/early_classification_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: A Helm chart for Kubernetes name: early-classifier type: application -version: 1.2.306 +version: 1.2.307 diff --git a/charts/feature_step/Chart.yaml b/charts/feature_step/Chart.yaml index 9c627745b..3a0855ca9 100644 --- a/charts/feature_step/Chart.yaml +++ b/charts/feature_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: Step for feature calculation name: feature-step type: application -version: 2.0.129 +version: 2.0.130 diff --git a/charts/lc_classification_step/Chart.yaml b/charts/lc_classification_step/Chart.yaml index e78b34d46..2b6860f1c 100644 --- a/charts/lc_classification_step/Chart.yaml +++ b/charts/lc_classification_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: Lightcurve classifier step name: lc-classifier-step type: application -version: 5.0.129 +version: 5.0.130 diff --git a/charts/lightcurve-step/Chart.yaml b/charts/lightcurve-step/Chart.yaml index 99a6c2291..49f444d71 100644 --- a/charts/lightcurve-step/Chart.yaml +++ b/charts/lightcurve-step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: Full lightcurve provider step name: lightcurve-step type: application -version: 2.0.129 +version: 2.0.130 diff --git a/charts/magstats_step/Chart.yaml b/charts/magstats_step/Chart.yaml index e10a6fe44..211794aa0 100644 --- a/charts/magstats_step/Chart.yaml +++ b/charts/magstats_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: A Helm chart for Kubernetes name: magstats-step type: application -version: 2.0.129 +version: 2.0.130 diff --git a/charts/metadata_step/Chart.yaml b/charts/metadata_step/Chart.yaml index 0ba621460..3339b4a68 100644 --- a/charts/metadata_step/Chart.yaml +++ b/charts/metadata_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: A Helm chart for Kubernetes name: metadata-step type: application -version: 2.0.129 +version: 2.0.130 diff --git a/charts/prv_candidates_step/Chart.yaml b/charts/prv_candidates_step/Chart.yaml index dc0fd432e..cae625372 100644 --- a/charts/prv_candidates_step/Chart.yaml +++ b/charts/prv_candidates_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: Previous candidates processor step name: prv-candidates type: application -version: 2.0.129 +version: 2.0.130 diff --git a/charts/s3_step/Chart.yaml b/charts/s3_step/Chart.yaml index 57becfff9..61049383e 100644 --- a/charts/s3_step/Chart.yaml +++ b/charts/s3_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: A Helm chart for Kubernetes name: s3-step type: application -version: 1.4.18 +version: 1.4.19 diff --git a/charts/scribe/Chart.yaml b/charts/scribe/Chart.yaml index 6c055fcf6..a0f6caa0e 100644 --- a/charts/scribe/Chart.yaml +++ b/charts/scribe/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 2.0.129 +version: 2.0.130 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 diff --git a/charts/sorting_hat_step/Chart.yaml b/charts/sorting_hat_step/Chart.yaml index 7c72028e8..cab6325c2 100644 --- a/charts/sorting_hat_step/Chart.yaml +++ b/charts/sorting_hat_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: Sorting Hat deployment chart name: sorting-hat type: application -version: 3.0.129 +version: 3.0.130 diff --git a/charts/watchlist_step/Chart.yaml b/charts/watchlist_step/Chart.yaml index 95bed3c8e..0aab66774 100644 --- a/charts/watchlist_step/Chart.yaml +++ b/charts/watchlist_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: A Helm chart for Kubernetes name: watchlist-step type: application -version: 1.2.302 +version: 1.2.303 diff --git a/charts/xmatch_step/Chart.yaml b/charts/xmatch_step/Chart.yaml index 9ecbb4145..0c1f1f4f1 100644 --- a/charts/xmatch_step/Chart.yaml +++ b/charts/xmatch_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.11a1 +appVersion: 23.12.11a2 description: A Helm chart for Kubernetes name: xmatch-step type: application -version: 3.0.129 +version: 3.0.130 diff --git a/correction_step/pyproject.toml b/correction_step/pyproject.toml index 0949c5e28..8cbcccd88 100644 --- a/correction_step/pyproject.toml +++ b/correction_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "correction" -version = "23.12.11a1" +version = "23.12.11a2" description = "Correction library for ALeRCE pipeline." authors = ["ALeRCE Broker "] readme = "README.md" diff --git a/early_classification_step/pyproject.toml b/early_classification_step/pyproject.toml index 25d4c3981..a1c689c16 100644 --- a/early_classification_step/pyproject.toml +++ b/early_classification_step/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "early_classification_step" -version = "23.12.11a1" +version = "23.12.11a2" description = "Early Classification Step" authors = [] readme = "README.md" @@ -288,6 +288,7 @@ line-length = 88 + [tool.poetry.group.test.dependencies] @@ -561,6 +562,7 @@ pytest-docker = "^1.0.1" + [tool.poetry.group.model.dependencies] diff --git a/feature_step/pyproject.toml b/feature_step/pyproject.toml index de29d8392..2f0655635 100644 --- a/feature_step/pyproject.toml +++ b/feature_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "feature-step" -version = "23.12.11a1" +version = "23.12.11a2" description = "" authors = [] readme = "README.md" diff --git a/lc_classification_step/pyproject.toml b/lc_classification_step/pyproject.toml index a2bc98f0b..49838cc85 100644 --- a/lc_classification_step/pyproject.toml +++ b/lc_classification_step/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "lc_classification_step" -version = "23.12.11a1" +version = "23.12.11a2" description = "LC Classification Step" authors = [] readme = "README.md" diff --git a/lightcurve-step/pyproject.toml b/lightcurve-step/pyproject.toml index d6e68f616..c1087834a 100644 --- a/lightcurve-step/pyproject.toml +++ b/lightcurve-step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "lightcurve-step" -version = "23.12.11a1" +version = "23.12.11a2" description = "Lightcurve Step" authors = [] readme = "README.md" diff --git a/magstats_step/pyproject.toml b/magstats_step/pyproject.toml index 05f3d00a1..c71ff04db 100644 --- a/magstats_step/pyproject.toml +++ b/magstats_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "magstats-step" -version = "23.12.11a1" +version = "23.12.11a2" description = "" authors = ["ASHuenchuleo ", "Pablo Castellanos"] readme = "README.md" diff --git a/metadata_step/pyproject.toml b/metadata_step/pyproject.toml index 18d7043d0..2cffc0227 100644 --- a/metadata_step/pyproject.toml +++ b/metadata_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metadata-step" -version = "23.12.11a1" +version = "23.12.11a2" description = "" authors = ["Pedro Gallardo "] readme = "README.md" diff --git a/prv_candidates_step/pyproject.toml b/prv_candidates_step/pyproject.toml index 411f1e1aa..a5bb3c859 100644 --- a/prv_candidates_step/pyproject.toml +++ b/prv_candidates_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "prv-candidates-step" -version = "23.12.11a1" +version = "23.12.11a2" description = "Previous Detections Step" authors = [] readme = "README.md" diff --git a/s3_step/pyproject.toml b/s3_step/pyproject.toml index 620014a5d..b09684114 100644 --- a/s3_step/pyproject.toml +++ b/s3_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "s3-step" -version = "23.12.11a1" +version = "23.12.11a2" description = "S3 step" authors = [] readme = "README.md" diff --git a/scribe/pyproject.toml b/scribe/pyproject.toml index 68a659a14..12b761d8b 100644 --- a/scribe/pyproject.toml +++ b/scribe/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "scribe" -version = "23.12.11a1" +version = "23.12.11a2" description = "Mongo scribe" authors = [] readme = "README.md" diff --git a/sorting_hat_step/pyproject.toml b/sorting_hat_step/pyproject.toml index 5d6a077ed..2ed4524d0 100644 --- a/sorting_hat_step/pyproject.toml +++ b/sorting_hat_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "sorting-hat-step" -version = "23.12.11a1" +version = "23.12.11a2" description = "Sorting hat step" authors = [] readme = "README.md" diff --git a/watchlist_step/pyproject.toml b/watchlist_step/pyproject.toml index 6714188d3..663ec90f1 100644 --- a/watchlist_step/pyproject.toml +++ b/watchlist_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "watchlist-step" -version = "23.12.11a1" +version = "23.12.11a2" description = "Watchlist Step" authors = [] readme = "README.md" diff --git a/xmatch_step/pyproject.toml b/xmatch_step/pyproject.toml index f84dc2e96..d588ea65d 100644 --- a/xmatch_step/pyproject.toml +++ b/xmatch_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "xmatch-step" -version = "23.12.11a1" +version = "23.12.11a2" description = "xmatch step" authors = [] readme = "README.md" From 10f285b0114ab6707c72a51d003c92016322c82c Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 6 Dec 2023 15:30:50 -0300 Subject: [PATCH 19/20] refactor: keep -999999 value (as ztf does) instead of nan --- .../prv_candidates_step/core/strategy/ztf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py index 242397352..725d91e1d 100644 --- a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py +++ b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py @@ -85,11 +85,11 @@ def __calculate_mag(cls, data): e_mag = 1.0857 * forcediffimfluxunc / np.abs(forcediffimflux) if np.isclose(data["forcediffimflux"], -99999.0): - mag = np.nan - e_mag = np.nan + mag = -999999 + e_mag = -999999 if np.isclose(data["forcediffimfluxunc"], -99999.0): - e_mag = np.nan + e_mag = -999999 return mag, e_mag From 86eee1cb7a4b1aea1b80107903892f2309ef4455 Mon Sep 17 00:00:00 2001 From: ale-munozarancibia Date: Wed, 6 Dec 2023 16:14:28 -0300 Subject: [PATCH 20/20] Fixed -99999 fluxes to _ZERO_MAG magnitudes and inf corrected magnitudes --- correction_step/correction/core/strategy/ztf.py | 8 ++++++++ .../prv_candidates_step/core/strategy/ztf.py | 12 +++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/correction_step/correction/core/strategy/ztf.py b/correction_step/correction/core/strategy/ztf.py index 0eded3b72..7cc38538d 100644 --- a/correction_step/correction/core/strategy/ztf.py +++ b/correction_step/correction/core/strategy/ztf.py @@ -9,6 +9,8 @@ SHARPNR_MAX = 0.1 SHARPNR_MIN = -0.13 +_ZERO_MAG = 100.0 + def is_corrected(detections: pd.DataFrame) -> pd.Series: """Whether the nearest source is closer than `DISTANCE_THRESHOLD`""" @@ -70,6 +72,12 @@ def correct(detections: pd.DataFrame) -> pd.DataFrame: e_mag_corr = np.where(aux4 < 0, np.inf, np.sqrt(aux4) / aux3) e_mag_corr_ext = aux2 * detections["e_mag"] / aux3 + mask = np.array(np.isclose(detections["mag"], _ZERO_MAG)) + mag_corr[mask] = np.inf + mask = np.array(np.isclose(detections["mag"], _ZERO_MAG) or np.isclose(detections["e_mag"], _ZERO_MAG)) + e_mag_corr[mask] = np.inf + e_mag_corr_ext[mask] = np.inf + return pd.DataFrame({"mag_corr": mag_corr, "e_mag_corr": e_mag_corr, "e_mag_corr_ext": e_mag_corr_ext}) diff --git a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py index 725d91e1d..e070e3ace 100644 --- a/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py +++ b/prv_candidates_step/prv_candidates_step/core/strategy/ztf.py @@ -7,6 +7,8 @@ from survey_parser_plugins.core.mapper import Mapper from survey_parser_plugins.parsers import ZTFParser +_ZERO_MAG = 100.0 + def prv_non_detections_mapper() -> dict: mapping = copy.deepcopy(ZTFParser._mapping) @@ -84,12 +86,12 @@ def __calculate_mag(cls, data): mag = -2.5 * np.log10(np.abs(forcediffimflux)) + 23.9 e_mag = 1.0857 * forcediffimfluxunc / np.abs(forcediffimflux) - if np.isclose(data["forcediffimflux"], -99999.0): - mag = -999999 - e_mag = -999999 + if np.isclose(data["forcediffimflux"], _ZERO_MAG): + mag = _ZERO_MAG + e_mag = _ZERO_MAG - if np.isclose(data["forcediffimfluxunc"], -99999.0): - e_mag = -999999 + if np.isclose(data["forcediffimfluxunc"], _ZERO_MAG): + e_mag = _ZERO_MAG return mag, e_mag