diff --git a/alert_archiving_step/pyproject.toml b/alert_archiving_step/pyproject.toml index 158777546..079820378 100644 --- a/alert_archiving_step/pyproject.toml +++ b/alert_archiving_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "alert-archiving-step" -version = "23.12.26a5" +version = "23.12.26a8" description = "Group alerts and upload to S3 by date" authors = ["ALeRCE"] readme = "README.md" diff --git a/charts/alert_archiving_step/Chart.yaml b/charts/alert_archiving_step/Chart.yaml index 72ef20bc2..850a6fac7 100644 --- a/charts/alert_archiving_step/Chart.yaml +++ b/charts/alert_archiving_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: A Helm chart for Kubernetes name: alert-archive-step type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/correction_step/Chart.yaml b/charts/correction_step/Chart.yaml index e820dd209..eb0ac5b25 100644 --- a/charts/correction_step/Chart.yaml +++ b/charts/correction_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: Correction step chart name: correction-step type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/early_classification_step/Chart.yaml b/charts/early_classification_step/Chart.yaml index a1751b914..1eade7a47 100644 --- a/charts/early_classification_step/Chart.yaml +++ b/charts/early_classification_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: A Helm chart for Kubernetes name: early-classifier type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/feature_step/Chart.yaml b/charts/feature_step/Chart.yaml index 9f4add534..83d18aa7b 100644 --- a/charts/feature_step/Chart.yaml +++ b/charts/feature_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: Step for feature calculation name: feature-step type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/lc_classification_step/Chart.yaml b/charts/lc_classification_step/Chart.yaml index d887a5562..5808f2097 100644 --- a/charts/lc_classification_step/Chart.yaml +++ b/charts/lc_classification_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: Lightcurve classifier step name: lc-classifier-step type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/lightcurve-step/Chart.yaml b/charts/lightcurve-step/Chart.yaml index 34af2290a..5c37f329e 100644 --- a/charts/lightcurve-step/Chart.yaml +++ b/charts/lightcurve-step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: Full lightcurve provider step name: lightcurve-step type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/magstats_step/Chart.yaml b/charts/magstats_step/Chart.yaml index 8b41ca02a..9689b333e 100644 --- a/charts/magstats_step/Chart.yaml +++ b/charts/magstats_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: A Helm chart for Kubernetes name: magstats-step type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/metadata_step/Chart.yaml b/charts/metadata_step/Chart.yaml index 4b1a77b68..e46893df0 100644 --- a/charts/metadata_step/Chart.yaml +++ b/charts/metadata_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: A Helm chart for Kubernetes name: metadata-step type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/prv_candidates_step/Chart.yaml b/charts/prv_candidates_step/Chart.yaml index 415d5fdf9..49953c1d9 100644 --- a/charts/prv_candidates_step/Chart.yaml +++ b/charts/prv_candidates_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: Previous candidates processor step name: prv-candidates type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/s3_step/Chart.yaml b/charts/s3_step/Chart.yaml index 7d595de73..3e2d56a15 100644 --- a/charts/s3_step/Chart.yaml +++ b/charts/s3_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: A Helm chart for Kubernetes name: s3-step type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/scribe/Chart.yaml b/charts/scribe/Chart.yaml index 266c441b0..3e6687883 100644 --- a/charts/scribe/Chart.yaml +++ b/charts/scribe/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 23.12.26-5 +version: 23.12.26-8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 diff --git a/charts/sorting_hat_step/Chart.yaml b/charts/sorting_hat_step/Chart.yaml index 7b355d473..816031a12 100644 --- a/charts/sorting_hat_step/Chart.yaml +++ b/charts/sorting_hat_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: Sorting Hat deployment chart name: sorting-hat type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/watchlist_step/Chart.yaml b/charts/watchlist_step/Chart.yaml index 8b48cf6cc..3a83199b0 100644 --- a/charts/watchlist_step/Chart.yaml +++ b/charts/watchlist_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: A Helm chart for Kubernetes name: watchlist-step type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/charts/xmatch_step/Chart.yaml b/charts/xmatch_step/Chart.yaml index 3bf74fa92..460412da9 100644 --- a/charts/xmatch_step/Chart.yaml +++ b/charts/xmatch_step/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 23.12.26a5 +appVersion: 23.12.26a8 description: A Helm chart for Kubernetes name: xmatch-step type: application -version: 23.12.26-5 +version: 23.12.26-8 diff --git a/correction_step/pyproject.toml b/correction_step/pyproject.toml index eac439c96..5406cda43 100644 --- a/correction_step/pyproject.toml +++ b/correction_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "correction" -version = "23.12.26a5" +version = "23.12.26a8" description = "Correction library for ALeRCE pipeline." authors = ["ALeRCE Broker "] readme = "README.md" diff --git a/early_classification_step/pyproject.toml b/early_classification_step/pyproject.toml index 1bf57fac2..a08320c5e 100644 --- a/early_classification_step/pyproject.toml +++ b/early_classification_step/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "early_classification_step" -version = "23.12.26a5" +version = "23.12.26a8" description = "Early Classification Step" authors = [] readme = "README.md" @@ -59,6 +59,9 @@ line-length = 88 + + + @@ -105,6 +108,9 @@ pytest-docker = "^1.0.1" + + + diff --git a/feature_step/features/core/_base.py b/feature_step/features/core/_base.py index ce19da6b5..6bc86d39c 100644 --- a/feature_step/features/core/_base.py +++ b/feature_step/features/core/_base.py @@ -173,7 +173,7 @@ def _discard_detections(self): self.detections.not_enough(self.MIN_DETS_FID, by_fid=True) def _create_xmatches(self, xmatches: list[dict]) -> pd.DataFrame: - """Ensures cross-matches contain `aid` in detections and selects required columns.""" + """Ensures cross-matches contain `oid` in detections and selects required columns.""" def expand_catalogues(xm): return { @@ -188,12 +188,12 @@ def get_required_columns(): ] xmatches = [ - {"aid": xm["aid"]} | expand_catalogues(xm) for xm in xmatches + {"oid": xm["oid"]} | expand_catalogues(xm) for xm in xmatches ] xmatches = pd.DataFrame( - xmatches, columns=["aid"] + get_required_columns() + xmatches, columns=["oid"] + get_required_columns() ) - xmatches = xmatches.rename(columns={"aid": "id"}).drop_duplicates( + xmatches = xmatches.rename(columns={"oid": "id"}).drop_duplicates( subset=["id"] ) return xmatches[xmatches["id"].isin(self.detections.ids())].set_index( diff --git a/feature_step/features/core/elasticc.py b/feature_step/features/core/elasticc.py index d0b51e0b1..016707884 100644 --- a/feature_step/features/core/elasticc.py +++ b/feature_step/features/core/elasticc.py @@ -49,7 +49,7 @@ def generate_features(self): def _create_lightcurve_dataframe(self, detections: List[dict]): self._rename_detections_columns(detections) - return pd.DataFrame.from_records(detections).set_index("aid") + return pd.DataFrame.from_records(detections).set_index("oid") def _rename_detections_columns(self, detections: List[dict]): for det in detections: diff --git a/feature_step/features/core/handlers/_base.py b/feature_step/features/core/handlers/_base.py index 8269ff0bb..3bfe5d618 100644 --- a/feature_step/features/core/handlers/_base.py +++ b/feature_step/features/core/handlers/_base.py @@ -17,7 +17,7 @@ class BaseHandler(abc.ABC): This provides the general base methods for extracting statistics and applying functions to the alerts considered. - Alerts require at the very least fields `sid`, `fid`, `mjd` and `aid` (the latter will be renamed to `id`). + Alerts require at the very least fields `sid`, `fid`, `mjd` and `oid` (the latter will be renamed to `id`). Usually, a subclass will define additional fields that are required, but missing one of the aforementioned fields will result in an initialization error, unless the initialization methods are adapted as well. These always required fields are defined the class attribute `COLUMNS`. @@ -73,7 +73,7 @@ def __init__( ) except KeyError: # extra_fields is not present self._alerts = pd.DataFrame.from_records(alerts) - self._alerts = self._alerts.rename(columns={"aid": "id"}) + self._alerts = self._alerts.rename(columns={"oid": "id"}) if self._alerts.size == 0: index = ( {self.INDEX} diff --git a/feature_step/features/core/handlers/detections.py b/feature_step/features/core/handlers/detections.py index 337358fc1..e2035f9a8 100644 --- a/feature_step/features/core/handlers/detections.py +++ b/feature_step/features/core/handlers/detections.py @@ -11,7 +11,7 @@ class DetectionsHandler(BaseHandler): Indexed by `candid`. - Criteria for uniqueness is based on `id` (`aid` or `oid`, depending on use of `legacy`), `fid` and `mjd`. + Criteria for uniqueness is based on `id` (`oid`, depending on use of `legacy`), `fid` and `mjd`. Required fields are: `id`, `sid`, `fid`, `mjd`, `mag` and `e_mag`. diff --git a/feature_step/features/core/handlers/non_detections.py b/feature_step/features/core/handlers/non_detections.py index f9efbf8be..c025877cc 100644 --- a/feature_step/features/core/handlers/non_detections.py +++ b/feature_step/features/core/handlers/non_detections.py @@ -11,7 +11,7 @@ class NonDetectionsHandler(BaseHandler): """Class for handling non-detections. - Criteria for uniqueness is based on `id` (`aid` or `oid`, depending on use of `legacy`), `fid` and `mjd`. + Criteria for uniqueness is based on `id` (or `oid`, depending on use of `legacy`), `fid` and `mjd`. Required fields are `id`, `sid`, `fid`, `mjd` and `diffmaglim`. diff --git a/feature_step/features/core/utils/decorators.py b/feature_step/features/core/utils/decorators.py index 1aac51ee2..046655a0f 100644 --- a/feature_step/features/core/utils/decorators.py +++ b/feature_step/features/core/utils/decorators.py @@ -51,7 +51,7 @@ def wrapper(self, *args, **kwargs): def fill_in_every_fid(counters: str = None): - """Decorated method must produce a multi-indexed data frame with two levels, `aid` and `fid` (in that order)""" + """Decorated method must produce a multi-indexed data frame with two levels, `oid` and `fid` (in that order)""" def decorator(method): @functools.wraps(method) diff --git a/feature_step/features/core/utils/functions.py b/feature_step/features/core/utils/functions.py index 600da158b..73d00e4ea 100644 --- a/feature_step/features/core/utils/functions.py +++ b/feature_step/features/core/utils/functions.py @@ -15,8 +15,8 @@ def fill_index( raise ValueError( f"DataFrame has MultiIndex with {df.index.names}. Requested filling: {check}" ) - aids = df.index.get_level_values("id").unique() - values = [aids] + [kwargs[k] for k in df.index.names if k != "id"] + oids = df.index.get_level_values("id").unique() + values = [oids] + [kwargs[k] for k in df.index.names if k != "id"] df = df.reindex(pd.MultiIndex.from_product(values, names=df.index.names)) if counters is None: diff --git a/feature_step/features/core/ztf.py b/feature_step/features/core/ztf.py index a8b0189c8..54914126b 100644 --- a/feature_step/features/core/ztf.py +++ b/feature_step/features/core/ztf.py @@ -93,11 +93,9 @@ def _legacy(cls, detections, non_detections, xmatches, metadata): non_detections["fid"].replace( {v: k for k, v in cls.BANDS_MAPPING.items()}, inplace=True ) - non_detections = non_detections.rename(columns={"oid": "aid"}) if isinstance(xmatches, pd.DataFrame): xmatches = xmatches.reset_index() - xmatches = xmatches.rename(columns={"oid": "aid"}) return detections, non_detections, xmatches diff --git a/feature_step/features/step.py b/feature_step/features/step.py index 0bd1e3aa9..97be0a757 100644 --- a/feature_step/features/step.py +++ b/feature_step/features/step.py @@ -4,6 +4,7 @@ import pandas as pd from apf.core import get_class from apf.core.step import GenericStep +from apf.consumers import KafkaConsumer from features.core.elasticc import ELAsTiCCFeatureExtractor from features.core.ztf import ZTFFeatureExtractor @@ -40,12 +41,8 @@ def __init__( self.config["SCRIBE_PRODUCER_CONFIG"] ) - def produce_to_scribe( - self, messages_aid_oid: dict, features: pd.DataFrame - ): - commands = parse_scribe_payload( - messages_aid_oid, features, self.features_extractor - ) + def produce_to_scribe(self, features: pd.DataFrame): + commands = parse_scribe_payload(features, self.features_extractor) count = 0 flush = False @@ -58,35 +55,29 @@ def produce_to_scribe( ) def pre_produce(self, result: Iterable[Dict[str, Any]] | Dict[str, Any]): - self.set_producer_key_field("aid") + self.set_producer_key_field("oid") return result def execute(self, messages): - detections, non_detections, xmatch, messages_aid_oid = [], [], [], {} + detections, non_detections, xmatch = [], [], [] candids = {} for message in messages: - if not message["aid"] in candids: - candids[message["aid"]] = [] - candids[message["aid"]].extend(message["candid"]) + if not message["oid"] in candids: + candids[message["oid"]] = [] + candids[message["oid"]].extend(message["candid"]) detections.extend(message.get("detections", [])) non_detections.extend(message.get("non_detections", [])) xmatch.append( - {"aid": message["aid"], **(message.get("xmatches", {}) or {})} + {"oid": message["oid"], **(message.get("xmatches", {}) or {})} ) - oids_of_aid = [] - oids_of_aid = [ - message_detection["oid"] - for message_detection in message["detections"] - ] - messages_aid_oid[message["aid"]] = list(set(oids_of_aid)) features_extractor = self.features_extractor( detections, non_detections, xmatch ) features = features_extractor.generate_features() if len(features) > 0: - self.produce_to_scribe(messages_aid_oid, features) + self.produce_to_scribe(features) output = parse_output( features, messages, self.features_extractor, candids @@ -96,3 +87,10 @@ def execute(self, messages): def post_execute(self, result): self.metrics["sid"] = get_sid(result) return result + + def tear_down(self): + if isinstance(self.consumer, KafkaConsumer): + self.consumer.teardown() + else: + self.consumer.__del__() + self.producer.__del__() diff --git a/feature_step/features/utils/parsers.py b/feature_step/features/utils/parsers.py index 78b334668..af1d79cb7 100644 --- a/feature_step/features/utils/parsers.py +++ b/feature_step/features/utils/parsers.py @@ -4,15 +4,11 @@ from features.core.utils.functions import collapse_fid_columns -def parse_scribe_payload( - messages_aid_oid: dict, features: pd.DataFrame, extractor_class -): +def parse_scribe_payload(features: pd.DataFrame, extractor_class): """Create the json with the messages for the scribe produccer fron the features dataframe. It adds the fid and correct the name. - :param messages_aid_oid: a dict that cointains list of oids for each - aid in the step input. - :param features: a dataframe that contains a colum with the aid, and + :param features: a dataframe that contains a colum with the oid, and a column for each feature. :param features_version: a string with the features version used :return: a list of json with Alerce Scribe commands @@ -20,9 +16,7 @@ def parse_scribe_payload( features.replace({np.nan: None, np.inf: None, -np.inf: None}, inplace=True) if extractor_class.NAME == "ztf_lc_features": - return _parse_scribe_payload_ztf( - messages_aid_oid, features, extractor_class - ) + return _parse_scribe_payload_ztf(features, extractor_class) if extractor_class.NAME == "elasticc_lc_features": return _parse_scribe_payload_elasticc(features, extractor_class) else: @@ -41,7 +35,7 @@ def get_fid(feature_name: str): return None commands_list = [] - for aid, features_df in features.iterrows(): + for oid, features_df in features.iterrows(): features_list = [ {"name": name, "fid": get_fid(name), "value": value} for (name, value) in features_df.items() @@ -49,7 +43,7 @@ def get_fid(feature_name: str): command = { "collection": "object", "type": "update_features", - "criteria": {"_id": aid}, + "criteria": {"_id": oid}, "data": { "features_version": extractor_class.VERSION, "features_group": extractor_class.NAME, @@ -62,11 +56,9 @@ def get_fid(feature_name: str): return commands_list -def _parse_scribe_payload_ztf( - messages_aid_oid: dict, features, extractor_class -): +def _parse_scribe_payload_ztf(features, extractor_class): commands_list = [] - for aid, features_df in features.iterrows(): + for oid, features_df in features.iterrows(): FID_MAP = {"g": 1, "r": 2, "gr": 12, "rg": 12} features_list = [ { @@ -79,7 +71,7 @@ def _parse_scribe_payload_ztf( command = { "collection": "object", "type": "update_features", - "criteria": {"_id": aid, "oid": messages_aid_oid[aid]}, + "criteria": {"_id": oid}, "data": { "features_version": extractor_class.VERSION, "features_group": extractor_class.NAME, @@ -100,10 +92,10 @@ def parse_output( ) -> list[dict]: """ Parse output of the step. It uses the input data to extend the schema to - add the features of each object, identified by its aid. + add the features of each object, identified by its oid. :param features: a dataframe with the calculated features, with a column with - the aid and a colum for each feature (with 2 levels one for the feature name + the oid and a colum for each feature (with 2 levels one for the feature name the next with the band of the feature calculated) :param alert_data: the imput for the step :returnn: a list of dictiories, each input object with its data and the @@ -133,19 +125,17 @@ def _parse_output_elasticc(features, alert_data, extractor_class, candids): {np.nan: None, np.inf: None, -np.inf: None}, inplace=True ) for message in alert_data: - aid = message["aid"] - candid = candids[aid] + oid = message["oid"] + candid = candids[oid] try: - features_dict = features.loc[aid].to_dict() + features_dict = features.loc[oid].to_dict() except KeyError: # No feature for the object logger = logging.getLogger("alerce") - logger.info("Could not calculate features of object %s", aid) + logger.info("Could not calculate features of object %s", oid) features_dict = None out_message = { - "aid": aid, + "oid": oid, "candid": candid, - "meanra": message["meanra"], - "meandec": message["meandec"], "detections": message["detections"], "non_detections": message["non_detections"], "xmatches": message["xmatches"], @@ -168,19 +158,17 @@ def _parse_output_ztf(features, alert_data, extractor_class, candids): ) for message in alert_data: - aid = message["aid"] - candid = candids[aid] + oid = message["oid"] + candid = candids[oid] try: - features_dict = features.loc[aid].to_dict() + features_dict = features.loc[oid].to_dict() except KeyError: # No feature for the object logger = logging.getLogger("alerce") - logger.info("Could not calculate features of object %s", aid) + logger.info("Could not calculate features of object %s", oid) features_dict = None out_message = { - "aid": aid, + "oid": oid, "candid": candid, - "meanra": message["meanra"], - "meandec": message["meandec"], "detections": message["detections"], "non_detections": message["non_detections"], "xmatches": message["xmatches"], diff --git a/feature_step/pyproject.toml b/feature_step/pyproject.toml index d58c06187..a9de16dc7 100644 --- a/feature_step/pyproject.toml +++ b/feature_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "feature-step" -version = "23.12.26a5" +version = "23.12.26a8" description = "" authors = [] readme = "README.md" diff --git a/feature_step/tests/data/data_for_unittest.py b/feature_step/tests/data/data_for_unittest.py index d25c8e36a..a8ea1c6a5 100644 --- a/feature_step/tests/data/data_for_unittest.py +++ b/feature_step/tests/data/data_for_unittest.py @@ -13,14 +13,14 @@ features_df_for_parse = pd.DataFrame( [[123, 456, 741, 963, None], [321, 654, 147, 369, 888]], - index=["aid1", "aid2"], + index=["oid1", "oid2"], columns=feature_multi_index_for_parse, ) -features_df_for_parse.index.name = "aid" +features_df_for_parse.index.name = "oid" messages_for_parsing = [ { - "aid": "aid1", + "oid": "oid1", "candid": "candid1", "meanra": 888, "meandec": 999, @@ -29,7 +29,7 @@ "xmatches": {}, }, { - "aid": "aid2", + "oid": "oid2", "candid": "candid2", "meanra": 444, "meandec": 555, @@ -52,14 +52,14 @@ features_df_for_execute = pd.DataFrame( [[123, 456, 741, 963, None], [321, 654, 147, 369, 888]], - index=["aid1", "aid2"], + index=["oid1", "oid2"], columns=feature_multi_index_for_execute, ) -features_df_for_execute.index.name = "aid" +features_df_for_execute.index.name = "oid" messages_for_execute = [ { - "aid": "aid1", + "oid": "oid1", "candid": ["1_candid_aid_1", "2_candid_aid_1"], "meanra": 888, "meandec": 999, @@ -142,7 +142,7 @@ }, }, { - "aid": "aid2", + "oid": "oid2", "candid": ["1_candid_aid_2"], "meanra": 444, "meandec": 555, diff --git a/feature_step/tests/data/elasticc_message_factory.py b/feature_step/tests/data/elasticc_message_factory.py index c51fee1fa..519c8cc83 100644 --- a/feature_step/tests/data/elasticc_message_factory.py +++ b/feature_step/tests/data/elasticc_message_factory.py @@ -30,7 +30,7 @@ def get_extra_fields(): def generate_alert( - aid: str, band: str, num_messages: int, identifier: int, **kwargs + oid: str, band: str, num_messages: int, identifier: int, **kwargs ) -> list[dict]: alerts = [] survey_id = kwargs.get("survey", "LSST") @@ -40,8 +40,8 @@ def generate_alert( extra_fields["diaObject"] = diaObject alert = { "candid": str(random.randint(1000000, 9000000)), - "oid": f"oid{identifier}", - "aid": aid, + "oid": oid, + "aid": f"aid{identifier}", "tid": survey_id, "mjd": random.uniform(59000, 60000), "sid": survey_id, @@ -104,20 +104,20 @@ def generate_input_batch( """ batch = [] for m in range(1, n + 1): - aid = f"AL2X{str(m+offset).zfill(5)}" + oid = f"AL2X{str(m+offset).zfill(5)}" meanra = random.uniform(0, 360) meandec = random.uniform(-90, 90) detections = [] for band in bands: detections.extend( generate_alert( - aid, band, random.randint(6, 10), m, survey=survey + oid, band, random.randint(6, 10), m, survey=survey ) ) - non_det = generate_non_det(aid, random.randint(0, 1), m) + non_det = generate_non_det(oid, random.randint(0, 1), m) xmatch = {} msg = { - "aid": aid, + "oid": oid, "candid": [det["candid"] for det in detections], "meanra": meanra, "meandec": meandec, diff --git a/feature_step/tests/data/features_factory.py b/feature_step/tests/data/features_factory.py index f592211c0..04c34b15d 100644 --- a/feature_step/tests/data/features_factory.py +++ b/feature_step/tests/data/features_factory.py @@ -16,15 +16,15 @@ def generate_features_df(messages: pd.DataFrame): ] feature_multi_index = pd.MultiIndex.from_tuples(feature_columns) - aids = messages["aid"].to_list() + oids = messages["oid"].to_list() features_data = map( lambda x: numpy.random.uniform(1000.0, 9999.9, len(feature_columns)), - aids, + oids, ) features_df = pd.DataFrame( - features_data, index=aids, columns=feature_multi_index + features_data, index=oids, columns=feature_multi_index ) - features_df.index.name = "aid" + features_df.index.name = "oid" return features_df diff --git a/feature_step/tests/data/message_factory.py b/feature_step/tests/data/message_factory.py index 3b147df55..2de7aaf5f 100644 --- a/feature_step/tests/data/message_factory.py +++ b/feature_step/tests/data/message_factory.py @@ -114,15 +114,15 @@ def generate_input_batch(n: int) -> list[dict]: """ batch = [] for m in range(1, n + 1): - aid = f"AL2X{str(m).zfill(5)}" + oid = f"AL2X{str(m).zfill(5)}" meanra = random.uniform(0, 360) meandec = random.uniform(-90, 90) - detections_g = generate_alert_ztf(aid, "g", random.randint(5, 10), m) - detections_r = generate_alert_ztf(aid, "r", random.randint(5, 10), m) - non_det = generate_non_det(aid, random.randint(1, 5), m) - xmatch = get_fake_xmatch(aid, meanra, meandec) + detections_g = generate_alert_ztf(oid, "g", random.randint(5, 10), m) + detections_r = generate_alert_ztf(oid, "r", random.randint(5, 10), m) + non_det = generate_non_det(oid, random.randint(1, 5), m) + xmatch = get_fake_xmatch(oid, meanra, meandec) msg = { - "aid": aid, + "oid": oid, "candid": [det["candid"] for det in detections_g + detections_r], "meanra": meanra, "meandec": meandec, diff --git a/feature_step/tests/unittest/test_elasticc_extractor.py b/feature_step/tests/unittest/test_elasticc_extractor.py index c92fad04e..5061b1a77 100644 --- a/feature_step/tests/unittest/test_elasticc_extractor.py +++ b/feature_step/tests/unittest/test_elasticc_extractor.py @@ -29,7 +29,7 @@ def test_border_case_non_present_bands(self): """ If there are many objects and some object have alerts in some bands but not other that where present in other objects, the result of calculating the features raises - an error + an error. (border case resolved) """ bands_incomplete = ["r", "i", "z", "Y"] bands_incomplete_2 = ["u", "g"] @@ -56,7 +56,7 @@ def test_border_case_non_detected_detections(self): There is a check to evaluate de correctnes of a detection in an object If theres is no correctly detected detection, the features still advance and at some point (calculating the sn features in particular) raises an - error + error. (border case resolved) """ messages = generate_bad_emag_ratio(1, ELASTICC_BANDS) diff --git a/feature_step/tests/unittest/test_elasticc_extractor_parsing.py b/feature_step/tests/unittest/test_elasticc_extractor_parsing.py index 867bded46..392e958ad 100644 --- a/feature_step/tests/unittest/test_elasticc_extractor_parsing.py +++ b/feature_step/tests/unittest/test_elasticc_extractor_parsing.py @@ -13,7 +13,7 @@ def test_create_lightcurve_dataframe(): preprocessor = ElasticcPreprocessor(stream=True) lc_classifier_extractor = ElasticcFeatureExtractor(round=2) detections = generate_alert( - aid="aid1", band="u", num_messages=1, identifier="1" + oid="oid1", band="u", num_messages=1, identifier="1" ) step_extractor = ELAsTiCCFeatureExtractor( preprocessor, lc_classifier_extractor, detections @@ -21,8 +21,8 @@ def test_create_lightcurve_dataframe(): lightcurve_dataframe = step_extractor._create_lightcurve_dataframe( detections ) - assert len(lightcurve_dataframe.loc["aid1"]) == 25 - assert lightcurve_dataframe.iloc[0]["oid"] == "oid1" + assert len(lightcurve_dataframe.loc["oid1"]) == 25 + assert lightcurve_dataframe.iloc[0]["aid"] == "aid1" assert lightcurve_dataframe.iloc[0]["BAND"] == "u" assert lightcurve_dataframe.iloc[0]["FLUXCAL"] >= 15 assert lightcurve_dataframe.iloc[0]["FLUXCAL"] <= 20 @@ -36,7 +36,7 @@ def test_create_metadata_dataframe(): preprocessor = ElasticcPreprocessor(stream=True) lc_classifier_extractor = ElasticcFeatureExtractor(round=2) detections = generate_alert( - aid="aid1", band="u", num_messages=1, identifier="1" + oid="oid1", band="u", num_messages=1, identifier="1" ) step_extractor = ELAsTiCCFeatureExtractor( preprocessor, lc_classifier_extractor, detections @@ -54,7 +54,7 @@ def test_preprocessor_can_run_with_parsed_data(): preprocessor = ElasticcPreprocessor(stream=True) lc_classifier_extractor = ElasticcFeatureExtractor(round=2) detections = generate_alert( - aid="aid1", band="u", num_messages=1, identifier="1" + oid="oid1", band="u", num_messages=1, identifier="1" ) step_extractor = ELAsTiCCFeatureExtractor( preprocessor, lc_classifier_extractor, detections diff --git a/feature_step/tests/unittest/test_parsers.py b/feature_step/tests/unittest/test_parsers.py index d65eb6d78..8643a18e5 100644 --- a/feature_step/tests/unittest/test_parsers.py +++ b/feature_step/tests/unittest/test_parsers.py @@ -19,10 +19,8 @@ def setUp(self) -> None: def test_parse_output(self): expected_payload = [ { - "aid": "aid1", + "oid": "oid1", "candid": "candid1", - "meanra": 888, - "meandec": 999, "detections": [], "non_detections": [], "xmatches": {}, @@ -35,10 +33,8 @@ def test_parse_output(self): }, }, { - "aid": "aid2", + "oid": "oid2", "candid": "candid2", - "meanra": 444, - "meandec": 555, "detections": [], "non_detections": [], "xmatches": {}, @@ -52,7 +48,7 @@ def test_parse_output(self): }, ] test_features_df = features_df_for_parse.copy() - candids = {"aid1": "candid1", "aid2": "candid2"} + candids = {"oid1": "candid1", "oid2": "candid2"} parsed_result = parse_output( test_features_df, messages_for_parsing, @@ -67,7 +63,7 @@ def test_parse_scribe_payload(self): { "collection": "object", "type": "update_features", - "criteria": {"_id": "aid1", "oid": ["oid1", "oid6", "oid7"]}, + "criteria": {"_id": "oid1"}, "data": { "features_version": metadata.version("feature-step"), "features_group": "ztf_lc_features", @@ -84,7 +80,7 @@ def test_parse_scribe_payload(self): { "collection": "object", "type": "update_features", - "criteria": {"_id": "aid2", "oid": ["oid2", "oid5", "oid8"]}, + "criteria": {"_id": "oid2"}, "data": { "features_version": metadata.version("feature-step"), "features_group": "ztf_lc_features", @@ -100,13 +96,8 @@ def test_parse_scribe_payload(self): }, ] test_features_df = features_df_for_parse.copy() - messages_aid_oid = { - "aid1": ["oid1", "oid6", "oid7"], - "aid2": ["oid2", "oid5", "oid8"], - "aid3": ["oid3", "oid4", "oid9"], - } parsed_result = parse_scribe_payload( - messages_aid_oid, test_features_df, self.mock_extractor_class + test_features_df, self.mock_extractor_class ) self.assertEqual(parsed_result, expected_payload) diff --git a/feature_step/tests/unittest/test_step.py b/feature_step/tests/unittest/test_step.py index 1b3283af2..f53d2a982 100644 --- a/feature_step/tests/unittest/test_step.py +++ b/feature_step/tests/unittest/test_step.py @@ -73,10 +73,8 @@ def setUp(self): def test_execute(self): expected_output = [ { - "aid": "aid1", + "oid": "oid1", "candid": ["1_candid_aid_1", "2_candid_aid_1"], - "meanra": 888, - "meandec": 999, "detections": [ { "candid": "1_candid_aid_1", @@ -163,10 +161,8 @@ def test_execute(self): }, }, { - "aid": "aid2", + "oid": "oid2", "candid": ["1_candid_aid_2"], - "meanra": 444, - "meandec": 555, "detections": [ { "candid": "1_candid_aid_2", @@ -327,13 +323,13 @@ def test_execute(self): ] expected_xmatches_for_extractor = [ { - "aid": "aid1", + "oid": "oid1", "W1mag": 123, "W2mag": 456, "W3mag": 789, }, { - "aid": "aid2", + "oid": "oid2", "W1mag": 123, "W2mag": 456, "W3mag": 789, diff --git a/lc_classification_step/pyproject.toml b/lc_classification_step/pyproject.toml index a175f0041..e2881e1c6 100644 --- a/lc_classification_step/pyproject.toml +++ b/lc_classification_step/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "lc_classification_step" -version = "23.12.26a5" +version = "23.12.26a8" description = "LC Classification Step" authors = [] readme = "README.md" diff --git a/libs/db-plugins/pyproject.toml b/libs/db-plugins/pyproject.toml index e9c96ae25..3d45aa571 100644 --- a/libs/db-plugins/pyproject.toml +++ b/libs/db-plugins/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [tool.poetry] name = "db-plugins" -version = "6.1.1a67" +version = "6.1.1a70" description = "ALeRCE database plugins." authors = [] readme = "README.rst" diff --git a/lightcurve-step/pyproject.toml b/lightcurve-step/pyproject.toml index d9943ec37..053b8d52f 100644 --- a/lightcurve-step/pyproject.toml +++ b/lightcurve-step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "lightcurve-step" -version = "23.12.26a5" +version = "23.12.26a8" description = "Lightcurve Step" authors = [] readme = "README.md" diff --git a/magstats_step/magstats_step/core/magstats.py b/magstats_step/magstats_step/core/magstats.py index db960c305..e66bc1508 100644 --- a/magstats_step/magstats_step/core/magstats.py +++ b/magstats_step/magstats_step/core/magstats.py @@ -7,7 +7,7 @@ class MagnitudeStatistics(BaseStatistics): - _JOIN = ["aid", "sid", "fid"] + _JOIN = ["oid", "sid", "fid"] # Saturation threshold for each survey (only applies to corrected magnitudes) _THRESHOLD = {"ZTF": 13.2} diff --git a/magstats_step/magstats_step/core/objstats.py b/magstats_step/magstats_step/core/objstats.py index d021e7666..be96da0c8 100644 --- a/magstats_step/magstats_step/core/objstats.py +++ b/magstats_step/magstats_step/core/objstats.py @@ -7,7 +7,7 @@ class ObjectStatistics(BaseStatistics): - _JOIN = "aid" + _JOIN = "oid" def __init__(self, detections: List[dict]): super().__init__(detections) @@ -47,7 +47,7 @@ def average(series): # Needs wrapper to use the sigmas in the agg call return self._weighted_mean(series, sigmas.loc[series.index]) sigmas = self._arcsec2deg(self._detections[f"e_{label}"]) - grouped_sigmas = self._group(sigmas.set_axis(self._detections["aid"])) + grouped_sigmas = self._group(sigmas.set_axis(self._detections["oid"])) return pd.DataFrame( { f"mean{label}": self._grouped_detections()[label].agg(average), diff --git a/magstats_step/magstats_step/step.py b/magstats_step/magstats_step/step.py index 9ff9431be..b2bd4e25c 100644 --- a/magstats_step/magstats_step/step.py +++ b/magstats_step/magstats_step/step.py @@ -35,12 +35,12 @@ def _execute(self, messages: dict): magstats = magstats_calculator.generate_statistics( self.excluded ).reset_index() - magstats = magstats.set_index("aid").replace({np.nan: None}) - for aid in stats: + magstats = magstats.set_index("oid").replace({np.nan: None}) + for oid in stats: try: - stats[aid]["magstats"] = magstats.loc[aid].to_dict("records") + stats[oid]["magstats"] = magstats.loc[oid].to_dict("records") except TypeError: - stats[aid]["magstats"] = [magstats.loc[aid].to_dict()] + stats[oid]["magstats"] = [magstats.loc[oid].to_dict()] return stats @@ -63,12 +63,12 @@ def _execute_ztf(self, messages: dict): magstats = magstats_calculator.generate_statistics( self.excluded ).reset_index() - magstats = magstats.set_index("aid").replace({np.nan: None}) - for aid in stats: + magstats = magstats.set_index("oid").replace({np.nan: None}) + for oid in stats: try: - stats[aid]["magstats"] = magstats.loc[aid].to_dict("records") + stats[oid]["magstats"] = magstats.loc[oid].to_dict("records") except TypeError: - stats[aid]["magstats"] = [magstats.loc[aid].to_dict()] + stats[oid]["magstats"] = [magstats.loc[oid].to_dict()] return stats @@ -80,11 +80,11 @@ def execute(self, messages: dict): # it seems that we'll have to produce different commands in this def produce_scribe(self, result: dict): - for aid, stats in result.items(): + for oid, stats in result.items(): command = { "collection": "object", "type": "update", - "criteria": {"_id": aid}, + "criteria": {"_id": oid}, "data": stats | { "loc": { @@ -107,7 +107,7 @@ def produce_scribe_ztf(self, result: dict): { "collection": "magstats", "type": "upsert", - "criteria": {"oid": oid}, + "criteria": {"_id": oid}, "data": stats, } for oid in oids diff --git a/magstats_step/pyproject.toml b/magstats_step/pyproject.toml index 42d37be71..4c093b66f 100644 --- a/magstats_step/pyproject.toml +++ b/magstats_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "magstats-step" -version = "23.12.26a5" +version = "23.12.26a8" description = "" authors = ["ASHuenchuleo ", "Pablo Castellanos"] readme = "README.md" diff --git a/magstats_step/tests/integration/conftest.py b/magstats_step/tests/integration/conftest.py index 842bf118f..2ce004d8e 100644 --- a/magstats_step/tests/integration/conftest.py +++ b/magstats_step/tests/integration/conftest.py @@ -1,6 +1,7 @@ import pytest import uuid import os +import pathlib from confluent_kafka.admin import AdminClient, NewTopic from apf.producers import KafkaProducer @@ -56,7 +57,7 @@ def env_variables(): env_variables_dict = { "PRODUCER_SCHEMA_PATH": "", "CONSUMER_SCHEMA_PATH": "", - "METRIS_SCHEMA_PATH": "../schemas/magstats_step//metrics.json", + "METRIS_SCHEMA_PATH": "../schemas/magstats_step/metrics.json", "SCRIBE_SCHEMA_PATH": "../schemas/scribe_step/scribe.avsc", "CONSUMER_SERVER": "localhost:9092", "CONSUMER_TOPICS": "correction", @@ -95,8 +96,12 @@ def produce_messages(topic): { "PARAMS": {"bootstrap.servers": "localhost:9092"}, "TOPIC": topic, - "SCHEMA_PATH": os.path.join( - os.path.dirname(__file__), "../../schema.avsc" + "SCHEMA_PATH": str( + pathlib.Path( + pathlib.Path(__file__).parent.parent.parent.parent, + "schemas/correction_step", + "output.avsc", + ) ), } ) diff --git a/magstats_step/tests/integration/test_step.py b/magstats_step/tests/integration/test_step.py index 4941b9cf3..77fb6d80d 100644 --- a/magstats_step/tests/integration/test_step.py +++ b/magstats_step/tests/integration/test_step.py @@ -6,7 +6,7 @@ def assert_message_schema(command): if command["collection"] == "magstats": assert command["type"] == "upsert" - assert "oid" in command["criteria"] + assert "_id" in command["criteria"] elif command["collection"] == "object": assert command["type"] == "update" assert "_id" in command["criteria"] diff --git a/magstats_step/tests/unittests/data/messages.py b/magstats_step/tests/unittests/data/messages.py index d08a2b588..042d0e9ae 100644 --- a/magstats_step/tests/unittests/data/messages.py +++ b/magstats_step/tests/unittests/data/messages.py @@ -1,9 +1,18 @@ import random +import pathlib from fastavro import schema from fastavro import utils -SCHEMA = schema.load_schema("schema.avsc") +SCHEMA_PATH = str( + pathlib.Path( + pathlib.Path(__file__).parent.parent.parent.parent.parent, + "schemas/correction_step", + "output.avsc", + ) +) + +SCHEMA = schema.load_schema(SCHEMA_PATH) random.seed(42) aids_pool = [f"AID22X{i}" for i in range(10)] @@ -12,20 +21,18 @@ data = list(utils.generate_many(SCHEMA, 10)) for d in data: aid = random.choice(aids_pool) - d["aid"] = aid - sid = "ZTF" if random.random() < 0.5 else "ATLAS" oid = random.choice(oids_pool) + d["oid"] = oid + sid = "ZTF" if random.random() < 0.5 else "ATLAS" for detection in d["detections"]: detection["aid"] = aid + detection["oid"] = oid detection["sid"] = sid detection["fid"] = "g" if random.random() < 0.5 else "r" detection["forced"] = False - if sid == "ZTF": - detection["oid"] = oid for non_detection in d["non_detections"]: non_detection["aid"] = aid + non_detection["oid"] = oid non_detection["sid"] = sid non_detection["fid"] = "g" if random.random() < 0.5 else "r" - if sid == "ZTF": - non_detection["oid"] = oid diff --git a/magstats_step/tests/unittests/test_magstats.py b/magstats_step/tests/unittests/test_magstats.py index 237d8f891..e7c380105 100644 --- a/magstats_step/tests/unittests/test_magstats.py +++ b/magstats_step/tests/unittests/test_magstats.py @@ -10,6 +10,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag": 2, @@ -18,6 +19,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag": 2, @@ -26,6 +28,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag": 5, @@ -34,6 +37,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mag": 1, @@ -42,6 +46,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mag": 1, @@ -50,6 +55,7 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mag": 2, @@ -67,13 +73,13 @@ def test_calculate_uncorrected_stats_gives_statistics_for_magnitudes_per_aid_and "magmax": [5, 1, 2], "magmin": [2, 1, 1], "magsigma": [np.sqrt(2), 0, 0.5], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -81,6 +87,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag_corr": 2, @@ -90,6 +97,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag_corr": 2, @@ -99,6 +107,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag_corr": 5, @@ -108,6 +117,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mag_corr": 5, @@ -117,6 +127,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mag_corr": 1, @@ -126,6 +137,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mag_corr": 1, @@ -135,6 +147,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mag_corr": 2, @@ -144,6 +157,7 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 2, "mag_corr": 2, @@ -162,13 +176,13 @@ def test_calculate_corrected_stats_gives_statistics_for_corrected_magnitudes_per "magmax_corr": [5, 1, 2], "magmin_corr": [2, 1, 1], "magsigma_corr": [np.sqrt(2), 0, 0.5], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -176,6 +190,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -185,6 +200,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe }, # last { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -194,6 +210,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe }, # first { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -203,6 +220,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -212,6 +230,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe }, # last and first { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -221,6 +240,7 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe }, # first { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -236,13 +256,13 @@ def test_calculate_uncorrected_stats_over_time_gives_first_and_last_magnitude_pe { "magfirst": [2, 1, 1], "maglast": [1, 1, 2], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -250,6 +270,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -260,6 +281,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -270,6 +292,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -280,6 +303,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 4, @@ -290,6 +314,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -300,6 +325,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -310,6 +336,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -320,6 +347,7 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn }, { "aid": "AID2", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 0, @@ -336,13 +364,13 @@ def test_calculate_corrected_stats_over_time_gives_first_and_last_corrected_magn { "magfirst_corr": [2, 1, 1], "maglast_corr": [1, 1, 2], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -364,6 +392,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -372,6 +401,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 0, @@ -380,6 +410,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -388,6 +419,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 0.5, @@ -396,6 +428,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -404,6 +437,7 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -417,13 +451,13 @@ def test_calculate_firstmjd_gives_first_date_per_aid_and_fid(): expected = pd.DataFrame( { "firstmjd": [0, 0.5, 1], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -431,6 +465,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -439,6 +474,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -447,6 +483,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -455,6 +492,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -463,6 +501,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -471,6 +510,7 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -484,13 +524,13 @@ def test_calculate_lastmjd_gives_last_date_per_aid_and_fid(): expected = pd.DataFrame( { "lastmjd": [3, 1, 2], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -498,6 +538,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -507,6 +548,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -516,6 +558,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -525,6 +568,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -534,6 +578,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -543,6 +588,7 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -557,13 +603,13 @@ def test_calculate_corrected_gives_whether_first_detection_per_aid_and_fid_is_co expected = pd.DataFrame( { "corrected": [False, True, True], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -571,6 +617,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 3, @@ -580,6 +627,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -589,6 +637,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "mjd": 2, @@ -598,6 +647,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "mjd": 1, @@ -607,6 +657,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 1, @@ -616,6 +667,7 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "mjd": 2, @@ -630,13 +682,13 @@ def test_calculate_stellar_gives_whether_first_detection_per_aid_and_fid_is_stel expected = pd.DataFrame( { "stellar": [False, True, True], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -644,6 +696,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "candid": "a", @@ -651,6 +704,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "candid": "b", @@ -658,6 +712,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "candid": "c", @@ -665,6 +720,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "candid": "d", @@ -672,6 +728,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "candid": "e", @@ -679,6 +736,7 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "candid": "f", @@ -691,13 +749,13 @@ def test_calculate_ndet_gives_number_of_detections_per_aid_and_fid(): expected = pd.DataFrame( { "ndet": [3, 1, 2], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -705,6 +763,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "dubious": True, @@ -713,6 +772,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "dubious": True, @@ -721,6 +781,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 1, "dubious": False, @@ -729,6 +790,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() }, { "aid": "AID2", + "oid": "OID2", "sid": "SURVEY", "fid": 1, "dubious": False, @@ -737,6 +799,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "dubious": True, @@ -745,6 +808,7 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 2, "dubious": False, @@ -758,13 +822,13 @@ def test_calculate_ndubious_gives_number_of_dubious_detections_per_aid_and_fid() expected = pd.DataFrame( { "ndubious": [2, 0, 1], - "aid": ["AID1", "AID2", "AID1"], + "oid": ["OID1", "OID2", "OID1"], "sid": ["SURVEY", "SURVEY", "SURVEY"], "fid": [1, 1, 2], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) @@ -772,6 +836,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "ZTF", "fid": 1, "corrected": True, @@ -781,6 +846,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "ZTF", "fid": 1, "corrected": True, @@ -790,6 +856,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "ZTF", "fid": 1, "corrected": True, @@ -799,6 +866,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "ZTF", "fid": 1, "corrected": True, @@ -808,6 +876,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "ZTF", "fid": 2, "corrected": False, @@ -817,6 +886,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "ZTF", "fid": 3, "corrected": False, @@ -826,6 +896,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID2", + "oid": "OID2", "sid": "ZTF", "fid": 3, "corrected": True, @@ -835,6 +906,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 10, "corrected": True, @@ -844,6 +916,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, # No threshold { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "fid": 10, "corrected": True, @@ -853,6 +926,7 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): }, # No threshold { "aid": "AID3", + "oid": "OID3", "sid": "SURVEY", "fid": 1, "corrected": False, @@ -867,13 +941,13 @@ def test_calculate_saturation_rate_gives_saturation_ratio_per_aid_and_fid(): expected = pd.DataFrame( { "saturation_rate": [0.5, np.nan, 0, np.nan, np.nan], - "aid": ["AID1", "AID2", "AID2", "AID1", "AID3"], + "oid": ["OID1", "OID2", "OID2", "OID1", "OID3"], "sid": ["ZTF", "ZTF", "ZTF", "SURVEY", "SURVEY"], "fid": [1, 2, 3, 10, 1], } ) assert_frame_equal( - result, expected.set_index(["aid", "sid", "fid"]), check_like=True + result, expected.set_index(["oid", "sid", "fid"]), check_like=True ) diff --git a/magstats_step/tests/unittests/test_objstats.py b/magstats_step/tests/unittests/test_objstats.py index 172f8f977..3b08d54da 100644 --- a/magstats_step/tests/unittests/test_objstats.py +++ b/magstats_step/tests/unittests/test_objstats.py @@ -73,9 +73,9 @@ def test_calculate_weighted_mean_error_with_one_very_large_error_has_that_error_ def test_calculate_coordinates_with_ra_uses_weighted_mean_and_weighted_mean_error_per_aid(): detections = [ - {"aid": "AID1", "ra": 10, "e_ra": 2, "candid": "a", "forced": False}, - {"aid": "AID2", "ra": 20, "e_ra": 4, "candid": "c", "forced": False}, - {"aid": "AID1", "ra": 20, "e_ra": 4, "candid": "b", "forced": False}, + {"aid": "AID1", "oid": "OID1", "ra": 10, "e_ra": 2, "candid": "a", "forced": False}, + {"aid": "AID2", "oid": "OID2", "ra": 10, "ra": 20, "e_ra": 4, "candid": "c", "forced": False}, + {"aid": "AID1", "oid": "OID1", "ra": 10, "ra": 20, "e_ra": 4, "candid": "b", "forced": False}, ] calculator = ObjectStatistics(detections) @@ -130,9 +130,9 @@ def test_calculate_coordinates_with_ra_uses_weighted_mean_and_weighted_mean_erro def test_calculate_coordinates_with_dec_uses_weighted_mean_and_weighted_mean_error_per_aid(): detections = [ - {"aid": "AID1", "dec": 10, "e_dec": 2, "candid": "a", "forced": False}, - {"aid": "AID2", "dec": 20, "e_dec": 4, "candid": "c", "forced": False}, - {"aid": "AID1", "dec": 20, "e_dec": 4, "candid": "b", "forced": False}, + {"aid": "AID1", "oid": "AID1", "dec": 10, "e_dec": 2, "candid": "a", "forced": False}, + {"aid": "AID2", "oid": "AID2", "dec": 20, "e_dec": 4, "candid": "c", "forced": False}, + {"aid": "AID1", "oid": "AID1", "dec": 20, "e_dec": 4, "candid": "b", "forced": False}, ] calculator = ObjectStatistics(detections) @@ -187,10 +187,10 @@ def test_calculate_coordinates_with_dec_uses_weighted_mean_and_weighted_mean_err def test_calculate_unique_gives_list_of_unique_values_in_field_per_aid(): detections = [ - {"aid": "AID1", "candid": "a", "extra": "A", "forced": False}, - {"aid": "AID2", "candid": "c", "extra": "A", "forced": False}, - {"aid": "AID1", "candid": "b", "extra": "A", "forced": False}, - {"aid": "AID1", "candid": "d", "extra": "B", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "a", "extra": "A", "forced": False}, + {"aid": "AID2", "oid": "OID2", "candid": "c", "extra": "A", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "b", "extra": "A", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "d", "extra": "B", "forced": False}, ] calculator = ObjectStatistics(detections) result = calculator._calculate_unique("extra") @@ -200,7 +200,7 @@ def test_calculate_unique_gives_list_of_unique_values_in_field_per_aid(): result["extra"], pd.Series( [["A", "B"], ["A"]], - index=pd.Index(["AID1", "AID2"], name="aid"), + index=pd.Index(["OID1", "OID2"], name="oid"), name="extra", ), ) @@ -228,10 +228,10 @@ def test_calculate_dec_uses_calculate_coordinates(): def test_calculate_ndet_gives_number_of_detections_per_aid(): detections = [ - {"aid": "AID1", "candid": "a", "forced": False}, - {"aid": "AID2", "candid": "c", "forced": False}, - {"aid": "AID1", "candid": "b", "forced": False}, - {"aid": "AID1", "candid": "d", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "a", "forced": False}, + {"aid": "AID2", "oid": "OID2", "candid": "c", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "b", "forced": False}, + {"aid": "AID1", "oid": "OID1", "candid": "d", "forced": False}, ] calculator = ObjectStatistics(detections) result = calculator.calculate_ndet() @@ -240,17 +240,17 @@ def test_calculate_ndet_gives_number_of_detections_per_aid(): assert_series_equal( result["ndet"], pd.Series( - [3, 1], index=pd.Index(["AID1", "AID2"], name="aid"), name="ndet" + [3, 1], index=pd.Index(["OID1", "OID2"], name="oid"), name="ndet" ), ) def test_calculate_firstmjd_gives_the_first_mjd_per_aid(): detections = [ - {"aid": "AID1", "mjd": 1, "candid": "a", "forced": False}, - {"aid": "AID2", "mjd": 2, "candid": "c", "forced": False}, - {"aid": "AID1", "mjd": 3, "candid": "b", "forced": False}, - {"aid": "AID1", "mjd": 2, "candid": "d", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 1, "candid": "a", "forced": False}, + {"aid": "AID2", "oid": "OID2", "mjd": 2, "candid": "c", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 3, "candid": "b", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 2, "candid": "d", "forced": False}, ] calculator = ObjectStatistics(detections) result = calculator.calculate_firstmjd() @@ -260,7 +260,7 @@ def test_calculate_firstmjd_gives_the_first_mjd_per_aid(): result["firstmjd"], pd.Series( [1, 2], - index=pd.Index(["AID1", "AID2"], name="aid"), + index=pd.Index(["OID1", "OID2"], name="oid"), name="firstmjd", ), ) @@ -268,10 +268,10 @@ def test_calculate_firstmjd_gives_the_first_mjd_per_aid(): def test_calculate_lastmjd_gives_the_last_mjd_per_aid(): detections = [ - {"aid": "AID1", "mjd": 1, "candid": "a", "forced": False}, - {"aid": "AID2", "mjd": 2, "candid": "c", "forced": False}, - {"aid": "AID1", "mjd": 3, "candid": "b", "forced": False}, - {"aid": "AID1", "mjd": 2, "candid": "d", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 1, "candid": "a", "forced": False}, + {"aid": "AID2", "oid": "OID2", "mjd": 2, "candid": "c", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 3, "candid": "b", "forced": False}, + {"aid": "AID1", "oid": "OID1", "mjd": 2, "candid": "d", "forced": False}, ] calculator = ObjectStatistics(detections) result = calculator.calculate_lastmjd() @@ -281,7 +281,7 @@ def test_calculate_lastmjd_gives_the_last_mjd_per_aid(): result["lastmjd"], pd.Series( [3, 2], - index=pd.Index(["AID1", "AID2"], name="aid"), + index=pd.Index(["OID1", "OID2"], name="oid"), name="lastmjd", ), ) @@ -311,6 +311,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "MOCK_SURVEY", "mjd": 1, "corrected": False, @@ -319,6 +320,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre }, # Should ignore { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 2, "corrected": True, @@ -327,6 +329,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre }, # True for AID1 { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 3, "corrected": False, @@ -335,6 +338,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre }, { "aid": "AID2", + "oid": "OID2", "sid": "MOCK_SURVEY", "mjd": 1, "corrected": True, @@ -343,6 +347,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre }, # Should ignore { "aid": "AID3", + "oid": "OID3", "sid": "SURVEY", "mjd": 2, "corrected": False, @@ -351,6 +356,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre }, # False for AID3 { "aid": "AID3", + "oid": "OID3", "sid": "SURVEY", "mjd": 3, "corrected": True, @@ -367,7 +373,7 @@ def test_calculate_corrected_gives_whether_first_detection_in_surveys_with_corre result["corrected"], pd.Series( [True, False], - index=pd.Index(["AID1", "AID3"], name="aid"), + index=pd.Index(["OID1", "OID3"], name="oid"), name="corrected", ), ) @@ -377,6 +383,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "MOCK_SURVEY", "mjd": 1, "stellar": False, @@ -385,6 +392,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar }, # Should ignore { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 2, "stellar": True, @@ -393,6 +401,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar }, # True for AID1 { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 3, "stellar": False, @@ -401,6 +410,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar }, { "aid": "AID2", + "oid": "OID2", "sid": "MOCK_SURVEY", "mjd": 1, "stellar": True, @@ -409,6 +419,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar }, # Should ignore { "aid": "AID3", + "oid": "OID3", "sid": "SURVEY", "mjd": 2, "stellar": False, @@ -417,6 +428,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar }, # False for AID3 { "aid": "AID3", + "oid": "OID3", "sid": "SURVEY", "mjd": 3, "stellar": True, @@ -433,7 +445,7 @@ def test_calculate_stellar_gives_whether_first_detection_in_surveys_with_stellar result["stellar"], pd.Series( [True, False], - index=pd.Index(["AID1", "AID3"], name="aid"), + index=pd.Index(["OID1", "OID3"], name="oid"), name="stellar", ), ) @@ -458,6 +470,7 @@ def test_object_statistics_deltajd(): detections = [ { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 1, "stellar": False, @@ -466,6 +479,7 @@ def test_object_statistics_deltajd(): }, { "aid": "AID1", + "oid": "OID1", "sid": "SURVEY", "mjd": 5, "stellar": True, @@ -477,5 +491,5 @@ def test_object_statistics_deltajd(): result = calculator.calculate_deltajd() assert_series_equal( result["deltajd"], - pd.Series([4], index=pd.Index(["AID1"], name="aid"), name="deltajd"), + pd.Series([4], index=pd.Index(["OID1"], name="oid"), name="deltajd"), ) diff --git a/magstats_step/tests/unittests/test_step.py b/magstats_step/tests/unittests/test_step.py index b7749acf9..92e6891c5 100644 --- a/magstats_step/tests/unittests/test_step.py +++ b/magstats_step/tests/unittests/test_step.py @@ -12,18 +12,19 @@ def test_execute_multistream(env_variables): formatted_data = step.pre_execute(data) result = step.execute(formatted_data) result = result["multistream"] + print(result.keys()) for d in data: - assert d["aid"] in result - assert "meanra" in result[d["aid"]] - assert "meandec" in result[d["aid"]] - assert "magstats" in result[d["aid"]] - assert "oid" in result[d["aid"]] - assert "tid" in result[d["aid"]] - assert "firstmjd" in result[d["aid"]] - assert "lastmjd" in result[d["aid"]] - assert "ndet" in result[d["aid"]] - assert "sigmara" in result[d["aid"]] - assert "sigmadec" in result[d["aid"]] + assert d["oid"] in result + assert "meanra" in result[d["oid"]] + assert "meandec" in result[d["oid"]] + assert "magstats" in result[d["oid"]] + assert "oid" in result[d["oid"]] + assert "tid" in result[d["oid"]] + assert "firstmjd" in result[d["oid"]] + assert "lastmjd" in result[d["oid"]] + assert "ndet" in result[d["oid"]] + assert "sigmara" in result[d["oid"]] + assert "sigmadec" in result[d["oid"]] def test_scribe_message_multistream(env_variables): @@ -36,7 +37,7 @@ def test_scribe_message_multistream(env_variables): step.post_execute(result) result = result["multistream"] for d in data: - to_write = result[d["aid"]] + to_write = result[d["oid"]] to_write.update( { "loc": { @@ -51,7 +52,7 @@ def test_scribe_message_multistream(env_variables): command = { "collection": "object", "type": "update", - "criteria": {"_id": d["aid"]}, + "criteria": {"_id": d["oid"]}, "data": to_write, "options": {"upsert": True}, } @@ -68,14 +69,14 @@ def test_execute_ztf(env_variables): for d in data: if not any([det["sid"] == "ZTF" for det in d["detections"]]): continue - assert d["aid"] in result - assert "meanra" in result[d["aid"]] - assert "meandec" in result[d["aid"]] - assert "magstats" in result[d["aid"]] - assert "oid" in result[d["aid"]] - assert "tid" in result[d["aid"]] - assert "firstmjd" in result[d["aid"]] - assert "lastmjd" in result[d["aid"]] - assert "ndet" in result[d["aid"]] - assert "sigmara" in result[d["aid"]] - assert "sigmadec" in result[d["aid"]] + assert d["oid"] in result + assert "meanra" in result[d["oid"]] + assert "meandec" in result[d["oid"]] + assert "magstats" in result[d["oid"]] + assert "oid" in result[d["oid"]] + assert "tid" in result[d["oid"]] + assert "firstmjd" in result[d["oid"]] + assert "lastmjd" in result[d["oid"]] + assert "ndet" in result[d["oid"]] + assert "sigmara" in result[d["oid"]] + assert "sigmadec" in result[d["oid"]] diff --git a/metadata_step/pyproject.toml b/metadata_step/pyproject.toml index ebdbe23ab..cb78bef8b 100644 --- a/metadata_step/pyproject.toml +++ b/metadata_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metadata-step" -version = "23.12.26a5" +version = "23.12.26a8" description = "" authors = ["Pedro Gallardo "] readme = "README.md" diff --git a/prv_candidates_step/pyproject.toml b/prv_candidates_step/pyproject.toml index fe51341e3..e06237676 100644 --- a/prv_candidates_step/pyproject.toml +++ b/prv_candidates_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "prv-candidates-step" -version = "23.12.26a5" +version = "23.12.26a8" description = "Previous Detections Step" authors = [] readme = "README.md" diff --git a/s3_step/pyproject.toml b/s3_step/pyproject.toml index 735b452b4..1404cbf87 100644 --- a/s3_step/pyproject.toml +++ b/s3_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "s3-step" -version = "23.12.26a5" +version = "23.12.26a8" description = "S3 step" authors = [] readme = "README.md" diff --git a/schemas/correction_step/output.avsc b/schemas/correction_step/output.avsc index 11da39f65..0cab6cc73 100644 --- a/schemas/correction_step/output.avsc +++ b/schemas/correction_step/output.avsc @@ -27,119 +27,119 @@ "type": { "type": "array", "items": { - "type": "record", - "name": "alert", - "fields": [ - { - "name": "aid", - "type": "string" - }, - { - "name": "oid", - "type": "string" - }, - { - "name": "sid", - "type": "string" - }, - { - "name": "pid", - "type": "long" - }, - { - "name": "tid", - "type": "string" - }, - { - "name": "fid", - "type": "string" - }, - { - "name": "candid", - "type": ["string", "long"] - }, - { - "name": "mjd", - "type": "double" - }, - { - "name": "ra", - "type": "double" - }, - { - "name": "e_ra", - "type": "float" - }, - { - "name": "dec", - "type": "double" - }, - { - "name": "e_dec", - "type": "float" - }, - { - "name": "mag", - "type": "float" - }, - { - "name": "e_mag", - "type": "float" - }, - { - "name": "mag_corr", - "type": ["float", "null"] - }, - { - "name": "e_mag_corr", - "type": ["float", "null"] - }, - { - "name": "e_mag_corr_ext", - "type": ["float", "null"] - }, - { - "name": "isdiffpos", - "type": "int" - }, - { - "name": "corrected", - "type": "boolean" - }, - { - "name": "dubious", - "type": "boolean" - }, - { - "name": "stellar", - "type": "boolean" - }, - { - "name": "has_stamp", - "type": "boolean" - }, - { - "name": "forced", - "type": "boolean" - }, - { - "name": "new", - "type": "boolean" - }, - { - "name": "parent_candid", - "type": ["string", "long", "null"] - }, - { - "name": "extra_fields", - "type": { - "default": {}, - "type": "map", - "values": ["null", "int", "float", "string", "bytes", "boolean"] + "type": "record", + "name": "alert", + "fields": [ + { + "name": "aid", + "type": ["string", "null"] + }, + { + "name": "oid", + "type": "string" + }, + { + "name": "sid", + "type": "string" + }, + { + "name": "pid", + "type": "long" + }, + { + "name": "tid", + "type": "string" + }, + { + "name": "fid", + "type": "string" + }, + { + "name": "candid", + "type": ["string", "long"] + }, + { + "name": "mjd", + "type": "double" + }, + { + "name": "ra", + "type": "double" + }, + { + "name": "e_ra", + "type": "float" + }, + { + "name": "dec", + "type": "double" + }, + { + "name": "e_dec", + "type": "float" + }, + { + "name": "mag", + "type": "float" + }, + { + "name": "e_mag", + "type": "float" + }, + { + "name": "mag_corr", + "type": ["float", "null"] + }, + { + "name": "e_mag_corr", + "type": ["float", "null"] + }, + { + "name": "e_mag_corr_ext", + "type": ["float", "null"] + }, + { + "name": "isdiffpos", + "type": "int" + }, + { + "name": "corrected", + "type": "boolean" + }, + { + "name": "dubious", + "type": "boolean" + }, + { + "name": "stellar", + "type": "boolean" + }, + { + "name": "has_stamp", + "type": "boolean" + }, + { + "name": "forced", + "type": "boolean" + }, + { + "name": "new", + "type": "boolean" + }, + { + "name": "parent_candid", + "type": ["string", "long", "null"] + }, + { + "name": "extra_fields", + "type": { + "default": {}, + "type": "map", + "values": ["null", "int", "float", "string", "bytes", "boolean"] + } } - } - ] - }, + ] + }, "default": [] } }, @@ -153,7 +153,7 @@ "fields": [ { "name": "aid", - "type": "string" + "type": ["string", "null"] }, { "name": "oid", diff --git a/schemas/feature_step/output.avsc b/schemas/feature_step/output.avsc index e1cb73b0b..841d0e360 100644 --- a/schemas/feature_step/output.avsc +++ b/schemas/feature_step/output.avsc @@ -5,8 +5,6 @@ "fields": [ { "name": "oid", "type": "string" }, { "name": "candid", "type": { "type": "array", "items": "string" } }, - { "name": "meanra", "type": "float" }, - { "name": "meandec", "type": "float" }, { "name": "detections", "type": { @@ -18,7 +16,7 @@ "fields": [ { "name": "candid", "type": ["long", "string"] }, { "name": "tid", "type": "string" }, - { "name": "aid", "type": "string" }, + { "name": "aid", "type": ["string", "null"] }, { "name": "oid", "type": ["long", "string"] }, { "name": "mjd", "type": "double" }, { "name": "sid", "type": "string" }, @@ -62,7 +60,7 @@ "name": "non_detections_record", "type": "record", "fields": [ - { "name": "aid", "type": "string" }, + { "name": "aid", "type": ["string", "null"] }, { "name": "tid", "type": "string" }, { "name": "sid", "type": "string" }, { "name": "oid", "type": ["long", "string"] }, diff --git a/schemas/lightcurve_step/output.avsc b/schemas/lightcurve_step/output.avsc index b28ab3bce..8f53d770d 100644 --- a/schemas/lightcurve_step/output.avsc +++ b/schemas/lightcurve_step/output.avsc @@ -24,7 +24,7 @@ "fields": [ { "name": "aid", - "type": "string" + "type": ["string", "null"] }, { "name": "oid", @@ -121,7 +121,7 @@ "fields": [ { "name": "aid", - "type": "string" + "type": ["string", "null"] }, { "name": "oid", diff --git a/schemas/xmatch_step/output.avsc b/schemas/xmatch_step/output.avsc index 16b45020c..1058d6234 100644 --- a/schemas/xmatch_step/output.avsc +++ b/schemas/xmatch_step/output.avsc @@ -33,7 +33,7 @@ }, { "name": "aid", - "type": "string" + "type": ["string", "null"] }, { "name": "oid", @@ -146,7 +146,7 @@ "fields": [ { "name": "aid", - "type": "string" + "type": ["string", "null"] }, { "name": "tid", diff --git a/scribe/pyproject.toml b/scribe/pyproject.toml index c6f031919..ba5d853dc 100644 --- a/scribe/pyproject.toml +++ b/scribe/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "scribe" -version = "23.12.26a5" +version = "23.12.26a8" description = "Mongo scribe" authors = [] readme = "README.md" diff --git a/sorting_hat_step/pyproject.toml b/sorting_hat_step/pyproject.toml index 9f61c660e..259eabf38 100644 --- a/sorting_hat_step/pyproject.toml +++ b/sorting_hat_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "sorting-hat-step" -version = "23.12.26a5" +version = "23.12.26a8" description = "Sorting hat step" authors = [] readme = "README.md" diff --git a/watchlist_step/pyproject.toml b/watchlist_step/pyproject.toml index 669e7efdc..ce0d1374f 100644 --- a/watchlist_step/pyproject.toml +++ b/watchlist_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "watchlist-step" -version = "23.12.26a5" +version = "23.12.26a8" description = "Watchlist Step" authors = [] readme = "README.md" diff --git a/xmatch_step/pyproject.toml b/xmatch_step/pyproject.toml index dd77c7e9b..b8f7e6bde 100644 --- a/xmatch_step/pyproject.toml +++ b/xmatch_step/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "xmatch-step" -version = "23.12.26a5" +version = "23.12.26a8" description = "xmatch step" authors = [] readme = "README.md"