From 2387126307a721aff47e9085e5b196d365f9d16e Mon Sep 17 00:00:00 2001 From: Ahmad Wahid Date: Tue, 25 Jun 2024 12:05:41 +0200 Subject: [PATCH 1/5] add get asset annotations endpoint and refactor annotations query --- flexmeasures/api/dev/sensors.py | 86 +++++++++++++++++++--- flexmeasures/data/models/generic_assets.py | 4 + flexmeasures/data/queries/annotations.py | 20 +++-- flexmeasures/ui/templates/base.html | 12 +-- 4 files changed, 97 insertions(+), 25 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 20e7bb64e..8dcc10be2 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -1,5 +1,6 @@ import json import warnings +from typing import Any from flask_classful import FlaskView, route from flask_security import current_user @@ -20,6 +21,10 @@ from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.services.annotations import prepare_annotations_for_chart from flexmeasures.ui.utils.view_utils import set_session_variables +from flexmeasures.data.models.annotations import ( + SensorAnnotationRelationship, + GenericAssetAnnotationRelationship, +) class SensorAPI(FlaskView): @@ -132,20 +137,13 @@ def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): Annotations for use in charts (in case you have the chart specs already). """ - event_starts_after = kwargs.get("event_starts_after", None) - event_ends_before = kwargs.get("event_ends_before", None) - df = sensor.generic_asset.search_annotations( - annotations_after=event_starts_after, - annotations_before=event_ends_before, - as_frame=True, + df = get_annotations_data( + sensor_id=id, + asset_or_sensor=sensor, + relationship_module=SensorAnnotationRelationship, + kwargs=kwargs, ) - # Wrap and stack annotations - df = prepare_annotations_for_chart(df) - - # Return JSON records - df = df.reset_index() - df["source"] = df["source"].astype(str) return df.to_json(orient="records") @route("/", strict_slashes=False) @@ -186,6 +184,70 @@ def get(self, id: int, asset: GenericAsset): attributes = ["name", "timezone", "timerange_of_sensors_to_show"] return {attr: getattr(asset, attr) for attr in attributes} + @route("//chart_annotations", strict_slashes=False) + @use_kwargs( + {"asset": AssetIdField(data_key="id")}, + location="path", + ) + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + }, + location="query", + ) + @permission_required_for_context("read", ctx_arg_name="asset") + def get_chart_annotations(self, id: int, asset: GenericAsset, **kwargs): + """GET from /asset//chart_annotations + + .. :quickref: Chart; Download annotations for use in charts + + Annotations for use in charts (in case you have the chart specs already). + """ + df = get_annotations_data( + sensor_id=None, + asset_or_sensor=asset, + relationship_module=GenericAssetAnnotationRelationship, + kwargs=kwargs, + ) + + return df.to_json(orient="records") + + +def get_annotations_data( + sensor_id: int | None, + asset_or_sensor: GenericAsset | Sensor, + relationship_module: Any, + kwargs, +): + """ + This function fetches a sensor or an asset annotations data + """ + event_starts_after = kwargs.get("event_starts_after", None) + event_ends_before = kwargs.get("event_ends_before", None) + if asset_or_sensor is GenericAsset: + asset_or_sensor_class = asset_or_sensor.generic_asset + else: + asset_or_sensor_class = asset_or_sensor + + df = asset_or_sensor_class.search_annotations( + annotations_after=event_starts_after, + annotations_before=event_ends_before, + as_frame=True, + sensor_id=sensor_id, + relationship_module=relationship_module, + ) + + # Wrap and stack annotations + df = prepare_annotations_for_chart(df) + + # Return JSON records + df = df.reset_index() + df["source"] = df["source"].astype(str) + return df + def get_sensor_or_abort(id: int) -> Sensor: """ diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 3ba9330d4..196d49b6b 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -357,6 +357,8 @@ def search_annotations( annotation_type: str = None, include_account_annotations: bool = False, as_frame: bool = False, + sensor_id: int = None, + relationship_module: Any = None, ) -> list[Annotation] | pd.DataFrame: """Return annotations assigned to this asset, and optionally, also those assigned to the asset's account. @@ -373,6 +375,8 @@ def search_annotations( annotations_before=annotations_before, sources=parsed_sources, annotation_type=annotation_type, + sensor_id=sensor_id, + relationship_module=relationship_module, ) ).all() if include_account_annotations and self.owner is not None: diff --git a/flexmeasures/data/queries/annotations.py b/flexmeasures/data/queries/annotations.py index 0a56e571c..ae7488bc1 100644 --- a/flexmeasures/data/queries/annotations.py +++ b/flexmeasures/data/queries/annotations.py @@ -8,26 +8,32 @@ from flexmeasures.data.models.annotations import ( Annotation, GenericAssetAnnotationRelationship, + SensorAnnotationRelationship, ) from flexmeasures.data.models.data_sources import DataSource def query_asset_annotations( asset_id: int, + sensor_id: int, + relationship_module, annotations_after: datetime | None = None, annotations_before: datetime | None = None, sources: list[DataSource] | None = None, annotation_type: str | None = None, ) -> Query: """Match annotations assigned to the given asset.""" - query = ( - select(Annotation) - .join(GenericAssetAnnotationRelationship) - .filter( - GenericAssetAnnotationRelationship.generic_asset_id == asset_id, + query = select(Annotation) + if relationship_module is GenericAssetAnnotationRelationship: + query = query.join( + GenericAssetAnnotationRelationship, GenericAssetAnnotationRelationship.annotation_id == Annotation.id, - ) - ) + ).filter(GenericAssetAnnotationRelationship.generic_asset_id == asset_id) + else: + query = query.join( + SensorAnnotationRelationship, + SensorAnnotationRelationship.annotation_id == Annotation.id, + ).filter(SensorAnnotationRelationship.sensor_id == sensor_id) if annotations_after is not None: query = query.filter( diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html index e45dd4b93..afa537262 100644 --- a/flexmeasures/ui/templates/base.html +++ b/flexmeasures/ui/templates/base.html @@ -287,7 +287,7 @@ async function embedAndLoad(chartSpecsPath, elementId, datasetName, previousResult, startDate, endDate) { - await vegaEmbed('#'+elementId, chartSpecsPath + 'dataset_name=' + datasetName + '&width=container&include_sensor_annotations=false&include_asset_annotations=false&chart_type=' + chartType, {{ chart_options | safe }}) + await vegaEmbed('#'+elementId, chartSpecsPath + 'dataset_name=' + datasetName + '&width=container&include_sensor_annotations=true&include_asset_annotations=true&chart_type=' + chartType, {{ chart_options | safe }}) .then(function (result) { // Create a custom menu item for exporting to CSV @@ -411,15 +411,15 @@ }) .then(function(response) { return response.json(); }), - /** + // Fetch annotations - fetch(dataPath + '/chart_annotations?event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate, { + fetch(dataDevPath + '/chart_annotations?event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate, { method: "GET", headers: {"Content-Type": "application/json"}, signal: signal, }) .then(function(response) { return response.json(); }), - */ + // Embed chart embedAndLoad(chartSpecsPath + 'event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate + '&', elementId, datasetName, previousResult, startDate, endDate), @@ -428,9 +428,9 @@ vegaView.change(datasetName, vega.changeset().remove(vega.truthy).insert(result[0])).resize().run(); previousResult = result[0]; checkSourceMasking(previousResult); - /** + vegaView.change(datasetName + '_annotations', vega.changeset().remove(vega.truthy).insert(result[1])).resize().run(); - */ + }).catch(console.error); }); From 8e88401ec3e2355d2ccfb80cd08d638d24347a64 Mon Sep 17 00:00:00 2001 From: Ahmad Wahid Date: Thu, 18 Jul 2024 19:27:11 +0200 Subject: [PATCH 2/5] refactor args --- flexmeasures/data/models/generic_assets.py | 3 +-- flexmeasures/data/queries/annotations.py | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 196d49b6b..3238dfa6c 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -370,12 +370,11 @@ def search_annotations( parsed_sources = parse_source_arg(source) annotations = db.session.scalars( query_asset_annotations( - asset_id=self.id, + asset_or_sensor_id=self.id if sensor_id is None else sensor_id, annotations_after=annotations_after, annotations_before=annotations_before, sources=parsed_sources, annotation_type=annotation_type, - sensor_id=sensor_id, relationship_module=relationship_module, ) ).all() diff --git a/flexmeasures/data/queries/annotations.py b/flexmeasures/data/queries/annotations.py index ae7488bc1..8bbf4c0fe 100644 --- a/flexmeasures/data/queries/annotations.py +++ b/flexmeasures/data/queries/annotations.py @@ -14,8 +14,7 @@ def query_asset_annotations( - asset_id: int, - sensor_id: int, + asset_or_sensor_id: int, relationship_module, annotations_after: datetime | None = None, annotations_before: datetime | None = None, @@ -28,12 +27,14 @@ def query_asset_annotations( query = query.join( GenericAssetAnnotationRelationship, GenericAssetAnnotationRelationship.annotation_id == Annotation.id, - ).filter(GenericAssetAnnotationRelationship.generic_asset_id == asset_id) + ).filter( + GenericAssetAnnotationRelationship.generic_asset_id == asset_or_sensor_id + ) else: query = query.join( SensorAnnotationRelationship, SensorAnnotationRelationship.annotation_id == Annotation.id, - ).filter(SensorAnnotationRelationship.sensor_id == sensor_id) + ).filter(SensorAnnotationRelationship.sensor_id == asset_or_sensor_id) if annotations_after is not None: query = query.filter( From 12539ff707c352998626dacfb9a42f043bf87132 Mon Sep 17 00:00:00 2001 From: Ahmad Wahid Date: Thu, 18 Jul 2024 19:32:48 +0200 Subject: [PATCH 3/5] import annotations --- flexmeasures/api/dev/sensors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 8dcc10be2..b5dd7786a 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import warnings from typing import Any From b00a806366a066e1077ae0820658106f6681145b Mon Sep 17 00:00:00 2001 From: Ahmad Wahid Date: Mon, 16 Sep 2024 18:31:18 +0200 Subject: [PATCH 4/5] fix: search asset annotations Signed-off-by: Ahmad Wahid --- flexmeasures/data/models/generic_assets.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 5d5e40d3f..e40d81b38 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -15,7 +15,11 @@ from timely_beliefs import BeliefsDataFrame, utils as tb_utils from flexmeasures.data import db -from flexmeasures.data.models.annotations import Annotation, to_annotation_frame +from flexmeasures.data.models.annotations import ( + Annotation, + to_annotation_frame, + GenericAssetAnnotationRelationship, +) from flexmeasures.data.models.charts import chart_type_to_chart_specs from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.parsing_utils import parse_source_arg @@ -358,7 +362,7 @@ def search_annotations( include_account_annotations: bool = False, as_frame: bool = False, sensor_id: int = None, - relationship_module: Any = None, + relationship_module: Any = GenericAssetAnnotationRelationship, ) -> list[Annotation] | pd.DataFrame: """Return annotations assigned to this asset, and optionally, also those assigned to the asset's account. From d154e401057e172e4d285d2e850fce32e1045c6b Mon Sep 17 00:00:00 2001 From: Ahmad Wahid Date: Fri, 27 Sep 2024 19:43:57 +0200 Subject: [PATCH 5/5] refactor query --- flexmeasures/data/queries/annotations.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/queries/annotations.py b/flexmeasures/data/queries/annotations.py index 8bbf4c0fe..b509d4dd4 100644 --- a/flexmeasures/data/queries/annotations.py +++ b/flexmeasures/data/queries/annotations.py @@ -22,19 +22,18 @@ def query_asset_annotations( annotation_type: str | None = None, ) -> Query: """Match annotations assigned to the given asset.""" - query = select(Annotation) + query = select(Annotation).join( + relationship_module, relationship_module.annotation_id == Annotation.id + ) + if relationship_module is GenericAssetAnnotationRelationship: - query = query.join( - GenericAssetAnnotationRelationship, - GenericAssetAnnotationRelationship.annotation_id == Annotation.id, - ).filter( + query = query.filter( GenericAssetAnnotationRelationship.generic_asset_id == asset_or_sensor_id ) else: - query = query.join( - SensorAnnotationRelationship, - SensorAnnotationRelationship.annotation_id == Annotation.id, - ).filter(SensorAnnotationRelationship.sensor_id == asset_or_sensor_id) + query = query.filter( + SensorAnnotationRelationship.sensor_id == asset_or_sensor_id + ) if annotations_after is not None: query = query.filter(