diff --git a/oonidata/src/oonidata/models/analysis.py b/oonidata/src/oonidata/models/analysis.py index 6da88959..2906bbc3 100644 --- a/oonidata/src/oonidata/models/analysis.py +++ b/oonidata/src/oonidata/models/analysis.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import List, Optional -from .base import table_model, ProcessingMeta +from .base import table_model from oonidata.models.observations import MeasurementMeta, ProbeMeta @@ -20,7 +20,6 @@ class WebAnalysis: probe_meta: ProbeMeta measurement_meta: MeasurementMeta - processing_meta: ProcessingMeta analysis_id: str observation_id: str diff --git a/oonipipeline/src/oonipipeline/analysis/datasources.py b/oonipipeline/src/oonipipeline/analysis/datasources.py index a7aa3f18..e052838d 100644 --- a/oonipipeline/src/oonipipeline/analysis/datasources.py +++ b/oonipipeline/src/oonipipeline/analysis/datasources.py @@ -2,7 +2,6 @@ from datetime import date, timedelta from typing import Generator, List, Optional -from oonidata.models.base import ProcessingMeta from oonidata.models.observations import MeasurementMeta, ProbeMeta, WebObservation from ..db.connections import ClickhouseConnection @@ -27,12 +26,10 @@ def iter_web_observations( measurement_meta_cols = [f.name for f in dataclasses.fields(MeasurementMeta)] probe_meta_cols = [f.name for f in dataclasses.fields(ProbeMeta)] - processing_meta_cols = [f.name for f in dataclasses.fields(ProcessingMeta)] obs_cols = [f.name for f in dataclasses.fields(WebObservation)] obs_cols.remove("probe_meta") obs_cols.remove("measurement_meta") - obs_cols.remove("processing_meta") - column_names = measurement_meta_cols + probe_meta_cols + processing_meta_cols + obs_cols + column_names = measurement_meta_cols + probe_meta_cols + obs_cols q = "SELECT (" q += ",\n".join(column_names) @@ -62,7 +59,7 @@ def iter_web_observations( # TODO(art): this is super sketchy. # We need to do this in order to obtain the correct offsets into the queried columns - # Basically probe_meta, measurement_meta and processing_meta are class + # Basically probe_meta, measurement_meta are class # attributes that are composed into the dataclass, however in the # database they need to be stored flat, since nesting is not desirable. # What we are doing here is figuring out how to construct the nested @@ -83,18 +80,14 @@ def iter_web_observations( ], ) ) - processing_meta = dict( - zip(processing_meta_cols, row[: len(processing_meta_cols)]) - ) rest = dict( - zip(obs_cols, row[len(measurement_meta_cols) + len(probe_meta_cols) + len(processing_meta_cols) :]) + zip(obs_cols, row[len(measurement_meta_cols) + len(probe_meta_cols) :]) ) obs_group.append( WebObservation( measurement_meta=MeasurementMeta(**measurement_meta), probe_meta=ProbeMeta(**probe_meta), - processing_meta=ProcessingMeta(**processing_meta), **rest, ) ) diff --git a/oonipipeline/src/oonipipeline/analysis/web_analysis.py b/oonipipeline/src/oonipipeline/analysis/web_analysis.py index 7c442949..6220159c 100644 --- a/oonipipeline/src/oonipipeline/analysis/web_analysis.py +++ b/oonipipeline/src/oonipipeline/analysis/web_analysis.py @@ -12,7 +12,6 @@ List, Dict, ) -from oonidata.models.base import ProcessingMeta from oonidata.models.analysis import WebAnalysis from oonidata.models.observations import WebControlObservation, WebObservation @@ -686,9 +685,6 @@ def make_web_analysis( website_analysis = WebAnalysis( measurement_meta=web_o.measurement_meta, probe_meta=web_o.probe_meta, - processing_meta=ProcessingMeta( - processing_start_time=datetime.now(timezone.utc) - ), observation_id=web_o.observation_id, created_at=created_at, analysis_id=f"{web_o.measurement_meta.measurement_uid}_{idx}", @@ -980,7 +976,4 @@ def make_web_analysis( http_analysis.is_http_fp_false_positive ) - website_analysis.processing_meta.processing_start_time = datetime.now( - timezone.utc - ) yield website_analysis diff --git a/oonipipeline/src/oonipipeline/dataviz/main.py b/oonipipeline/src/oonipipeline/dataviz/main.py index 6ed6309b..647479c8 100644 --- a/oonipipeline/src/oonipipeline/dataviz/main.py +++ b/oonipipeline/src/oonipipeline/dataviz/main.py @@ -30,19 +30,17 @@ templates = Jinja2Templates(directory=f"{current_dir}/templates") -def extract_meta(orig_obs_list) -> Tuple[List, Dict, Dict, Dict]: +def extract_meta(orig_obs_list) -> Tuple[List, Dict, Dict]: obs_list = [] measurement_meta = {} probe_meta = {} - processing_meta = {} for obs in orig_obs_list: wo_dict = asdict(obs) measurement_meta = wo_dict.pop("probe_meta", None) probe_meta = wo_dict.pop("measurement_meta", None) - processing_meta = wo_dict.pop("processing_meta", None) obs_list.append(wo_dict) - return obs_list, measurement_meta, probe_meta, processing_meta + return obs_list, measurement_meta, probe_meta @app.get("/analysis/") @@ -116,7 +114,7 @@ def analysis_by_msmt( # print(analysis_transcript_list) web_analysis_list, _, _, _ = extract_meta(web_analysis) - web_observations_list, measurement_meta, probe_meta, processing_meta = extract_meta( + web_observations_list, measurement_meta, probe_meta = extract_meta( web_observations ) return templates.TemplateResponse( @@ -132,7 +130,7 @@ def analysis_by_msmt( web_observations=web_observations_list, measurement_meta=measurement_meta, probe_meta=probe_meta, - processing_meta=processing_meta, + processing_meta={}, loni_blocked_dict=dict(zip(wer.loni_blocked_keys, wer.loni_blocked_values)), loni_blocked_value=sum(wer.loni_blocked_values), loni_down_dict=dict(zip(wer.loni_down_keys, wer.loni_down_values)), @@ -157,10 +155,10 @@ def observations_by_msmt( msmt, netinfodb=netinfodb ) - web_observations_list, probe_meta, measurement_meta, processing_meta = extract_meta( + web_observations_list, probe_meta, measurement_meta = extract_meta( web_observations ) - web_control_observations, _, _, _ = extract_meta(web_control_observations) + web_control_observations, _, _ = extract_meta(web_control_observations) return templates.TemplateResponse( request=request, name="observations.html", diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py b/oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py index 8d30f771..db16a9e0 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py @@ -1,6 +1,5 @@ from datetime import datetime, timezone from typing import List, Tuple -from oonidata.models.base import ProcessingMeta from oonidata.models.nettests import BrowserWeb from oonidata.models.observations import WebObservation @@ -12,10 +11,6 @@ def make_observations(self, msmt: BrowserWeb) -> Tuple[List[WebObservation]]: bw_obs = WebObservation( measurement_meta=self.measurement_meta, probe_meta=self.probe_meta, - processing_meta=ProcessingMeta( - processing_start_time=datetime.now(timezone.utc), - processing_end_time=datetime.now(timezone.utc), - ), http_failure=msmt.test_keys.result, http_runtime=msmt.test_keys.load_time_ms, ) diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py b/oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py index 82433e86..b6136639 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py @@ -3,7 +3,6 @@ from typing import Dict, List, Tuple from urllib.parse import urlparse from oonidata.datautils import is_ip_bogon -from oonidata.models.base import ProcessingMeta from oonidata.models.nettests import WebConnectivity from oonidata.models.observations import ( MeasurementMeta, @@ -44,9 +43,6 @@ def make_web_control_observations( if msmt.test_keys.control.dns and msmt.test_keys.control.dns.failure: obs = WebControlObservation( measurement_meta=measurement_meta, - processing_meta=ProcessingMeta( - processing_start_time=datetime.now(timezone.utc) - ), hostname=hostname, created_at=created_at, ) @@ -64,9 +60,6 @@ def make_web_control_observations( obs = WebControlObservation( measurement_meta=measurement_meta, - processing_meta=ProcessingMeta( - processing_start_time=datetime.now(timezone.utc) - ), hostname=hostname, created_at=created_at, ) @@ -89,9 +82,6 @@ def make_web_control_observations( measurement_meta=measurement_meta, hostname=p.hostname, port=p.port, - processing_meta=ProcessingMeta( - processing_start_time=datetime.now(timezone.utc) - ), created_at=created_at, ) @@ -127,9 +117,6 @@ def make_web_control_observations( for ip in dns_ips - mapped_dns_ips: obs = WebControlObservation( measurement_meta=measurement_meta, - processing_meta=ProcessingMeta( - processing_start_time=datetime.now(timezone.utc) - ), hostname=hostname, created_at=created_at, ) @@ -140,9 +127,6 @@ def make_web_control_observations( if msmt.test_keys.control.http_request: obs = WebControlObservation( measurement_meta=measurement_meta, - processing_meta=ProcessingMeta( - processing_start_time=datetime.now(timezone.utc) - ), hostname=hostname, created_at=created_at, ) @@ -154,7 +138,6 @@ def make_web_control_observations( for idx, obs in enumerate(web_ctrl_obs): obs.observation_id = f"{obs.measurement_meta.measurement_uid}_{idx}" - obs.processing_meta.processing_end_time = datetime.now(timezone.utc) return web_ctrl_obs diff --git a/oonipipeline/tests/test_ctrl.py b/oonipipeline/tests/test_ctrl.py index a073397b..90f844e4 100644 --- a/oonipipeline/tests/test_ctrl.py +++ b/oonipipeline/tests/test_ctrl.py @@ -1,7 +1,6 @@ from datetime import date, datetime, timezone import time -from oonidata.models.base import ProcessingMeta from oonidata.models.observations import MeasurementMeta, ProbeMeta, WebObservation from oonipipeline.analysis.datasources import iter_web_observations @@ -90,10 +89,6 @@ def test_web_ground_truth_from_clickhouse(db, datadir, netinfodb, tmp_path): WebObservation( probe_meta=DUMMY_PROBE_META, measurement_meta=DUMMY_MEASUREMENT_META, - processing_meta=ProcessingMeta( - processing_start_time=datetime.now(timezone.utc), - processing_end_time=datetime.now(timezone.utc), - ), # The only things we look at to find the groundtruth are hostname, ip, http_request_url hostname="explorer.ooni.org", ip="37.218.242.149",