Remove all references to ProcessingMeta

oonidata/src/oonidata/models/analysis.py

@@ -3,7 +3,7 @@
 from datetime import datetime
 from typing import List, Optional
 
-from .base import table_model, ProcessingMeta
+from .base import table_model
 from oonidata.models.observations import MeasurementMeta, ProbeMeta
 
 
@@ -20,7 +20,6 @@
 class WebAnalysis:
     probe_meta: ProbeMeta
     measurement_meta: MeasurementMeta
-    processing_meta: ProcessingMeta
 
     analysis_id: str
     observation_id: str

oonipipeline/src/oonipipeline/analysis/datasources.py

@@ -2,7 +2,6 @@
 from datetime import date, timedelta
 from typing import Generator, List, Optional
 
-from oonidata.models.base import ProcessingMeta
 from oonidata.models.observations import MeasurementMeta, ProbeMeta, WebObservation
 
 from ..db.connections import ClickhouseConnection
@@ -27,12 +26,10 @@ def iter_web_observations(
 
     measurement_meta_cols = [f.name for f in dataclasses.fields(MeasurementMeta)]
     probe_meta_cols = [f.name for f in dataclasses.fields(ProbeMeta)]
-    processing_meta_cols = [f.name for f in dataclasses.fields(ProcessingMeta)]
     obs_cols = [f.name for f in dataclasses.fields(WebObservation)]
     obs_cols.remove("probe_meta")
     obs_cols.remove("measurement_meta")
-    obs_cols.remove("processing_meta")
-    column_names = measurement_meta_cols + probe_meta_cols + processing_meta_cols + obs_cols
+    column_names = measurement_meta_cols + probe_meta_cols + obs_cols
 
     q = "SELECT ("
     q += ",\n".join(column_names)
@@ -62,7 +59,7 @@ def iter_web_observations(
 
         # TODO(art): this is super sketchy.
         # We need to do this in order to obtain the correct offsets into the queried columns
-        # Basically probe_meta, measurement_meta and processing_meta are class
+        # Basically probe_meta, measurement_meta are class
         # attributes that are composed into the dataclass, however in the
         # database they need to be stored flat, since nesting is not desirable.
         # What we are doing here is figuring out how to construct the nested
@@ -83,18 +80,14 @@ def iter_web_observations(
                 ],
             )
         )
-        processing_meta = dict(
-            zip(processing_meta_cols, row[: len(processing_meta_cols)])
-        )
 
         rest = dict(
-            zip(obs_cols, row[len(measurement_meta_cols) + len(probe_meta_cols) + len(processing_meta_cols) :])
+            zip(obs_cols, row[len(measurement_meta_cols) + len(probe_meta_cols) :])
         )
         obs_group.append(
             WebObservation(
                 measurement_meta=MeasurementMeta(**measurement_meta),
                 probe_meta=ProbeMeta(**probe_meta),
-                processing_meta=ProcessingMeta(**processing_meta),
                 **rest,
             )
         )

oonipipeline/src/oonipipeline/analysis/web_analysis.py

@@ -12,7 +12,6 @@
     List,
     Dict,
 )
-from oonidata.models.base import ProcessingMeta
 from oonidata.models.analysis import WebAnalysis
 from oonidata.models.observations import WebControlObservation, WebObservation
 
@@ -686,9 +685,6 @@ def make_web_analysis(
         website_analysis = WebAnalysis(
             measurement_meta=web_o.measurement_meta,
             probe_meta=web_o.probe_meta,
-            processing_meta=ProcessingMeta(
-                processing_start_time=datetime.now(timezone.utc)
-            ),
             observation_id=web_o.observation_id,
             created_at=created_at,
             analysis_id=f"{web_o.measurement_meta.measurement_uid}_{idx}",
@@ -980,7 +976,4 @@ def make_web_analysis(
                 http_analysis.is_http_fp_false_positive
             )
 
-        website_analysis.processing_meta.processing_start_time = datetime.now(
-            timezone.utc
-        )
         yield website_analysis

oonipipeline/src/oonipipeline/dataviz/main.py

@@ -30,19 +30,17 @@
 templates = Jinja2Templates(directory=f"{current_dir}/templates")
 
 
-def extract_meta(orig_obs_list) -> Tuple[List, Dict, Dict, Dict]:
+def extract_meta(orig_obs_list) -> Tuple[List, Dict, Dict]:
     obs_list = []
 
     measurement_meta = {}
     probe_meta = {}
-    processing_meta = {}
     for obs in orig_obs_list:
         wo_dict = asdict(obs)
         measurement_meta = wo_dict.pop("probe_meta", None)
         probe_meta = wo_dict.pop("measurement_meta", None)
-        processing_meta = wo_dict.pop("processing_meta", None)
         obs_list.append(wo_dict)
-    return obs_list, measurement_meta, probe_meta, processing_meta
+    return obs_list, measurement_meta, probe_meta
 
 
 @app.get("/analysis/")
@@ -116,7 +114,7 @@ def analysis_by_msmt(
     # print(analysis_transcript_list)
 
     web_analysis_list, _, _, _ = extract_meta(web_analysis)
-    web_observations_list, measurement_meta, probe_meta, processing_meta = extract_meta(
+    web_observations_list, measurement_meta, probe_meta = extract_meta(
         web_observations
     )
     return templates.TemplateResponse(
@@ -132,7 +130,7 @@ def analysis_by_msmt(
             web_observations=web_observations_list,
             measurement_meta=measurement_meta,
             probe_meta=probe_meta,
-            processing_meta=processing_meta,
+            processing_meta={},
             loni_blocked_dict=dict(zip(wer.loni_blocked_keys, wer.loni_blocked_values)),
             loni_blocked_value=sum(wer.loni_blocked_values),
             loni_down_dict=dict(zip(wer.loni_down_keys, wer.loni_down_values)),
@@ -157,10 +155,10 @@ def observations_by_msmt(
         msmt, netinfodb=netinfodb
     )
 
-    web_observations_list, probe_meta, measurement_meta, processing_meta = extract_meta(
+    web_observations_list, probe_meta, measurement_meta = extract_meta(
         web_observations
     )
-    web_control_observations, _, _, _ = extract_meta(web_control_observations)
+    web_control_observations, _, _ = extract_meta(web_control_observations)
     return templates.TemplateResponse(
         request=request,
         name="observations.html",

oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py

@@ -1,6 +1,5 @@
 from datetime import datetime, timezone
 from typing import List, Tuple
-from oonidata.models.base import ProcessingMeta
 from oonidata.models.nettests import BrowserWeb
 from oonidata.models.observations import WebObservation
 
@@ -12,10 +11,6 @@ def make_observations(self, msmt: BrowserWeb) -> Tuple[List[WebObservation]]:
         bw_obs = WebObservation(
             measurement_meta=self.measurement_meta,
             probe_meta=self.probe_meta,
-            processing_meta=ProcessingMeta(
-                processing_start_time=datetime.now(timezone.utc),
-                processing_end_time=datetime.now(timezone.utc),
-            ),
             http_failure=msmt.test_keys.result,
             http_runtime=msmt.test_keys.load_time_ms,
         )

oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py

@@ -3,7 +3,6 @@
 from typing import Dict, List, Tuple
 from urllib.parse import urlparse
 from oonidata.datautils import is_ip_bogon
-from oonidata.models.base import ProcessingMeta
 from oonidata.models.nettests import WebConnectivity
 from oonidata.models.observations import (
     MeasurementMeta,
@@ -44,9 +43,6 @@ def make_web_control_observations(
     if msmt.test_keys.control.dns and msmt.test_keys.control.dns.failure:
         obs = WebControlObservation(
             measurement_meta=measurement_meta,
-            processing_meta=ProcessingMeta(
-                processing_start_time=datetime.now(timezone.utc)
-            ),
             hostname=hostname,
             created_at=created_at,
         )
@@ -64,9 +60,6 @@ def make_web_control_observations(
 
             obs = WebControlObservation(
                 measurement_meta=measurement_meta,
-                processing_meta=ProcessingMeta(
-                    processing_start_time=datetime.now(timezone.utc)
-                ),
                 hostname=hostname,
                 created_at=created_at,
             )
@@ -89,9 +82,6 @@ def make_web_control_observations(
                     measurement_meta=measurement_meta,
                     hostname=p.hostname,
                     port=p.port,
-                    processing_meta=ProcessingMeta(
-                        processing_start_time=datetime.now(timezone.utc)
-                    ),
                     created_at=created_at,
                 )
 
@@ -127,9 +117,6 @@ def make_web_control_observations(
     for ip in dns_ips - mapped_dns_ips:
         obs = WebControlObservation(
             measurement_meta=measurement_meta,
-            processing_meta=ProcessingMeta(
-                processing_start_time=datetime.now(timezone.utc)
-            ),
             hostname=hostname,
             created_at=created_at,
         )
@@ -140,9 +127,6 @@ def make_web_control_observations(
     if msmt.test_keys.control.http_request:
         obs = WebControlObservation(
             measurement_meta=measurement_meta,
-            processing_meta=ProcessingMeta(
-                processing_start_time=datetime.now(timezone.utc)
-            ),
             hostname=hostname,
             created_at=created_at,
         )
@@ -154,7 +138,6 @@ def make_web_control_observations(
 
     for idx, obs in enumerate(web_ctrl_obs):
         obs.observation_id = f"{obs.measurement_meta.measurement_uid}_{idx}"
-        obs.processing_meta.processing_end_time = datetime.now(timezone.utc)
 
     return web_ctrl_obs
 

oonipipeline/tests/test_ctrl.py

@@ -1,7 +1,6 @@
 from datetime import date, datetime, timezone
 import time
 
-from oonidata.models.base import ProcessingMeta
 from oonidata.models.observations import MeasurementMeta, ProbeMeta, WebObservation
 
 from oonipipeline.analysis.datasources import iter_web_observations
@@ -90,10 +89,6 @@ def test_web_ground_truth_from_clickhouse(db, datadir, netinfodb, tmp_path):
         WebObservation(
             probe_meta=DUMMY_PROBE_META,
             measurement_meta=DUMMY_MEASUREMENT_META,
-            processing_meta=ProcessingMeta(
-                processing_start_time=datetime.now(timezone.utc),
-                processing_end_time=datetime.now(timezone.utc),
-            ),
             # The only things we look at to find the groundtruth are hostname, ip, http_request_url
             hostname="explorer.ooni.org",
             ip="37.218.242.149",