diff --git a/ooniapi/common/src/common/clickhouse.py b/ooniapi/common/src/common/clickhouse.py new file mode 100644 index 00000000..bd9e7872 --- /dev/null +++ b/ooniapi/common/src/common/clickhouse.py @@ -0,0 +1,4 @@ +from clickhouse_sqlalchemy import get_declarative_base + + +Base = get_declarative_base() \ No newline at end of file diff --git a/ooniapi/common/src/common/routers.py b/ooniapi/common/src/common/routers.py index 305d3079..a6602b1a 100644 --- a/ooniapi/common/src/common/routers.py +++ b/ooniapi/common/src/common/routers.py @@ -1,4 +1,5 @@ from datetime import date, datetime +from typing import Union from pydantic import BaseModel as PydandicBaseModel from pydantic import ConfigDict diff --git a/ooniapi/services/oonimeasurements/pyproject.toml b/ooniapi/services/oonimeasurements/pyproject.toml index 751f0106..628176f5 100644 --- a/ooniapi/services/oonimeasurements/pyproject.toml +++ b/ooniapi/services/oonimeasurements/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "fastapi ~= 0.108.0", "psycopg2 ~= 2.9.5", "clickhouse-driver ~= 0.2.6", + "clickhouse-sqlalchemy ~= 0.3.2", "sqlalchemy ~= 2.0.27", "ujson ~= 5.9.0", "urllib3 ~= 2.1.0", diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/dependencies.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/dependencies.py index f3d86738..60b177c8 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/dependencies.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/dependencies.py @@ -7,9 +7,10 @@ from .common.config import Settings from .common.dependencies import get_settings + def get_clickhouse_session(settings: Annotated[Settings, Depends(get_settings)]): db = Clickhouse.from_url(settings.clickhouse_url) try: yield db - finally: + finally: db.disconnect() diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py index b6201f7d..01d4fe79 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py @@ -8,7 +8,8 @@ from prometheus_fastapi_instrumentator import Instrumentator -from .routers import aggregation, measurements +from .routers.v1 import aggregation +from .routers.v1 import measurements from .dependencies import get_clickhouse_session from .common.dependencies import get_settings diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/aggregation.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/v1/aggregation.py similarity index 90% rename from ooniapi/services/oonimeasurements/src/oonimeasurements/routers/aggregation.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/v1/aggregation.py index 36ab79f1..03abdee0 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/aggregation.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/v1/aggregation.py @@ -1,15 +1,13 @@ """ Aggregation API -The routes are mounted under /api """ -from datetime import datetime, timedelta, date +from datetime import datetime, timedelta, date, timezone from typing import List, Any, Dict, Optional, Union import logging from fastapi import APIRouter, Depends, Query, HTTPException from fastapi.responses import Response -from pydantic import BaseModel from typing_extensions import Annotated from clickhouse_driver import Client as ClickhouseClient @@ -20,8 +18,8 @@ from oonimeasurements.common.clickhouse_utils import query_click, query_click_one_row from oonimeasurements.common.utils import jerror, commasplit, convert_to_csv -from ..dependencies import get_clickhouse_session - +from ...dependencies import get_clickhouse_session +from ...common.routers import BaseModel router = APIRouter() @@ -116,7 +114,7 @@ class AggregationResult(BaseModel): failure_count: int ok_count: int measurement_count: int - measurement_start_day: Optional[date] = None + measurement_start_day: Optional[str] = None blocking_type: Optional[str] = None category_code: Optional[str] = None domain: Optional[str] = None @@ -134,8 +132,9 @@ class MeasurementAggregation(BaseModel): @router.get( - "/v1/aggregation", - response_model_exclude_none=True + "/v1/aggregation", + response_model_exclude_none=True, + response_model=MeasurementAggregation, ) async def get_measurements( response: Response, @@ -247,7 +246,9 @@ async def get_measurements( int(i[2:]) if i.startswith("AS") else i for i in commasplit(probe_asn) ] except ValueError: - raise HTTPException(status_code=400, detail="Invalid ASN value in parameter probe_asn") + raise HTTPException( + status_code=400, detail="Invalid ASN value in parameter probe_asn" + ) probe_cc_s = [] if probe_cc: @@ -342,12 +343,16 @@ async def get_measurements( group_by: List = [] try: if axis_x == "measurement_start_day": - group_by_date(since, until, time_grain, cols, colnames, group_by) + time_grain = group_by_date( + since, until, time_grain, cols, colnames, group_by + ) elif axis_x: add_axis(axis_x, cols, colnames, group_by) if axis_y == "measurement_start_day": - group_by_date(since, until, time_grain, cols, colnames, group_by) + time_grain = group_by_date( + since, until, time_grain, cols, colnames, group_by + ) elif axis_y: add_axis(axis_y, cols, colnames, group_by) @@ -372,7 +377,17 @@ async def get_measurements( try: if dimension_cnt > 0: - r: Any = list(query_click(db, query, query_params, query_prio=4)) + str_format = "%Y-%m-%d" + if time_grain == "hour": + str_format = "%Y-%m-%dT%H:%M:%SZ" + r: Any = [] + for row in query_click(db, query, query_params, query_prio=4): + ## Handle the difference in formatting between hourly and daily measurement_start_day + if "measurement_start_day" in row: + row["measurement_start_day"] = row[ + "measurement_start_day" + ].strftime(str_format) + r.append(row) else: r = query_click_one_row(db, query, query_params, query_prio=4) @@ -410,7 +425,8 @@ async def get_measurements( elapsed_seconds=pq.elapsed, ), result=r, - ).model_dump(exclude_none=True) + ) except Exception as e: + print(e) raise HTTPException(status_code=400, detail=str(e)) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/measurements.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/v1/measurements.py similarity index 65% rename from ooniapi/services/oonimeasurements/src/oonimeasurements/routers/measurements.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/v1/measurements.py index 5672ef1c..c889c200 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/measurements.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/v1/measurements.py @@ -1,31 +1,29 @@ """ Measurements API -The routes are mounted under /api """ from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import List, Optional, Any, Dict, Union +from typing import List, Optional, Any, Dict, Union, TypedDict, Tuple import gzip import json import logging import math import time -import ujson -import urllib3 - -from fastapi import ( - APIRouter, - Depends, - Query, - HTTPException, - Header, - Response, - Request -) +import ujson +import urllib3 + +from fastapi import APIRouter, Depends, Query, HTTPException, Header, Response, Request +from fastapi.encoders import jsonable_encoder +from fastapi.responses import JSONResponse from typing_extensions import Annotated +from pydantic import Field + +import sqlalchemy as sa +from sqlalchemy import tuple_, Row, sql +from sqlalchemy.orm import Session from sqlalchemy.sql.expression import and_, text, select, column from sqlalchemy.sql.expression import text as sql_text from sqlalchemy.sql.expression import table as sql_table @@ -37,76 +35,62 @@ from urllib.request import urlopen from urllib.parse import urljoin, urlencode -from ..common.config import Settings -from ..common.dependencies import get_settings -from ..common.routers import BaseModel -from ..common.utils import setcacheresponse, commasplit, setnocacheresponse -from ..common.clickhouse_utils import query_click, query_click_one_row -from ..dependencies import get_clickhouse_session - +from ...common.config import Settings +from ...common.dependencies import get_settings +from ...common.routers import BaseModel +from ...common.utils import setcacheresponse, commasplit, setnocacheresponse +from ...common.clickhouse_utils import query_click, query_click_one_row +from ...dependencies import get_clickhouse_session log = logging.getLogger(__name__) router = APIRouter() -FASTPATH_MSM_ID_PREFIX = "temp-fid-" -FASTPATH_SERVER = "fastpath.ooni.nu" -FASTPATH_PORT = 8000 - - urllib_pool = urllib3.PoolManager() -MsmtNotFound = HTTPException(status_code=500, detail="Measurement not found") - +MeasurementNotFound = HTTPException(status_code=500, detail="Measurement not found") +ReportInputNotFound = HTTPException( + status_code=500, detail="Report and input not found" +) +MeasurementFileNotFound = HTTPException( + status_code=404, detail="Measurement S3 file not found" +) +AbortMeasurementList = HTTPException( + status_code=403, detail="Disallowed list_measurements" +) +Abort504 = HTTPException(status_code=504, detail="Error in list_measurements") -def measurement_uid_to_s3path_linenum(db: ClickhouseClient, measurement_uid: str): - # TODO: cleanup this - query = """SELECT s3path, linenum FROM jsonl - PREWHERE (report_id, input) IN ( - SELECT report_id, input FROM fastpath WHERE measurement_uid = :uid - ) - LIMIT 1""" - query_params = dict(uid=measurement_uid) - lookup = query_click_one_row(db, sql_text(query), query_params, query_prio=3) - if lookup is None: - raise MsmtNotFound - s3path = lookup["s3path"] - linenum = lookup["linenum"] - return s3path, linenum +@router.get( + "/v1/files", + tags=["files"], +) +def list_files(): + """List files - unsupported""" + response = JSONResponse(content=jsonable_encoder({"msg": "not implemented"})) + setcacheresponse("1d", response) + return response def _fetch_jsonl_measurement_body_from_s3( s3path: str, linenum: int, + s3_bucket_name: str, ) -> bytes: - baseurl = f"https://{settings.s3_bucket_name}.s3.amazonaws.com/" + """ + Fetch jsonl from S3, decompress it, extract single msmt + """ + baseurl = f"https://{s3_bucket_name}.s3.amazonaws.com/" url = urljoin(baseurl, s3path) - log.info(f"Fetching {url}") + + log.info(f"Fetching {url}") r = urlopen(url) + f = gzip.GzipFile(fileobj=r, mode="r") for n, line in enumerate(f): if n == linenum: return line - - raise MsmtNotFound - - -class NotImplemented(BaseModel): - msg: str - - -@router.get( - "/v1/files", - tags=["files"], - response_model=NotImplemented, -) -def list_files( - response: Response, -): - """List files - unsupported""" - setcacheresponse("1d", response) - return NotImplemented(msg="not implemented") + raise MeasurementFileNotFound @router.get( @@ -116,86 +100,32 @@ def list_files( def get_measurement( measurement_uid: str, download: bool, - response: Response, - db=Depends(get_clickhouse_session) + db=Depends(get_clickhouse_session), + settings=Depends(get_settings), ): """ Get one measurement by measurement_id, Returns only the measurement without extra data from the database """ assert measurement_uid - try: - s3path, linenum = measurement_uid_to_s3path_linenum(db, measurement_uid) - except: - raise + s3path, linenum = measurement_uid_to_s3path_linenum(db, measurement_uid) log.debug(f"Fetching file {s3path} from S3") - body = _fetch_jsonl_measurement_body_from_s3(s3path, linenum) + body = _fetch_jsonl_measurement_body_from_s3( + s3path, linenum, settings.s3_bucket_name + ) + response = Response(content=body) if download: response.headers["Content-Disposition"] = ( f"attachment; filename=ooni_measurement-{measurement_uid}.json" ) setcacheresponse("1h", response) - response.content = body response.media_type = "application/json" return response -### Fetching measurement bodies - -def report_id_input_to_s3path_linenum(db: ClickhouseClient, report_id: str, input: str): - query = """SELECT s3path, linenum FROM jsonl - PREWHERE report_id = :report_id AND input = :inp - LIMIT 1""" - query_params = dict(inp=input, report_id=report_id) - lookup = query_click_one_row(db, sql_text(query), query_params, query_prio=3) - - if lookup is None: - m = f"Missing row in jsonl table: {report_id} {input}" - log.error(m) - raise HTTPException - - s3path = lookup["s3path"] - linenum = lookup["linenum"] - return s3path, linenum - - -def _fetch_jsonl_measurement_body_clickhouse( - db: ClickhouseClient, - report_id: str, - input: Optional[str], - measurement_uid: Optional[str], -) -> Optional[bytes]: - """ - Fetch jsonl from S3, decompress it, extract single msmt - """ - # TODO: switch to _fetch_measurement_body_by_uid - if measurement_uid is not None: - try: - s3path, linenum = measurement_uid_to_s3path_linenum(db, measurement_uid) - except MsmtNotFound: - log.error(f"Measurement {measurement_uid} not found in jsonl") - return None - - else: - inp = input or "" # NULL/None input is stored as '' - try: - s3path, linenum = report_id_input_to_s3path_linenum(db, report_id, inp) - except Exception: - log.error(f"Measurement {report_id} {inp} not found in jsonl") - return None - - try: - log.debug(f"Fetching file {s3path} from S3") - # TODO(arturo): remove ignore once https://github.com/jsocol/pystatsd/pull/184 lands - return _fetch_jsonl_measurement_body_from_s3(s3path, linenum) # type: ignore - except Exception: # pragma: no cover - log.error(f"Failed to fetch file {s3path} from S3") - return None - - def _unwrap_post(post: dict) -> dict: fmt = post.get("format", "") if fmt == "json": @@ -203,43 +133,22 @@ def _unwrap_post(post: dict) -> dict: raise Exception("Unexpected format") -def _fetch_measurement_body_on_disk_by_msmt_uid(msmt_uid: str) -> Optional[bytes]: - """ - Fetch raw POST from disk, extract msmt - This is used only for msmts that have been processed by the fastpath - but are not uploaded to S3 yet. - YAML msmts not supported: requires implementing normalization here - """ - assert msmt_uid.startswith("20") - tstamp, cc, testname, hash_ = msmt_uid.split("_") - hour = tstamp[:10] - int(hour) # raise if the string does not contain an integer - spooldir = Path("/var/lib/ooniapi/measurements/incoming/") - postf = spooldir / f"{hour}_{cc}_{testname}/{msmt_uid}.post" - log.debug(f"Attempt at reading {postf}") - try: - with postf.open() as f: - post = ujson.load(f) - except FileNotFoundError: - return None - body = _unwrap_post(post) - return ujson.dumps(body).encode() - - -def _fetch_measurement_body_from_hosts(other_collectors: List[str], msmt_uid: str) -> Optional[bytes]: +def _fetch_measurement_body_from_hosts( + other_collectors: List[str], measurement_uid: str +) -> Optional[bytes]: """ Fetch raw POST from another API host, extract msmt - This is used only for msmts that have been processed by the fastpath + Note: This is used only for msmts that have been processed by the fastpath but are not uploaded to S3 yet. """ try: - assert msmt_uid.startswith("20") - tstamp, cc, testname, hash_ = msmt_uid.split("_") + assert measurement_uid.startswith("20") + tstamp, cc, testname, _ = measurement_uid.split("_") hour = tstamp[:10] int(hour) - path = f"{hour}_{cc}_{testname}/{msmt_uid}.post" + path = f"{hour}_{cc}_{testname}/{measurement_uid}.post" except Exception: - log.info("Error", exc_info=True) + log.info(f"Failed to process measurement {measurement_uid}", exc_info=True) return None for hostname in other_collectors: @@ -248,102 +157,157 @@ def _fetch_measurement_body_from_hosts(other_collectors: List[str], msmt_uid: st try: r = urllib_pool.request("GET", url) if r.status == 404: - log.debug("not found") + log.error(f"Measurement {measurement_uid} not found on host {hostname}") continue elif r.status != 200: - log.error(f"unexpected status {r.status}") + log.error( + f"Unexpected status {r.status} for {measurement_uid} on host {hostname}" + ) continue post = ujson.loads(r.data) body = _unwrap_post(post) return ujson.dumps(body).encode() except Exception: - log.info("Error", exc_info=True) + log.info( + f"Failed to load fetch {measurement_uid} from {hostname}", exc_info=True + ) pass return None +def measurement_uid_to_s3path_linenum(db: ClickhouseClient, measurement_uid: str): + """ + Fetch measurement S3 location using measurement_uid + """ + # TODO: cleanup this + query = """SELECT s3path, linenum FROM jsonl + PREWHERE (report_id, input) IN ( + SELECT report_id, input FROM fastpath WHERE measurement_uid = :uid + ) + LIMIT 1""" + query_params = dict(uid=measurement_uid) + lookup = query_click_one_row(db, sql.text(query), query_params, query_prio=3) + if lookup is None: + raise MeasurementNotFound + + s3path = lookup["s3path"] + linenum = lookup["linenum"] + return s3path, linenum + + +def _fetch_jsonl_measurement_body_clickhouse( + db: ClickhouseClient, + measurement_uid: Optional[str], + s3_bucket_name: str, +) -> Optional[bytes]: + """ + Find measurement location in S3 and fetch the measurement + """ + # TODO: switch to _fetch_measurement_body_by_uid + if (measurement_uid is None) or (len(measurement_uid) == 0): + log.error("Invalid measurement_uid provided") + return None + + try: + log.debug(f"Fetching s3path and linenum for measurement {measurement_uid}") + s3path, linenum = measurement_uid_to_s3path_linenum(db, measurement_uid) + + log.debug(f"Fetching file {s3path} from S3") + return _fetch_jsonl_measurement_body_from_s3(s3path, linenum, s3_bucket_name) + except Exception as e: + log.error(f"Failed to fetch {measurement_uid}: {e}", exc_info=True) + return None + + def _fetch_measurement_body( - db: ClickhouseClient, + db: ClickhouseClient, settings: Settings, - report_id: str, - input: Optional[str], - measurement_uid: str -) -> bytes: + report_id: str, + measurement_uid: Optional[str], +) -> str: """ Fetch measurement body from either: - - local measurement spool dir (.post files) - JSONL files on S3 - remote measurement spool dir (another API/collector host) """ - # TODO: uid_cleanup - log.debug(f"Fetching body for {report_id} {input}") + log.debug( + f"Fetching body for report_id: {report_id}, measurement_uid: {measurement_uid}" + ) + u_count = report_id.count("_") - # 5: Current format e.g. - # 20210124T210009Z_webconnectivity_VE_22313_n1_Ojb + # Current format e.g. 20210124T210009Z_webconnectivity_VE_22313_n1_Ojb new_format = u_count == 5 and measurement_uid - fresh = False - if new_format: + if not new_format: + body = _fetch_jsonl_measurement_body_clickhouse( + db, measurement_uid, settings.s3_bucket_name + ) + else: + assert measurement_uid ts = (datetime.now(timezone.utc) - timedelta(hours=1)).strftime("%Y%m%d%H%M") fresh = measurement_uid > ts - other_collectors = settings.other_collectors - # Do the fetching in different orders based on the likelyhood of success - if new_format and fresh: - body = ( - _fetch_measurement_body_on_disk_by_msmt_uid(measurement_uid) - or _fetch_measurement_body_from_hosts(other_collectors, measurement_uid) - or _fetch_jsonl_measurement_body_clickhouse( - db, report_id, input, measurement_uid + # Do the fetching in different orders based on the likelyhood of success + if new_format and fresh: + body = _fetch_measurement_body_from_hosts( + settings.other_collectors, measurement_uid + ) or _fetch_jsonl_measurement_body_clickhouse( + db, measurement_uid, settings.s3_bucket_name ) - ) - - elif new_format and not fresh: - body = ( - _fetch_jsonl_measurement_body_clickhouse( - db, report_id, input, measurement_uid + elif new_format and not fresh: + body = _fetch_jsonl_measurement_body_clickhouse( + db, measurement_uid, settings.s3_bucket_name + ) or _fetch_measurement_body_from_hosts( + settings.other_collectors, measurement_uid ) - or _fetch_measurement_body_on_disk_by_msmt_uid(measurement_uid) - or _fetch_measurement_body_from_hosts(other_collectors, measurement_uid) - ) - - else: - body = _fetch_jsonl_measurement_body_clickhouse( - db, report_id, input, measurement_uid - ) - if body: - return body # type: ignore + return body.decode("utf-8") - raise MsmtNotFound + raise MeasurementNotFound - -def format_msmt_meta(msmt_meta: dict) -> dict: - keys = ( - "input", - "measurement_start_time", - "measurement_uid", - "report_id", - "test_name", - "test_start_time", - "probe_asn", - "probe_cc", - "scores", +class MeasurementMeta(BaseModel): + input: Optional[str] = None + measurement_start_time: Optional[datetime] = None + measurement_uid: Optional[str] = None + report_id: Optional[str] = None + test_name: Optional[str] = None + test_start_time: Optional[datetime] = None + probe_asn: Optional[str] = None + probe_cc: Optional[str] = None + scores: Optional[str] = None + category_code: Optional[str] = None + anomaly: Optional[bool] = None + confirmed: Optional[bool] = None + failure: Optional[bool] = None + raw_measurement: Optional[str] = None + category_code: Optional[str] = None + + +def format_msmt_meta(msmt_meta: dict) -> MeasurementMeta: + formatted_msmt_meta = MeasurementMeta( + input=msmt_meta["input"], + measurement_start_time=msmt_meta["measurement_start_time"], + measurement_uid=msmt_meta["measurement_uid"], + report_id=msmt_meta["report_id"], + test_name=msmt_meta["test_name"], + test_start_time=msmt_meta["test_start_time"], + probe_asn=msmt_meta["probe_asn"], + probe_cc=msmt_meta["probe_cc"], + scores=msmt_meta["scores"], + anomaly=(msmt_meta["anomaly"] == "t"), + confirmed=(msmt_meta["confirmed"] == "t"), + failure=(msmt_meta["failure"] == "t"), + category_code=msmt_meta.get("category_code", None), ) - out = {k: msmt_meta[k] for k in keys} - out["category_code"] = msmt_meta.get("category_code", None) - out["anomaly"] = msmt_meta["anomaly"] == "t" - out["confirmed"] = msmt_meta["confirmed"] == "t" - out["failure"] = msmt_meta["msm_failure"] == "t" - return out + return formatted_msmt_meta def _get_measurement_meta_clickhouse( db: ClickhouseClient, report_id: str, input_: Optional[str] -) -> dict: +) -> MeasurementMeta: # Given report_id + input, fetch measurement data from fastpath table query = "SELECT * FROM fastpath " if input_ is None: @@ -358,31 +322,36 @@ def _get_measurement_meta_clickhouse( """ query_params = dict(input=input_, report_id=report_id) query += "LIMIT 1" - msmt_meta = query_click_one_row(db, sql_text(query), query_params, query_prio=3) + msmt_meta = query_click_one_row(db, sql.text(query), query_params, query_prio=3) if not msmt_meta: - return {} # measurement not found + return MeasurementMeta() # measurement not found if msmt_meta["probe_asn"] == 0: # https://ooni.org/post/2020-ooni-probe-asn-incident-report/ # https://github.com/ooni/explorer/issues/495 - return {} # unwanted + return MeasurementMeta() # unwanted return format_msmt_meta(msmt_meta) -def _get_measurement_meta_by_uid(db: ClickhouseClient, measurement_uid: str) -> dict: +def _get_measurement_meta_by_uid( + db: ClickhouseClient, measurement_uid: str +) -> MeasurementMeta: + """ + Get measurement meta from measurement_uid + """ query = """SELECT * FROM fastpath LEFT OUTER JOIN citizenlab ON citizenlab.url = fastpath.input WHERE measurement_uid = :uid LIMIT 1 """ query_params = dict(uid=measurement_uid) - msmt_meta = query_click_one_row(db, sql_text(query), query_params, query_prio=3) + msmt_meta = query_click_one_row(db, sql.text(query), query_params, query_prio=3) if not msmt_meta: - return {} # measurement not found + return MeasurementMeta() # measurement not found if msmt_meta["probe_asn"] == 0: # https://ooni.org/post/2020-ooni-probe-asn-incident-report/ # https://github.com/ooni/explorer/issues/495 - return {} # unwanted + return MeasurementMeta() # unwanted return format_msmt_meta(msmt_meta) @@ -411,47 +380,54 @@ async def get_raw_measurement( settings=Depends(get_settings), ) -> Response: """ - Get raw measurement body by report_id + input + Get raw measurement body """ # This is used by Explorer to let users download msmts if measurement_uid: - # TODO: uid_cleanup + log.info(f"get_raw_measurement {measurement_uid}") msmt_meta = _get_measurement_meta_by_uid(db, measurement_uid) elif report_id: - # _fetch_measurement_body needs the UID + log.info(f"get_raw_measurement {report_id} {input}") msmt_meta = _get_measurement_meta_clickhouse(db, report_id, input) else: - HTTPException(status_code=400, detail="Either report_id or measurement_uid must be provided") + raise HTTPException( + status_code=400, + detail="Either report_id or measurement_uid must be provided", + ) - body = "{}" - if msmt_meta: - # TODO(arturo): remove ignore once https://github.com/jsocol/pystatsd/pull/184 lands + if msmt_meta.report_id: body = _fetch_measurement_body( - db, settings, msmt_meta["report_id"], msmt_meta["input"], msmt_meta["measurement_uid"] # type: ignore + db, settings, msmt_meta.report_id, msmt_meta.measurement_uid ) + else: + body = {} + response = JSONResponse(content=jsonable_encoder(body)) setcacheresponse("1d", response) - response.content = body - response.media_type = "application/json" return response -class MeasurementMeta(BaseModel): - anomaly: bool - confirmed: bool - category_code: str - failure: bool - input: str - probe_asn: int - probe_cc: str - raw_measurement: str - report_id: str - scores: str - test_name: str - test_start_time: datetime - - -@router.get("/v1/measurement_meta") +class MeasurementBase(BaseModel): + anomaly: Optional[bool] = Field( + default=None, title="check if the measurement is an anomaly" + ) + confirmed: Optional[bool] = Field( + default=None, title="check if the measurement is a confirmed block" + ) + failure: Optional[bool] = Field( + default=None, title="failure check if measurement is marked as failed" + ) + input_: Optional[str] = Field(default=None, alias="input") + probe_asn: Optional[str] = Field(default=None, title="ASN of the measurement probe") + probe_cc: Optional[str] = Field(default=None, title="country code of the probe ASN") + report_id: Optional[str] = Field(default=None, title="report id of the measurement") + scores: Optional[str] = Field( + default=None, title="blocking scores of the measurement" + ) + test_name: Optional[str] = Field(default=None, title="test name of the measurement") + + +@router.get("/v1/measurement_meta", response_model_exclude_unset=True) async def get_measurement_meta( response: Response, measurement_uid: Annotated[ @@ -478,18 +454,14 @@ async def get_measurement_meta( min_length=3, ), ] = None, - full: Annotated[ - bool, - Query(description="Include JSON measurement data") - ] = False, + full: Annotated[bool, Query(description="Include JSON measurement data")] = False, + settings=Depends(get_settings), db=Depends(get_clickhouse_session), ) -> MeasurementMeta: """ Get metadata on one measurement by measurement_uid or report_id + input """ - # TODO: input can be '' or NULL in the fastpath table - fix it - # TODO: see integ tests for TODO items if measurement_uid: log.info(f"get_measurement_meta {measurement_uid}") msmt_meta = _get_measurement_meta_by_uid(db, measurement_uid) @@ -497,68 +469,57 @@ async def get_measurement_meta( log.info(f"get_measurement_meta {report_id} {input}") msmt_meta = _get_measurement_meta_clickhouse(db, report_id, input) else: - raise HTTPException(status_code=400, detail="Either report_id or measurement_uid must be provided") + raise HTTPException( + status_code=400, + detail="Either report_id or measurement_uid must be provided", + ) - assert isinstance(msmt_meta, dict) - setcacheresponse("1m", response) - if not full: - return MeasurementMeta(**msmt_meta) + body = "" + + if not full: # return without raw_measurement + return msmt_meta if msmt_meta == {}: # measurement not found - return MeasurementMeta( - raw_measurement="", - **msmt_meta, - ) + return MeasurementMeta(raw_measurement=body) try: - # TODO: uid_cleanup + assert isinstance(msmt_meta.report_id, str) and isinstance( + msmt_meta.measurement_uid, str + ) body = _fetch_measurement_body( - db, msmt_meta["report_id"], msmt_meta["input"], msmt_meta["measurement_uid"] + db, settings, msmt_meta.report_id, msmt_meta.measurement_uid ) assert isinstance(body, bytes) body = body.decode() except Exception as e: log.error(e, exc_info=True) - body = "" - - return MeasurementMeta( - raw_measurement=body, - **msmt_meta, - ) + msmt_meta.raw_measurement = body + return msmt_meta -### Listing measurements -# TODO(art): Isn't this the same as the above MeasurementMeta? Check it -class MeasurementMeta2(BaseModel): - measurement_url: str - anomaly: Optional[bool] = None - confirmed: Optional[bool] = None - failure: Optional[bool] = None - input: Optional[str] = None - measurement_start_time: Optional[datetime] = None - measurement_uid: Optional[str] = None - probe_asn: Optional[str] = None - probe_cc: Optional[str] = None - report_id: Optional[str] = None - scores: Optional[dict] = None - test_name: Optional[str] = None +class Measurement(MeasurementBase): + measurement_url: str = Field(title="url of the measurement") + measurement_start_time: Optional[datetime] = Field( + default=None, title="start time of the measurement" + ) + measurement_uid: Optional[str] = Field(default=None, title="uid of the measurement") class ResultsMetadata(BaseModel): - count: int - current_page: int - limit: int - next_url: Optional[str] - offset: int - pages: int - query_time: float + count: int = Field(title="") + current_page: int = Field(title="") + limit: int = Field(title="") + next_url: Optional[str] = Field(title="") + offset: int = Field(title="") + pages: int = Field(title="") + query_time: float = Field(title="") class MeasurementList(BaseModel): - metadata: ResultsMetadata - results: List[MeasurementMeta2] + metadata: ResultsMetadata = Field(title="metadata for query results") + results: List[Measurement] = Field(title="measurement results") def genurl(base_url: str, path: str, **kw) -> str: @@ -585,19 +546,17 @@ async def list_measurements( Optional[str], Query(description="Domain to search measurements for", min_length=3), ], - probe_cc: Annotated[ - Optional[str], Query(description="Two letter country code") - ], + probe_cc: Annotated[Optional[str], Query(description="Two letter country code")], probe_asn: Annotated[ Union[str, int, None], Query(description='Autonomous system number in the format "ASXXX"'), ], test_name: Annotated[ - Optional[str], + Optional[str], Query(description="Name of the test"), ], category_code: Annotated[ - Optional[str], + Optional[str], Query(description="Category code from the citizenlab list"), ], since: Annotated[ @@ -721,7 +680,7 @@ async def list_measurements( pages=1, query_time=0.001, ), - results=[MeasurementMeta2(measurement_url="")], + results=[Measurement(measurement_url="")], ) ### Prepare query parameters @@ -757,70 +716,63 @@ async def list_measurements( INULL = "" # Special value for input = NULL to merge rows with FULL OUTER JOIN - ## Create fastpath columns for query - # TODO cast scores, coalesce input as "" fpwhere = [] + query_params: Dict[str, Any] = {} # Populate WHERE clauses and query_params dict if since is not None: - query_params["since"] = since_dt - fpwhere.append(sql_text("measurement_start_time > :since")) + query_params["since"] = since + fpwhere.append(sql.text("measurement_start_time > :since")) if until is not None: - query_params["until"] = until_dt - fpwhere.append(sql_text("measurement_start_time <= :until")) + query_params["until"] = until + fpwhere.append(sql.text("measurement_start_time <= :until")) if report_id: query_params["report_id"] = report_id - fpwhere.append(sql_text("report_id = :report_id")) + fpwhere.append(sql.text("report_id = :report_id")) if probe_cc: if probe_cc == "ZZ": log.info("Refusing list_measurements with probe_cc set to ZZ") - raise HTTPException( - status_code=403, - detail="Refusing list_measurements with probe_cc set to ZZ", - ) + raise AbortMeasurementList query_params["probe_cc"] = probe_cc - fpwhere.append(sql_text("probe_cc = :probe_cc")) + fpwhere.append(sql.text("probe_cc = :probe_cc")) else: - fpwhere.append(sql_text("probe_cc != 'ZZ'")) + fpwhere.append(sql.text("probe_cc != 'ZZ'")) if probe_asn is not None: if probe_asn == 0: log.info("Refusing list_measurements with probe_asn set to 0") - raise HTTPException( - status_code=403, - detail="Refusing list_measurements with probe_asn set to 0", - ) + raise AbortMeasurementList query_params["probe_asn"] = probe_asn - fpwhere.append(sql_text("probe_asn = :probe_asn")) + fpwhere.append(sql.text("probe_asn = :probe_asn")) else: # https://ooni.org/post/2020-ooni-probe-asn-incident-report/ # https://github.com/ooni/explorer/issues/495 - fpwhere.append(sql_text("probe_asn != 0")) + fpwhere.append(sql.text("probe_asn != 0")) if test_name is not None: query_params["test_name"] = test_name - fpwhere.append(sql_text("test_name = :test_name")) + fpwhere.append(sql.text("test_name = :test_name")) if software_versions is not None: query_params["software_versions"] = software_versions - fpwhere.append(sql_text("software_version IN :software_versions")) + fpwhere.append(sql.text("software_version IN :software_versions")) if test_versions is not None: query_params["test_versions"] = test_versions - fpwhere.append(sql_text("test_version IN :test_versions")) + fpwhere.append(sql.text("test_version IN :test_versions")) if engine_versions is not None: query_params["engine_versions"] = engine_versions - fpwhere.append(sql_text("engine_version IN :engine_versions")) + fpwhere.append(sql.text("engine_version IN :engine_versions")) if ooni_run_link_id is not None: query_params["ooni_run_link_id"] = ooni_run_link_id - fpwhere.append(sql_text("ooni_run_link_id = :ooni_run_link_id")) + fpwhere.append(sql.text("ooni_run_link_id = :ooni_run_link_id")) # Filter on anomaly, confirmed and failure: # The database stores anomaly and confirmed as boolean + NULL and stores @@ -830,43 +782,43 @@ async def list_measurements( # See test_list_measurements_filter_flags_fastpath if anomaly is True: - fpwhere.append(sql_text("fastpath.anomaly = 't'")) + fpwhere.append(sql.text("fastpath.anomaly = 't'")) elif anomaly is False: - fpwhere.append(sql_text("fastpath.anomaly = 'f'")) + fpwhere.append(sql.text("fastpath.anomaly = 'f'")) if confirmed is True: - fpwhere.append(sql_text("fastpath.confirmed = 't'")) + fpwhere.append(sql.text("fastpath.confirmed = 't'")) elif confirmed is False: - fpwhere.append(sql_text("fastpath.confirmed = 'f'")) + fpwhere.append(sql.text("fastpath.confirmed = 'f'")) if failure is True: - fpwhere.append(sql_text("fastpath.msm_failure = 't'")) + fpwhere.append(sql.text("fastpath.msm_failure = 't'")) elif failure is False: - fpwhere.append(sql_text("fastpath.msm_failure = 'f'")) + fpwhere.append(sql.text("fastpath.msm_failure = 'f'")) - fpq_table = sql_table("fastpath") + fpq_table = sql.table("fastpath") if input: # input_ overrides domain and category_code query_params["input"] = input - fpwhere.append(sql_text("input = :input")) + fpwhere.append(sql.text("input = :input")) elif domain or category_code: # both domain and category_code can be set at the same time if domain: query_params["domain"] = domain - fpwhere.append(sql_text("domain = :domain")) + fpwhere.append(sql.text("domain = :domain")) if category_code: query_params["category_code"] = category_code fpq_table = fpq_table.join( - sql_table("citizenlab"), - sql_text("citizenlab.url = fastpath.input"), + sql.table("citizenlab"), + sql.text("citizenlab.url = fastpath.input"), ) - fpwhere.append(sql_text("citizenlab.category_code = :category_code")) + fpwhere.append(sql.text("citizenlab.category_code = :category_code")) fp_query = select("*").where(and_(*fpwhere)).select_from(fpq_table) @@ -889,9 +841,11 @@ async def list_measurements( results = [] for row in rows: msmt_uid = row["measurement_uid"] - url = genurl(settings.base_url, "/api/v1/raw_measurement", measurement_uid=msmt_uid) + url = genurl( + settings.base_url, "/api/v1/raw_measurement", measurement_uid=msmt_uid + ) results.append( - MeasurementMeta2( + Measurement( measurement_uid=msmt_uid, measurement_url=url, report_id=row["report_id"], @@ -906,20 +860,37 @@ async def list_measurements( scores=json.loads(row["scores"]), ) ) + + results.append( + { + "measurement_uid": msmt_uid, + "measurement_url": url, + "report_id": row["report_id"], + "probe_cc": row["probe_cc"], + "probe_asn": "AS{}".format(row["probe_asn"]), + "test_name": row["test_name"], + "measurement_start_time": row["measurement_start_time"], + "input": row["input"], + "anomaly": row["anomaly"] == "t", + "confirmed": row["confirmed"] == "t", + "failure": row["msm_failure"] == "t", + "scores": json.loads(row["scores"]), + } + ) except OperationalError as exc: log.error(exc) if isinstance(exc.orig, QueryCanceledError): # FIXME: this is a postgresql exception! # Timeout due to a slow query. Generate metric and do not feed it # to Sentry. - raise HTTPException(status_code=504) + raise Abort504 raise exc # Replace the special value INULL for "input" with None for i, r in enumerate(results): - if r.input == INULL: - results[i].input = None + if r["input"] == INULL: + results[i]["input"] = None pages = -1 count = -1 @@ -988,36 +959,33 @@ async def get_torsf_stats( """ cacheable = False - table = sql_table("fastpath") - where = [sql_text("test_name = 'torsf'")] + cols = [ + sql.text("toDate(measurement_start_time) AS measurement_start_day"), + column("probe_cc"), + sql.text("countIf(anomaly = 't') AS anomaly_count"), + sql.text("countIf(confirmed = 't') AS confirmed_count"), + sql.text("countIf(msm_failure = 't') AS failure_count"), + ] + table = sql.table("fastpath") + where = [sql.text("test_name = 'torsf'")] query_params: Dict[str, Any] = {} if probe_cc: - where.append(sql_text("probe_cc = :probe_cc")) + where.append(sql.text("probe_cc = :probe_cc")) query_params["probe_cc"] = probe_cc if since: - where.append(sql_text("measurement_start_time > :since")) + where.append(sql.text("measurement_start_time > :since")) query_params["since"] = since if until: - where.append(sql_text("measurement_start_time <= :until")) + where.append(sql.text("measurement_start_time <= :until")) query_params["until"] = until cacheable = until < datetime.now() - timedelta(hours=72) # Assemble query where_expr = and_(*where) - query = ( - select( - sql_text("toDate(measurement_start_time) AS measurement_start_day"), - column("probe_cc"), - sql_text("countIf(anomaly = 't') AS anomaly_count"), - sql_text("countIf(confirmed = 't') AS confirmed_count"), - sql_text("countIf(msm_failure = 't') AS failure_count"), - ) - .where(where_expr) - .select_from(table) - ) + query = select(cols).where(where_expr).select_from(table) query = query.group_by(column("measurement_start_day"), column("probe_cc")) query = query.order_by(column("measurement_start_day"), column("probe_cc")) diff --git a/ooniapi/services/oonimeasurements/tests/conftest.py b/ooniapi/services/oonimeasurements/tests/conftest.py index b943a131..9bab9029 100644 --- a/ooniapi/services/oonimeasurements/tests/conftest.py +++ b/ooniapi/services/oonimeasurements/tests/conftest.py @@ -42,7 +42,11 @@ def run_migration(path: Path, click: ClickhouseClient): q = q.strip() if not q: continue - click.execute(q) + try: + click.execute(q) + except Exception as e: + print(f"Error running migration {path}: {e}") + raise def create_db_for_fixture(conn_url): @@ -72,7 +76,7 @@ def override_get_settings(): @pytest.fixture def client_with_bad_settings(): app.dependency_overrides[get_settings] = make_override_get_settings( - clickhouse_url = "clickhouse://badhost:9000" + clickhouse_url="clickhouse://badhost:9000" ) client = TestClient(app) @@ -85,9 +89,9 @@ def client(db): clickhouse_url=db, jwt_encryption_key="super_secure", prometheus_metrics_password="super_secure", - account_id_hashing_key="super_secure" + account_id_hashing_key="super_secure", ) - + client = TestClient(app) yield client diff --git a/ooniapi/services/oonimeasurements/tests/docker-compose.yml b/ooniapi/services/oonimeasurements/tests/docker-compose.yml index 7546ca5b..7453035b 100644 --- a/ooniapi/services/oonimeasurements/tests/docker-compose.yml +++ b/ooniapi/services/oonimeasurements/tests/docker-compose.yml @@ -3,4 +3,4 @@ services: clickhouse: image: "clickhouse/clickhouse-server" ports: - - "9000:9000" + - "9000" diff --git a/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_init_tables.sql b/ooniapi/services/oonimeasurements/tests/migrations/0_clickhouse_init_tables.sql similarity index 100% rename from ooniapi/services/oonimeasurements/tests/migrations/clickhouse_init_tables.sql rename to ooniapi/services/oonimeasurements/tests/migrations/0_clickhouse_init_tables.sql diff --git a/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_citizenlab.sql b/ooniapi/services/oonimeasurements/tests/migrations/1_clickhouse_populate_citizenlab.sql similarity index 100% rename from ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_citizenlab.sql rename to ooniapi/services/oonimeasurements/tests/migrations/1_clickhouse_populate_citizenlab.sql diff --git a/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_fastpath.sql b/ooniapi/services/oonimeasurements/tests/migrations/2_clickhouse_populate_fastpath.sql similarity index 93% rename from ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_fastpath.sql rename to ooniapi/services/oonimeasurements/tests/migrations/2_clickhouse_populate_fastpath.sql index b15b86e7..15cb1c84 100644 --- a/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_fastpath.sql +++ b/ooniapi/services/oonimeasurements/tests/migrations/2_clickhouse_populate_fastpath.sql @@ -998,3 +998,80 @@ INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_na INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000059.576253_IT_dnscheck_4f526339965409f2','20240101T000056Z_dnscheck_IT_3269_n1_GsBrOjKnxGQSIRbo','dot://9.9.9.9:853/','IT','3269','dnscheck','2024-01-01 00:00:59','2024-01-01 00:01:02','','{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','android','f','f','t','9.9.9.9:853','ooniprobe-android-unattended','3.8.5.1','','0','0','0','0','','0','','0.9.2','arm64','ooniprobe-engine','3.19.1','0.19385019','','','','\N'); INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000059.817152_IT_dnscheck_3d7b742209d4f326','20240101T000056Z_dnscheck_IT_3269_n1_GsBrOjKnxGQSIRbo','dot://dns.quad9.net/','IT','3269','dnscheck','2024-01-01 00:00:59','2024-01-01 00:01:02','','{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','android','f','f','t','dns.quad9.net','ooniprobe-android-unattended','3.8.5.1','','0','0','0','0','','0','','0.9.2','arm64','ooniprobe-engine','3.19.1','0.18705674','','','','\N'); INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000100.128817_IT_dnscheck_d544bc263686846c','20240101T000056Z_dnscheck_IT_3269_n1_GsBrOjKnxGQSIRbo','dot://family.cloudflare-dns.com/dns-query','IT','3269','dnscheck','2024-01-01 00:00:59','2024-01-01 00:01:02','','{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','android','f','f','t','family.cloudflare-dns.com','ooniprobe-android-unattended','3.8.5.1','','0','0','0','0','','0','','0.9.2','arm64','ooniprobe-engine','3.19.1','0.09800492','','','','\N'); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709160959.043360_IT_tcpconnect_dd7470fff2493ae4','20210707T121126Z_tcpconnect_IT_12874_n1_mF9eqWrisSsmrXei','109.105.109.146:22','IT',12874,'tcp_connect','2021-07-07 12:11:25','2021-07-09 16:09:19',NULL,'{"blocking_general":0.8,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','t','f','f',NULL,'ooniprobe','2.3.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709163915.625496_IT_tcpconnect_4931fdd615881233','20210709T163837Z_tcpconnect_IT_12874_n1_pMEMABUrXIQKohDL','109.105.109.146:22','IT',12874,'tcp_connect','2021-07-09 16:38:36','2021-07-09 16:38:44',NULL,'{"blocking_general":0.8,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','t','f','f',NULL,'ooniprobe','2.3.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709072445.835274_IT_webconnectivity_6028c5bb650db787','20210709T072008Z_webconnectivity_IT_30722_n1_QTZb4ADoNjZY0FIb','http://twitter.com/youranonnews/','IT',30722,'web_connectivity','2021-07-09 07:20:07','2021-07-09 07:24:44',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe-cli','3.11.0-alpha',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709024737.304746_IT_webconnectivity_255f87202861cf5f','20210709T024624Z_webconnectivity_IT_30722_n1_4TLQzotzToAMSlqM','http://twitter.com/anonops','IT',30722,'web_connectivity','2021-07-09 02:46:24','2021-07-09 02:47:35',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709024813.577239_IT_webconnectivity_eee689109f6f8c56','20210709T024624Z_webconnectivity_IT_30722_n1_4TLQzotzToAMSlqM','https://twitter.com/','IT',30722,'web_connectivity','2021-07-09 02:46:24','2021-07-09 02:48:12',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709025407.095568_IT_webconnectivity_2650f23f3e0b4cc2','20210709T024709Z_webconnectivity_IT_30722_n1_eMVyLOzYQSamieGa','https://twitter.com/','IT',30722,'web_connectivity','2021-07-09 02:47:12','2021-07-09 02:54:09',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','twitter.com','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709025434.142451_IT_webconnectivity_359e48eb0f3cb1e8','20210709T024709Z_webconnectivity_IT_30722_n1_eMVyLOzYQSamieGa','http://twitter.com/anonops','IT',30722,'web_connectivity','2021-07-09 02:47:12','2021-07-09 02:54:35',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','twitter.com','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709030548.991551_IT_webconnectivity_d0a62d6bc8ef9c45','20210709T030036Z_webconnectivity_IT_30722_n1_NtTmGWuXqlLdr6yc','https://twitter.com/','IT',30722,'web_connectivity','2021-07-09 03:00:35','2021-07-09 03:05:45',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','twitter.com','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709093319.986501_IT_webconnectivity_166013bde913bb88','20210709T092741Z_webconnectivity_IT_3269_n1_jad1CZ5j5PmJq4Z4','http://twitter.com/','IT',3269,'web_connectivity','2021-07-09 09:27:41','2021-07-09 09:33:18',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f','twitter.com','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709093340.176625_IT_webconnectivity_494c534d20aa779d','20210709T092741Z_webconnectivity_IT_3269_n1_jad1CZ5j5PmJq4Z4','http://twitter.com/anonops','IT',3269,'web_connectivity','2021-07-09 09:27:41','2021-07-09 09:33:38',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f','twitter.com','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709093729.729025_IT_webconnectivity_0972c5f72e8f4bd8','20210709T093508Z_webconnectivity_IT_30722_n1_IOTXl6m1iVl0oSXV','http://twitter.com/anonops','IT',30722,'web_connectivity','2021-07-09 09:35:07','2021-07-09 09:37:26',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe-cli','3.11.0-alpha',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709093835.281884_IT_webconnectivity_7a4398bdf008ae69','20210709T093508Z_webconnectivity_IT_30722_n1_IOTXl6m1iVl0oSXV','http://twitter.com/','IT',30722,'web_connectivity','2021-07-09 09:35:07','2021-07-09 09:38:32',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe-cli','3.11.0-alpha',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709094305.441543_IT_webconnectivity_227d6ef3db13901b','20210709T093508Z_webconnectivity_IT_30722_n1_IOTXl6m1iVl0oSXV','http://twitter.com/youranonnews/','IT',30722,'web_connectivity','2021-07-09 09:35:07','2021-07-09 09:43:02',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe-cli','3.11.0-alpha',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709100134.594930_IT_webconnectivity_b3935405afca6554','20210709T100037Z_webconnectivity_IT_30722_n1_hOWRjEiBDo4ttStn','http://twitter.com/youranonnews/','IT',30722,'web_connectivity','2021-07-09 10:00:36','2021-07-09 10:01:31',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','twitter.com','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709040419.511318_IT_webconnectivity_aab6dcc4ca6174fd','20210709T040046Z_webconnectivity_IT_30722_n1_QNKIgzw4iwkIJaGI','http://twitter.com/','IT',30722,'web_connectivity','2021-07-09 04:00:44','2021-07-09 04:04:15',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','twitter.com','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709121129.368513_IT_webconnectivity_abe758c0baf63c6d','20210709T120341Z_webconnectivity_IT_30722_n1_jewRFQziMxO56bU1','https://twitter.com/','IT',30722,'web_connectivity','2021-07-09 12:03:41','2021-07-09 12:11:27',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe-cli','3.11.0-alpha',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709123018.507187_IT_webconnectivity_764efe0518a78570','20210709T122812Z_webconnectivity_IT_3269_n1_FoPvRrZl7EopmsSl','https://twitter.com/','IT',3269,'web_connectivity','2021-07-09 12:28:12','2021-07-09 12:30:17',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f','twitter.com','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709145053.403933_IT_webconnectivity_47b0273a43c6920b','20210707T121123Z_webconnectivity_IT_12874_n1_cijtrowwpQmoC7fK','http://twitter.com/','IT',12874,'web_connectivity','2021-07-07 12:11:22','2021-07-09 14:50:58',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe','2.3.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709145053.790266_IT_webconnectivity_dfa25b0b44fc0f0a','20210707T121123Z_webconnectivity_IT_12874_n1_cijtrowwpQmoC7fK','https://twitter.com/','IT',12874,'web_connectivity','2021-07-07 12:11:22','2021-07-09 14:50:59',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe','2.3.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709145413.570612_IT_webconnectivity_1c8c87696ab0a426','20210707T121123Z_webconnectivity_IT_12874_n1_cijtrowwpQmoC7fK','http://twitter.com/','IT',12874,'web_connectivity','2021-07-07 12:11:22','2021-07-09 14:54:18',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe','2.3.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709151714.844897_IT_webconnectivity_3f1c607886d2b82c','20210707T121123Z_webconnectivity_IT_12874_n1_cijtrowwpQmoC7fK','http://twitter.com/anonops','IT',12874,'web_connectivity','2021-07-07 12:11:22','2021-07-09 15:17:11',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe','2.3.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709154646.912915_IT_webconnectivity_3b23a95a1a6966c7','20210707T121123Z_webconnectivity_IT_12874_n1_cijtrowwpQmoC7fK','http://twitter.com/youranonnews/','IT',12874,'web_connectivity','2021-07-07 12:11:22','2021-07-09 15:46:44',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe','2.3.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709170116.008122_IT_webconnectivity_b2ef5d5fdda079ef','20210709T165805Z_webconnectivity_IT_12874_n1_aXjgyRLoq90hBCJi','http://twitter.com/','IT',12874,'web_connectivity','2021-07-09 16:58:04','2021-07-09 17:01:13',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe','2.3.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709170117.569620_IT_webconnectivity_2ba9eb28b09167cb','20210709T165805Z_webconnectivity_IT_12874_n1_aXjgyRLoq90hBCJi','https://twitter.com/','IT',12874,'web_connectivity','2021-07-09 16:58:04','2021-07-09 17:01:15',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe','2.3.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709171238.093704_IT_webconnectivity_234e81ac6e934489','20210709T165805Z_webconnectivity_IT_12874_n1_aXjgyRLoq90hBCJi','http://twitter.com/youranonnews/','IT',12874,'web_connectivity','2021-07-09 16:58:04','2021-07-09 17:12:34',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe','2.3.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709191422.444165_IT_webconnectivity_21a07ef7982fec55','20210709T191123Z_webconnectivity_IT_30722_n1_pWGKqTncYlQ8w4Ea','http://twitter.com/anonops','IT',30722,'web_connectivity','2021-07-09 19:11:23','2021-07-09 19:14:20',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','macos','f','f','f','twitter.com','ooniprobe-cli','3.11.0-alpha',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709084705.960331_CH_ndt_0ba6ece582052d2b','20210709T084643Z_ndt_CH_3303_n1_WXQrVhwJVoPY0gKB',NULL,'CH',3303,'ndt','2021-07-09 08:46:42','2021-07-09 08:46:43',NULL,'{}','android','f','f','f',NULL,'measurement_kit','0.7.1',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000024.440526_CH_httpinvalidrequestline_3937f817503ed4ea','20210709T000017Z_httpinvalidrequestline_CH_3303_n1_8mr2M3dzkoFmmjIU',NULL,'CH',3303,'http_invalid_request_line','2021-07-09 00:00:16','2021-07-09 00:00:18',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','linux','f','f','f',NULL,'ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000047.319640_CH_whatsapp_37551d6d6200b6ac','20210709T000019Z_whatsapp_CH_3303_n1_K6HBNpu4GOQ9dTk4',NULL,'CH',3303,'whatsapp','2021-07-09 00:00:18','2021-07-09 00:00:23',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"registration_server_accessible":true,"whatsapp_web_accessible":true,"whatsapp_endpoints_accessible":true}}','linux','f','f','f',NULL,'ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000036.774712_CH_vanillator_8eaf5561dfc5505c','20210709T000020Z_vanillator_CH_3303_n1_PKzS86X4ykdfiIoR',NULL,'CH',3303,'vanilla_tor','2021-07-09 00:00:19','2021-07-09 00:00:23',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','linux','f','f','f',NULL,'ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000025.671670_CH_httpheaderfieldmanipulation_da104261e2040b05','20210709T000023Z_httpheaderfieldmanipulation_CH_3303_n1_QcTq3o5pmiCDcXFo',NULL,'CH',3303,'http_header_field_manipulation','2021-07-09 00:00:22','2021-07-09 00:00:23',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','linux','f','f','f',NULL,'ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000033.189293_CH_webconnectivity_3facc5cbce3a3a87','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://4genderjustice.org/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:00:29',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','linux','f','f','f','4genderjustice.org','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000035.000059_CH_webconnectivity_c4345abb4658f8ce','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','https://2600.org/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:00:29',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','linux','f','f','f','2600.org','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000035.674507_CH_webconnectivity_bc8ce196be0a8619','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://666games.net/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:00:32',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','linux','f','f','f','666games.net','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000044.052990_CH_webconnectivity_167f1b368ceba3e8','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://8thstreetlatinas.com/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:00:34',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','linux','f','f','f','8thstreetlatinas.com','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000036.476702_CH_webconnectivity_4938b976a3335c3f','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://a1408.g.akamai.net/5/1408/1388/2005110406/1a1a1ad948be278cff2d96046ad90768d848b41947aa1986/sample_sorenson.mov.zip','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:00:34',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','linux','f','f','f','a1408.g.akamai.net','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000326.557224_CH_webconnectivity_ca7af3d04d61b622','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://guardster.com/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:03:19',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"dns"}}','linux','t','f','f','guardster.com','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000646.770289_CH_webconnectivity_80c62cb823d861c4','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://translation.langenberg.com/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:06:38',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-diff"}}','linux','t','f','f','translation.langenberg.com','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000657.118672_CH_webconnectivity_c2144b66d603565d','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://twilight.ws/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:06:53',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-diff"}}','linux','t','f','f','twilight.ws','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000742.944087_CH_webconnectivity_b29cb95e55c2b2d5','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://wwf.panda.org/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:07:37',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-diff"}}','linux','t','f','f','wwf.panda.org','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000912.503643_CH_webconnectivity_67578b79c692a9f5','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://www.arabnews.com/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:09:10',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-diff"}}','linux','t','f','f','www.arabnews.com','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709173519.683803_CH_webconnectivity_e6ac18b22a8dfa39','20210709T173404Z_webconnectivity_CH_3303_n1_6M8PnQRo0gb8wsf6','http://anonym.to/','CH',3303,'web_connectivity','2021-07-09 17:34:03','2021-07-09 17:35:11',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"tcp_ip"}}','ios','t','f','f','anonym.to','ooniprobe-ios','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709001153.862448_CH_webconnectivity_17a760bc4c28f32b','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://www.betfair.com/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:11:52',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-diff"}}','linux','t','f','f','www.betfair.com','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709001222.808240_CH_webconnectivity_19acbedeeabaeeaf','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://www.cannabis.info/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:12:20',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"dns"}}','linux','t','f','f','www.cannabis.info','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709001255.358462_CH_webconnectivity_a2db3abb50cd3efb','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://www.crackz.ws/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:12:53',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"dns"}}','linux','t','f','f','www.crackz.ws','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709001401.918989_CH_webconnectivity_3403f3bc35290e28','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://www.efindlove.com/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:13:59',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-diff"}}','linux','t','f','f','www.efindlove.com','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709173657.594507_CH_riseupvpn_bd25bbcff2d7777a','20210709T173652Z_riseupvpn_CH_3303_n1_lG6bVYnaRkS6cfMB',NULL,'CH',3303,'riseupvpn','2021-07-09 17:36:52','2021-07-09 17:36:52',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','ios','f','f','t',NULL,'ooniprobe-ios','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709173835.344734_CH_stunreachability_0eed53cddab59f2b','20210709T173835Z_stunreachability_CH_3303_n1_DhkURTk4MYExoNxV',NULL,'CH',3303,'stunreachability','2021-07-09 17:38:35','2021-07-09 17:38:35',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','ios','f','f','t',NULL,'ooniprobe-ios','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000204.929491_CH_webconnectivity_ad5cb65dd2c89f8e','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','http://company.wizards.com/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:01:11',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','linux','f','f','t','company.wizards.com','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709000327.335908_CH_webconnectivity_67306307a75353ae','20210709T000028Z_webconnectivity_CH_3303_n1_cNxbZSc8uRph4Wjl','https://extratorrent.cc/','CH',3303,'web_connectivity','2021-07-09 00:00:28','2021-07-09 00:02:41',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','linux','f','f','t','extratorrent.cc','ooniprobe','2.1.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053726.554910_BR_ndt_dc2632942eb7a16e','20210709T053700Z_ndt_BR_263262_n1_BkTx5opPjUp6tUPH',NULL,'BR',263262,'ndt','2021-07-09 05:37:27','2021-07-09 05:37:32',NULL,'{}','android','f','f','f',NULL,'measurement_kit','0.7.1',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053736.558656_BR_webconnectivity_81fd3fdb0d4000a5','20210709T053648Z_webconnectivity_BR_262616_n1_oY3VoewOgoYrsBD1','https://www.xroxy.com/','BR',262616,'web_connectivity','2021-07-09 05:36:48','2021-07-09 05:37:33',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f','www.xroxy.com','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053736.132589_GB_webconnectivity_ad087e488f3f5da0','20210709T052050Z_webconnectivity_GB_5089_n1_RNIURyorMhSyAORJ','http://www.worldwildlife.org/','GB',5089,'web_connectivity','2021-07-09 05:20:50','2021-07-09 05:37:34',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','www.worldwildlife.org','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053735.289304_BR_httpheaderfieldmanipulation_a33902aa41b11b2f','20210709T053734Z_httpheaderfieldmanipulation_BR_28573_n1_Ly6OlIsCmXfwfbKB',NULL,'BR',28573,'http_header_field_manipulation','2021-07-09 05:37:33','2021-07-09 05:37:34',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f',NULL,'ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053740.634363_BR_webconnectivity_ba8211bd386e6891','20210709T053611Z_webconnectivity_BR_262970_n1_fZ7ICIqZPwFps9ly','http://www.osce.org/','BR',262970,'web_connectivity','2021-07-09 05:36:12','2021-07-09 05:37:35',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f','www.osce.org','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053737.595022_GB_webconnectivity_990787a9451fd20f','20210709T052050Z_webconnectivity_GB_5089_n1_RNIURyorMhSyAORJ','http://peta.xxx/','GB',5089,'web_connectivity','2021-07-09 05:20:50','2021-07-09 05:37:36',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','peta.xxx','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053746.382922_BR_webconnectivity_ea1b235be7e4c9f8','20210709T053648Z_webconnectivity_BR_262616_n1_oY3VoewOgoYrsBD1','https://makeamazonpay.com/','BR',262616,'web_connectivity','2021-07-09 05:36:48','2021-07-09 05:37:36',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f','makeamazonpay.com','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053739.185348_GB_webconnectivity_10501df9a77a2d17','20210709T052050Z_webconnectivity_GB_5089_n1_RNIURyorMhSyAORJ','https://www.gambling.com/','GB',5089,'web_connectivity','2021-07-09 05:20:50','2021-07-09 05:37:38',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','www.gambling.com','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053742.100837_GB_webconnectivity_6528e91da1d7cca2','20210709T052050Z_webconnectivity_GB_5089_n1_RNIURyorMhSyAORJ','http://www.christianity.com/','GB',5089,'web_connectivity','2021-07-09 05:20:50','2021-07-09 05:37:39',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','www.christianity.com','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053740.530741_BR_facebookmessenger_e3710e08d70b56b0','20210709T053740Z_facebookmessenger_BR_28573_n1_POMaYez61qipoUqg',NULL,'BR',28573,'facebook_messenger','2021-07-09 05:37:39','2021-07-09 05:37:39',NULL,'{"blocking_general":0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f',NULL,'ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053744.041145_BR_telegram_465774ca77051570','20210709T053740Z_telegram_BR_28573_n1_5b4wcqd7b3TRZjpp',NULL,'BR',28573,'telegram','2021-07-09 05:37:40','2021-07-09 05:37:40',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"web_failure":null,"accessible_endpoints":15,"unreachable_endpoints":0,"http_success_cnt":12,"http_failure_cnt":0}','windows','f','f','f',NULL,'ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053743.848535_BR_webconnectivity_9b2c46ca801efc3c','20210709T053611Z_webconnectivity_BR_262970_n1_fZ7ICIqZPwFps9ly','http://www.ciscozappa.blogspot.com/','BR',262970,'web_connectivity','2021-07-09 05:36:12','2021-07-09 05:37:41',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f','www.ciscozappa.blogspot.com','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053746.309095_GB_webconnectivity_4d9f56d1f93d2c29','20210709T052050Z_webconnectivity_GB_5089_n1_RNIURyorMhSyAORJ','https://www.om.org/','GB',5089,'web_connectivity','2021-07-09 05:20:50','2021-07-09 05:37:42',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','www.om.org','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053801.319686_BR_webconnectivity_c0e446f7b143a1ab','20210709T053611Z_webconnectivity_BR_262970_n1_fZ7ICIqZPwFps9ly','http://www.folhadacidade.com.br/','BR',262970,'web_connectivity','2021-07-09 05:36:12','2021-07-09 05:37:44',NULL,'{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-failure"}}','windows','t','f','f','www.folhadacidade.com.br','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053746.752265_BR_whatsapp_449c990b9f66afef','20210709T053744Z_whatsapp_BR_28573_n1_uIrrog7pFURcrLVs',NULL,'BR',28573,'whatsapp','2021-07-09 05:37:43','2021-07-09 05:37:44',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"registration_server_accessible":true,"whatsapp_web_accessible":true,"whatsapp_endpoints_accessible":true}}','windows','f','f','f',NULL,'ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053750.929403_GB_webconnectivity_b4692c4b02a9bfe0','20210709T052050Z_webconnectivity_GB_5089_n1_RNIURyorMhSyAORJ','http://www.888casino.com/','GB',5089,'web_connectivity','2021-07-09 05:20:50','2021-07-09 05:37:46',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','www.888casino.com','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053748.732532_BR_webconnectivity_39436c752ddf79e4','20210709T053648Z_webconnectivity_BR_262616_n1_oY3VoewOgoYrsBD1','http://www.mainichi.co.jp/','BR',262616,'web_connectivity','2021-07-09 05:36:48','2021-07-09 05:37:46',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f','www.mainichi.co.jp','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053749.384243_BR_signal_5219f41971dc4e28','20210709T053747Z_signal_BR_28573_n1_j9FWVowc7de5ptGL',NULL,'BR',28573,'signal','2021-07-09 05:37:46','2021-07-09 05:37:46',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f',NULL,'ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053751.673558_BR_webconnectivity_6005c3537a1ef8c6','20210709T053648Z_webconnectivity_BR_262616_n1_oY3VoewOgoYrsBD1','http://www.acrehoje.com/','BR',262616,'web_connectivity','2021-07-09 05:36:48','2021-07-09 05:37:48',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','windows','f','f','f','www.acrehoje.com','ooniprobe-desktop-unattended','3.10.0-beta.3',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210709053753.175149_GB_webconnectivity_27f0d8b3437f4203','20210709T052050Z_webconnectivity_GB_5089_n1_RNIURyorMhSyAORJ','https://www.epa.gov/','GB',5089,'web_connectivity','2021-07-09 05:20:50','2021-07-09 05:37:51',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','www.epa.gov','ooniprobe-android','3.0.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210708064100.264013_SS_whatsapp_41a9f6326893ee1f','20210708T064054Z_whatsapp_SS_327786_n1_ghec1Uo93VRh7ee5',NULL,'SS',327786,'whatsapp','2021-07-08 06:41:02','2021-07-08 06:41:03',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"registration_server_accessible":true,"whatsapp_web_accessible":true,"whatsapp_endpoints_accessible":true}}','windows','f','f','f',NULL,'ooniprobe-desktop-unattended','3.9.2',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210708064202.980944_SS_whatsapp_fa716d08566fd81d','20210708T064158Z_whatsapp_SS_327786_n1_bp4eor3EIgSaSPk8',NULL,'SS',327786,'whatsapp','2021-07-08 06:42:05','2021-07-08 06:42:06',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"registration_server_accessible":true,"whatsapp_web_accessible":true,"whatsapp_endpoints_accessible":true}}','windows','f','f','f',NULL,'ooniprobe-desktop-unattended','3.9.2',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210708064255.317809_HR_whatsapp_1fa0ce560684cae6','20210708T064252Z_whatsapp_HR_34594_n1_WKjsbpBAUE38uiTj',NULL,'HR',34594,'whatsapp','2021-07-08 06:42:51','2021-07-08 06:42:52',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"registration_server_accessible":true,"whatsapp_web_accessible":true,"whatsapp_endpoints_accessible":true}}','android','f','f','f',NULL,'measurement_kit','0.8.1',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210708064358.006383_ES_whatsapp_c80e733323201168','20210708T064356Z_whatsapp_ES_3352_n1_ITBxRQ8f4B9nWtzS',NULL,'ES',3352,'whatsapp','2021-07-08 06:43:55','2021-07-08 06:43:55',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"registration_server_accessible":true,"whatsapp_web_accessible":true,"whatsapp_endpoints_accessible":true}}','windows','f','f','f',NULL,'ooniprobe-desktop-unattended','3.9.2',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210708064435.632040_MM_whatsapp_b3fe1a6061d720e4','20210708T064431Z_whatsapp_MM_133384_n1_yhkQtlaBRuoDnD9B',NULL,'MM',133384,'whatsapp','2021-07-08 06:44:31','2021-07-08 06:44:31',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"registration_server_accessible":true,"whatsapp_web_accessible":true,"whatsapp_endpoints_accessible":true}}','windows','f','f','f',NULL,'ooniprobe-desktop-unattended','3.8.0',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210708064412.083471_AM_whatsapp_8db72fe2cc57adee','20210708T064408Z_whatsapp_AM_49800_n1_MvhXsYKzDSooGPCy',NULL,'AM',49800,'whatsapp','2021-07-08 06:45:05','2021-07-08 06:45:05',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"registration_server_accessible":true,"whatsapp_web_accessible":true,"whatsapp_endpoints_accessible":true}}','unknown','f','f','f',NULL,'ooniprobe-cli','3.9.2',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); +INSERT INTO fastpath(measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20210708064537.694608_HR_whatsapp_cbee59e3f8d64e84','20210708T064536Z_whatsapp_HR_31012_n1_6bKkL89MKhYySdfK',NULL,'HR',31012,'whatsapp','2021-07-08 06:45:35','2021-07-08 06:45:35',NULL,'{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"registration_server_accessible":true,"whatsapp_web_accessible":true,"whatsapp_endpoints_accessible":true}}','android','f','f','f',NULL,'measurement_kit','0.8.1',NULL,0,0,0,0,NULL,0,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); diff --git a/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_jsonl.sql b/ooniapi/services/oonimeasurements/tests/migrations/3_clickhouse_populate_jsonl.sql similarity index 100% rename from ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_jsonl.sql rename to ooniapi/services/oonimeasurements/tests/migrations/3_clickhouse_populate_jsonl.sql diff --git a/ooniapi/services/oonimeasurements/tests/test_measurements.py b/ooniapi/services/oonimeasurements/tests/test_measurements.py index ab4fc518..558c54d5 100644 --- a/ooniapi/services/oonimeasurements/tests/test_measurements.py +++ b/ooniapi/services/oonimeasurements/tests/test_measurements.py @@ -1,4 +1,3 @@ - import pytest from textwrap import dedent @@ -22,7 +21,7 @@ def api(client, subpath, **kw): url += "?" + urlencode(kw) response = client.get(url) - assert response.status_code == 200, response.data + assert response.status_code == 200, response.json() assert is_json(response) return response.json() @@ -40,7 +39,7 @@ def test_aggregation_no_axis_with_caching(client): # 0-dimensional data url = "aggregation?probe_cc=IT&probe_asn=AS3269&since=2024-01-01&until=2024-02-01" resp = client.get(f"/api/v1/{url}") - assert resp.status_code == 200, resp + assert resp.status_code == 200, resp.json() j = resp.json() assert j["dimension_count"] == 0 assert j["v"] == 0 @@ -149,9 +148,9 @@ def test_aggregation_no_axis_filter_multi_domain(client): "result": { "anomaly_count": 0, "confirmed_count": 0, - "failure_count": 1, - "measurement_count": 16, - "ok_count": 15, + "failure_count": 0, + "measurement_count": 24, + "ok_count": 24, }, "v": 0, }, fjd(r) @@ -165,11 +164,11 @@ def test_aggregation_no_axis_filter_multi_probe_asn(client): assert r == { "dimension_count": 0, "result": { - "anomaly_count": 187, + "anomaly_count": 10, "confirmed_count": 0, - "failure_count": 5, - "measurement_count": 1689, - "ok_count": 1497, + "failure_count": 4, + "measurement_count": 24, + "ok_count": 10, }, "v": 0, }, fjd(r) @@ -183,11 +182,11 @@ def test_aggregation_no_axis_filter_multi_probe_cc(client): assert r == { "dimension_count": 0, "result": { - "anomaly_count": 123, + "anomaly_count": 1, "confirmed_count": 0, - "failure_count": 113, - "measurement_count": 2435, - "ok_count": 2199, + "failure_count": 0, + "measurement_count": 20, + "ok_count": 19, }, "v": 0, }, fjd(r) @@ -201,11 +200,11 @@ def test_aggregation_no_axis_filter_multi_test_name(client): assert r == { "dimension_count": 0, "result": { - "anomaly_count": 319, - "confirmed_count": 42, - "failure_count": 340, - "measurement_count": 8547, - "ok_count": 7846, + "anomaly_count": 11, + "confirmed_count": 0, + "failure_count": 2, + "measurement_count": 57, + "ok_count": 44, }, "v": 0, }, fjd(r) @@ -220,19 +219,19 @@ def test_aggregation_no_axis_filter_multi_test_name_1_axis(client): "dimension_count": 1, "result": [ { - "anomaly_count": 317, - "confirmed_count": 42, - "failure_count": 339, - "measurement_count": 8488, - "ok_count": 7790, + "anomaly_count": 11, + "confirmed_count": 0, + "failure_count": 2, + "measurement_count": 55, + "ok_count": 42, "test_name": "web_connectivity", }, { - "anomaly_count": 2, + "anomaly_count": 0, "confirmed_count": 0, - "failure_count": 1, - "measurement_count": 59, - "ok_count": 56, + "failure_count": 0, + "measurement_count": 2, + "ok_count": 2, "test_name": "whatsapp", }, ], @@ -267,12 +266,12 @@ def test_aggregation_x_axis_only(client): "dimension_count": 1, "result": [ { - "anomaly_count": 187, + "anomaly_count": 10, "confirmed_count": 0, - "failure_count": 5, - "measurement_count": 1689, + "failure_count": 4, + "measurement_count": 24, "measurement_start_day": "2021-07-09", - "ok_count": 1497, + "ok_count": 10, }, ], "v": 0, @@ -314,19 +313,123 @@ def test_aggregation_x_axis_only_hour(client): "dimension_count": 1, "result": [ { - "anomaly_count": 686, - "confirmed_count": 42, - "failure_count": 777, - "measurement_count": 9990, + "anomaly_count": 9, + "confirmed_count": 0, + "failure_count": 2, + "measurement_count": 20, "measurement_start_day": "2021-07-09T00:00:00Z", - "ok_count": 8485, + "ok_count": 9, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 4, + "measurement_start_day": "2021-07-09T02:00:00Z", + "ok_count": 4, }, { "anomaly_count": 0, "confirmed_count": 0, "failure_count": 0, "measurement_count": 1, - "measurement_start_day": "2021-07-09T01:00:00Z", + "measurement_start_day": "2021-07-09T03:00:00Z", + "ok_count": 1, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 1, + "measurement_start_day": "2021-07-09T04:00:00Z", + "ok_count": 1, + }, + { + "anomaly_count": 1, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 20, + "measurement_start_day": "2021-07-09T05:00:00Z", + "ok_count": 19, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 1, + "measurement_start_day": "2021-07-09T07:00:00Z", + "ok_count": 1, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 1, + "measurement_start_day": "2021-07-09T08:00:00Z", + "ok_count": 1, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 5, + "measurement_start_day": "2021-07-09T09:00:00Z", + "ok_count": 5, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 1, + "measurement_start_day": "2021-07-09T10:00:00Z", + "ok_count": 1, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 2, + "measurement_start_day": "2021-07-09T12:00:00Z", + "ok_count": 2, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 3, + "measurement_start_day": "2021-07-09T14:00:00Z", + "ok_count": 3, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 2, + "measurement_start_day": "2021-07-09T15:00:00Z", + "ok_count": 2, + }, + { + "anomaly_count": 2, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 2, + "measurement_start_day": "2021-07-09T16:00:00Z", + "ok_count": 0, + }, + { + "anomaly_count": 1, + "confirmed_count": 0, + "failure_count": 2, + "measurement_count": 6, + "measurement_start_day": "2021-07-09T17:00:00Z", + "ok_count": 3, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 1, + "measurement_start_day": "2021-07-09T19:00:00Z", "ok_count": 1, }, ], @@ -342,14 +445,14 @@ def test_aggregation_x_axis_domain(client): r.pop("db_stats", None) assert r["dimension_count"] == 1 for x in r["result"]: - if x["domain"] == "www.theregister.co.uk": + if x["domain"] == "anonym.to": assert x == { - "anomaly_count": 0, + "anomaly_count": 1, "confirmed_count": 0, - "domain": "www.theregister.co.uk", + "domain": "anonym.to", "failure_count": 0, "measurement_count": 1, - "ok_count": 1, + "ok_count": 0, } return @@ -384,34 +487,25 @@ def test_aggregation_x_axis_only_probe_cc(client): url = "aggregation?since=2021-07-09&until=2021-07-10&axis_x=probe_cc" r = api(client, url) assert r["dimension_count"] == 1 - assert len(r["result"]) == 33 + assert len(r["result"]) == 4 def test_aggregation_x_axis_only_category_code(client): # 1-dimensional data - url = "aggregation?probe_cc=IE&category_code=HACK&since=2021-07-09&until=2021-07-10&axis_x=measurement_start_day" + url = "aggregation?probe_cc=CH&category_code=GAME&since=2021-07-09&until=2021-07-10&axis_x=measurement_start_day" r = api(client, url) - expected = { - "dimension_count": 1, - "result": [ - { - "anomaly_count": 32, - "confirmed_count": 0, - "failure_count": 0, - "measurement_count": 1302, - "measurement_start_day": "2021-07-10", - }, - { - "anomaly_count": 13, - "confirmed_count": 0, - "failure_count": 0, - "measurement_count": 1236, - "measurement_start_day": "2021-07-10", - }, - ], - "v": 0, - } - assert r == expected, fjd(r) + assert r["result"] == [ + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 1, + "ok_count": 1, + "measurement_start_day": "2021-07-09T00:00:00Z", + }, + ] + assert r["dimension_count"] == 1 + assert r["v"] == 0 def test_aggregation_x_axis_only_csv(client):