feat(minipipeline): implement linear analysis (#1417)

The linear analysis takes advantage of recent changes in WebConnectivity LTE to build a linear chain of observations where the entries at the beginning of the chain are from the latest redirect followed by the penultimate with the chain ending with observations from the initial request. Within each redirect, entries are sorted by type such that HTTP appears before TLS which appears before TCP which appears before DNS. Within each protocol, entries without errors sort before entries with errors. Finally, if two entries have the same redirect depth, protocol type and error, we sort using the transaction ID (which is enough to avoid making tests nondeterministic). With this functionality, we should be able to write relatively easily an analysis algorithm for Web Connectivity LTE. Part of ooni/probe#2634
ooni · Nov 30, 2023 · c056f79 · c056f79
1 parent 4dc6e7d
commit c056f79
Show file tree

Hide file tree

Showing 139 changed files with 30,350 additions and 986 deletions.
diff --git a/internal/cmd/minipipeline/testdata/analysis.json b/internal/cmd/minipipeline/testdata/analysis.json
@@ -38,5 +38,384 @@
   "HTTPFinalResponseDiffUncommonHeadersIntersection": {
     "x-drupal-cache": true,
     "x-generator": true
-  }
+  },
+  "Linear": [
+    {
+      "TagDepth": 0,
+      "Type": 3,
+      "Failure": "",
+      "TransactionID": 4,
+      "TagFetchBody": true,
+      "DNSTransactionID": 1,
+      "DNSDomain": "nexa.polito.it",
+      "DNSLookupFailure": "",
+      "DNSQueryType": null,
+      "DNSEngine": null,
+      "DNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
+      "IPAddress": "130.192.16.171",
+      "IPAddressASN": 137,
+      "IPAddressBogon": false,
+      "EndpointTransactionID": 4,
+      "EndpointProto": "tcp",
+      "EndpointPort": "443",
+      "EndpointAddress": "130.192.16.171:443",
+      "TCPConnectFailure": "",
+      "TLSHandshakeFailure": "",
+      "TLSServerName": "nexa.polito.it",
+      "HTTPRequestURL": "https://nexa.polito.it/",
+      "HTTPFailure": "",
+      "HTTPResponseStatusCode": 200,
+      "HTTPResponseBodyLength": 36546,
+      "HTTPResponseBodyIsTruncated": false,
+      "HTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "HTTPResponseLocation": null,
+      "HTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino",
+      "HTTPResponseIsFinal": true,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
+      "ControlTCPConnectFailure": "",
+      "ControlTLSHandshakeFailure": "",
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
+    },
+    {
+      "TagDepth": 0,
+      "Type": 0,
+      "Failure": "",
+      "TransactionID": 3,
+      "TagFetchBody": null,
+      "DNSTransactionID": 3,
+      "DNSDomain": "nexa.polito.it",
+      "DNSLookupFailure": "",
+      "DNSQueryType": "A",
+      "DNSEngine": "doh",
+      "DNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
+      "IPAddress": "130.192.16.171",
+      "IPAddressASN": 137,
+      "IPAddressBogon": false,
+      "EndpointTransactionID": null,
+      "EndpointProto": null,
+      "EndpointPort": null,
+      "EndpointAddress": null,
+      "TCPConnectFailure": null,
+      "TLSHandshakeFailure": null,
+      "TLSServerName": null,
+      "HTTPRequestURL": null,
+      "HTTPFailure": null,
+      "HTTPResponseStatusCode": null,
+      "HTTPResponseBodyLength": null,
+      "HTTPResponseBodyIsTruncated": null,
+      "HTTPResponseHeadersKeys": null,
+      "HTTPResponseLocation": null,
+      "HTTPResponseTitle": null,
+      "HTTPResponseIsFinal": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
+      "ControlTCPConnectFailure": null,
+      "ControlTLSHandshakeFailure": null,
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
+    },
+    {
+      "TagDepth": 0,
+      "Type": 0,
+      "Failure": "",
+      "TransactionID": 2,
+      "TagFetchBody": null,
+      "DNSTransactionID": 2,
+      "DNSDomain": "nexa.polito.it",
+      "DNSLookupFailure": "",
+      "DNSQueryType": "ANY",
+      "DNSEngine": "getaddrinfo",
+      "DNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
+      "IPAddress": "130.192.16.171",
+      "IPAddressASN": 137,
+      "IPAddressBogon": false,
+      "EndpointTransactionID": null,
+      "EndpointProto": null,
+      "EndpointPort": null,
+      "EndpointAddress": null,
+      "TCPConnectFailure": null,
+      "TLSHandshakeFailure": null,
+      "TLSServerName": null,
+      "HTTPRequestURL": null,
+      "HTTPFailure": null,
+      "HTTPResponseStatusCode": null,
+      "HTTPResponseBodyLength": null,
+      "HTTPResponseBodyIsTruncated": null,
+      "HTTPResponseHeadersKeys": null,
+      "HTTPResponseLocation": null,
+      "HTTPResponseTitle": null,
+      "HTTPResponseIsFinal": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
+      "ControlTCPConnectFailure": null,
+      "ControlTLSHandshakeFailure": null,
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
+    },
+    {
+      "TagDepth": 0,
+      "Type": 0,
+      "Failure": "",
+      "TransactionID": 1,
+      "TagFetchBody": null,
+      "DNSTransactionID": 1,
+      "DNSDomain": "nexa.polito.it",
+      "DNSLookupFailure": "",
+      "DNSQueryType": "A",
+      "DNSEngine": "udp",
+      "DNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
+      "IPAddress": "130.192.16.171",
+      "IPAddressASN": 137,
+      "IPAddressBogon": false,
+      "EndpointTransactionID": null,
+      "EndpointProto": null,
+      "EndpointPort": null,
+      "EndpointAddress": null,
+      "TCPConnectFailure": null,
+      "TLSHandshakeFailure": null,
+      "TLSServerName": null,
+      "HTTPRequestURL": null,
+      "HTTPFailure": null,
+      "HTTPResponseStatusCode": null,
+      "HTTPResponseBodyLength": null,
+      "HTTPResponseBodyIsTruncated": null,
+      "HTTPResponseHeadersKeys": null,
+      "HTTPResponseLocation": null,
+      "HTTPResponseTitle": null,
+      "HTTPResponseIsFinal": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
+      "ControlTCPConnectFailure": null,
+      "ControlTLSHandshakeFailure": null,
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
+    },
+    {
+      "TagDepth": 0,
+      "Type": 0,
+      "Failure": "dns_no_answer",
+      "TransactionID": 3,
+      "TagFetchBody": null,
+      "DNSTransactionID": 3,
+      "DNSDomain": "nexa.polito.it",
+      "DNSLookupFailure": "dns_no_answer",
+      "DNSQueryType": "AAAA",
+      "DNSEngine": "doh",
+      "DNSResolvedAddrs": null,
+      "IPAddress": null,
+      "IPAddressASN": null,
+      "IPAddressBogon": null,
+      "EndpointTransactionID": null,
+      "EndpointProto": null,
+      "EndpointPort": null,
+      "EndpointAddress": null,
+      "TCPConnectFailure": null,
+      "TLSHandshakeFailure": null,
+      "TLSServerName": null,
+      "HTTPRequestURL": null,
+      "HTTPFailure": null,
+      "HTTPResponseStatusCode": null,
+      "HTTPResponseBodyLength": null,
+      "HTTPResponseBodyIsTruncated": null,
+      "HTTPResponseHeadersKeys": null,
+      "HTTPResponseLocation": null,
+      "HTTPResponseTitle": null,
+      "HTTPResponseIsFinal": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
+      "ControlTCPConnectFailure": null,
+      "ControlTLSHandshakeFailure": null,
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
+    },
+    {
+      "TagDepth": 0,
+      "Type": 0,
+      "Failure": "dns_no_answer",
+      "TransactionID": 1,
+      "TagFetchBody": null,
+      "DNSTransactionID": 1,
+      "DNSDomain": "nexa.polito.it",
+      "DNSLookupFailure": "dns_no_answer",
+      "DNSQueryType": "AAAA",
+      "DNSEngine": "udp",
+      "DNSResolvedAddrs": null,
+      "IPAddress": null,
+      "IPAddressASN": null,
+      "IPAddressBogon": null,
+      "EndpointTransactionID": null,
+      "EndpointProto": null,
+      "EndpointPort": null,
+      "EndpointAddress": null,
+      "TCPConnectFailure": null,
+      "TLSHandshakeFailure": null,
+      "TLSServerName": null,
+      "HTTPRequestURL": null,
+      "HTTPFailure": null,
+      "HTTPResponseStatusCode": null,
+      "HTTPResponseBodyLength": null,
+      "HTTPResponseBodyIsTruncated": null,
+      "HTTPResponseHeadersKeys": null,
+      "HTTPResponseLocation": null,
+      "HTTPResponseTitle": null,
+      "HTTPResponseIsFinal": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
+      "ControlTCPConnectFailure": null,
+      "ControlTLSHandshakeFailure": null,
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
+    }
+  ]
 }