x

ooni · Nov 29, 2023 · 8bbb0fa · 8bbb0fa
1 parent b86a6c5
commit 8bbb0fa
Show file tree

Hide file tree

Showing 69 changed files with 5,784 additions and 518 deletions.
diff --git a/internal/cmd/minipipeline/testdata/observations.json b/internal/cmd/minipipeline/testdata/observations.json
@@ -6,6 +6,7 @@
       "DNSLookupFailure": "dns_no_answer",
       "DNSQueryType": "AAAA",
       "DNSEngine": "udp",
+      "DNSResolvedAddrs": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,
@@ -29,6 +30,9 @@
       "TagFetchBody": null,
       "ControlDNSDomain": "nexa.polito.it",
       "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "ControlTCPConnectFailure": null,
       "MatchWithControlIPAddress": null,
       "MatchWithControlIPAddressASN": null,
@@ -45,6 +49,7 @@
       "DNSLookupFailure": "dns_no_answer",
       "DNSQueryType": "AAAA",
       "DNSEngine": "doh",
+      "DNSResolvedAddrs": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,
@@ -68,6 +73,9 @@
       "TagFetchBody": null,
       "ControlDNSDomain": "nexa.polito.it",
       "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "ControlTCPConnectFailure": null,
       "MatchWithControlIPAddress": null,
       "MatchWithControlIPAddressASN": null,
@@ -86,6 +94,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": "A",
       "DNSEngine": "udp",
+      "DNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -107,8 +118,11 @@
       "HTTPResponseIsFinal": null,
       "TagDepth": null,
       "TagFetchBody": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "ControlTCPConnectFailure": null,
       "MatchWithControlIPAddress": null,
       "MatchWithControlIPAddressASN": null,
@@ -125,6 +139,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": "ANY",
       "DNSEngine": "getaddrinfo",
+      "DNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -146,8 +163,11 @@
       "HTTPResponseIsFinal": null,
       "TagDepth": null,
       "TagFetchBody": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "ControlTCPConnectFailure": null,
       "MatchWithControlIPAddress": null,
       "MatchWithControlIPAddressASN": null,
@@ -164,6 +184,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": "A",
       "DNSEngine": "doh",
+      "DNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -185,8 +208,11 @@
       "HTTPResponseIsFinal": null,
       "TagDepth": null,
       "TagFetchBody": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "ControlTCPConnectFailure": null,
       "MatchWithControlIPAddress": null,
       "MatchWithControlIPAddressASN": null,
@@ -205,6 +231,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": null,
       "DNSEngine": null,
+      "DNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -243,6 +272,9 @@
       "TagFetchBody": null,
       "ControlDNSDomain": "nexa.polito.it",
       "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/internal/cmd/minipipeline/testdata/observations_classic.json b/internal/cmd/minipipeline/testdata/observations_classic.json
@@ -7,6 +7,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": "ANY",
       "DNSEngine": "getaddrinfo",
+      "DNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -28,8 +31,11 @@
       "HTTPResponseIsFinal": null,
       "TagDepth": null,
       "TagFetchBody": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "ControlTCPConnectFailure": null,
       "MatchWithControlIPAddress": null,
       "MatchWithControlIPAddressASN": null,
@@ -48,6 +54,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": null,
       "DNSEngine": null,
+      "DNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
       "IPAddressBogon": false,
@@ -86,6 +95,9 @@
       "TagFetchBody": null,
       "ControlDNSDomain": "nexa.polito.it",
       "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "130.192.16.171"
+      ],
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/internal/minipipeline/observation.go b/internal/minipipeline/observation.go
@@ -485,6 +485,9 @@ func (c *WebObservationsContainer) controlMatchDNSLookupResults(inputDomain stri
 		// handle the case in which the IP address has been provided by the control, which
 		// is a case where the domain is empty and the IP address is in thAddrMap
 		if domain == "" && thAddrMap[addr] {
+			obs.ControlDNSDomain = optional.Some(inputDomain)
+			obs.ControlDNSLookupFailure = optional.Some(utilsStringPointerToString(resp.DNS.Failure))
+			obs.ControlDNSResolvedAddrs = optional.Some(NewSet(resp.DNS.Addrs...))
 			obs.MatchWithControlIPAddress = optional.Some(true)
 			obs.MatchWithControlIPAddressASN = optional.Some(true)
 			continue
@@ -496,14 +499,17 @@ func (c *WebObservationsContainer) controlMatchDNSLookupResults(inputDomain stri
 		}
 
 		// register the control DNS domain
-		obs.ControlDNSDomain = optional.Some(domain)
+		obs.ControlDNSDomain = optional.Some(inputDomain)
 
 		// register whether the control failed and skip in such a case
 		obs.ControlDNSLookupFailure = optional.Some(utilsStringPointerToString(resp.DNS.Failure))
 		if resp.DNS.Failure != nil {
 			continue
 		}
 
+		// register the resolved IP addresses
+		obs.ControlDNSResolvedAddrs = optional.Some(NewSet(resp.DNS.Addrs...))
+
 		// compute whether also the TH observed this addr
 		obs.MatchWithControlIPAddress = optional.Some(thAddrMap[addr])
 

diff --git a/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json b/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json
@@ -6,6 +6,7 @@
       "DNSLookupFailure": "dns_no_answer",
       "DNSQueryType": "AAAA",
       "DNSEngine": "udp",
+      "DNSResolvedAddrs": null,
       "IPAddress": null,
       "IPAddressASN": null,
       "IPAddressBogon": null,
@@ -29,6 +30,9 @@
       "TagFetchBody": null,
       "ControlDNSDomain": "expired.badssl.com",
       "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "ControlTCPConnectFailure": null,
       "MatchWithControlIPAddress": null,
       "MatchWithControlIPAddressASN": null,
@@ -47,6 +51,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": "ANY",
       "DNSEngine": "getaddrinfo",
+      "DNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -68,8 +75,11 @@
       "HTTPResponseIsFinal": null,
       "TagDepth": 0,
       "TagFetchBody": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "ControlDNSDomain": "expired.badssl.com",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "ControlTCPConnectFailure": null,
       "MatchWithControlIPAddress": null,
       "MatchWithControlIPAddressASN": null,
@@ -86,6 +96,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": "A",
       "DNSEngine": "udp",
+      "DNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -107,8 +120,11 @@
       "HTTPResponseIsFinal": null,
       "TagDepth": 0,
       "TagFetchBody": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "ControlDNSDomain": "expired.badssl.com",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "ControlTCPConnectFailure": null,
       "MatchWithControlIPAddress": null,
       "MatchWithControlIPAddressASN": null,
@@ -127,6 +143,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": null,
       "DNSEngine": null,
+      "DNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -150,6 +169,9 @@
       "TagFetchBody": true,
       "ControlDNSDomain": "expired.badssl.com",
       "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/...testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations_classic.json b/...testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations_classic.json
@@ -7,6 +7,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": "ANY",
       "DNSEngine": "getaddrinfo",
+      "DNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -28,8 +31,11 @@
       "HTTPResponseIsFinal": null,
       "TagDepth": 0,
       "TagFetchBody": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "ControlDNSDomain": "expired.badssl.com",
+      "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "ControlTCPConnectFailure": null,
       "MatchWithControlIPAddress": null,
       "MatchWithControlIPAddressASN": null,
@@ -48,6 +54,9 @@
       "DNSLookupFailure": "",
       "DNSQueryType": null,
       "DNSEngine": null,
+      "DNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
       "IPAddressBogon": false,
@@ -71,6 +80,9 @@
       "TagFetchBody": true,
       "ControlDNSDomain": "expired.badssl.com",
       "ControlDNSLookupFailure": "",
+      "ControlDNSResolvedAddrs": [
+        "104.154.89.105"
+      ],
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,