feat: rewrite the pipeline to match ooni/data more closely

ooni · Nov 24, 2023 · 5ad88d5 · 5ad88d5
1 parent ff42f3c
commit 5ad88d5
Show file tree

Hide file tree

Showing 10 changed files with 566 additions and 11 deletions.
diff --git a/internal/cmd/minipipeline/main.go b/internal/cmd/minipipeline/main.go
@@ -0,0 +1,32 @@
+package main
+
+import (
+	"os"
+
+	"github.com/ooni/probe-cli/v3/internal/minipipeline"
+	"github.com/ooni/probe-cli/v3/internal/must"
+	"github.com/ooni/probe-cli/v3/internal/pipeline"
+)
+
+func main() {
+	rawMeasurement := must.ReadFile(os.Args[1])
+	var meas pipeline.CanonicalMeasurement
+	must.UnmarshalJSON(rawMeasurement, &meas)
+
+	container := minipipeline.NewWebObservationsContainer()
+	container.CreateDNSLookupFailures(meas.TestKeys.Unwrap().Queries...)
+	container.CreateKnownIPAddresses(meas.TestKeys.Unwrap().Queries...)
+	container.CreateKnownTCPEndpoints(meas.TestKeys.Unwrap().TCPConnect...)
+	container.NoteTLSHandshakeResults(meas.TestKeys.Unwrap().TLSHandshakes...)
+	container.NoteHTTPRoundTripResults(meas.TestKeys.Unwrap().Requests...)
+	container.NoteControlResults(meas.TestKeys.Unwrap().XControlRequest.Unwrap(), meas.TestKeys.Unwrap().Control.Unwrap())
+
+	must.WriteFile("db.json", must.MarshalJSON(container), 0600)
+
+	/*
+		ax := &pipeline.Analysis{}
+		ax.ComputeAllValues(db)
+
+		must.WriteFile("ax.json", must.MarshalJSON(ax), 0600)
+	*/
+}
diff --git a/internal/experiment/webconnectivity/httpanalysis.go b/internal/experiment/webconnectivity/httpanalysis.go
@@ -202,7 +202,7 @@ func HTTPTitleMatch(tk urlgetter.TestKeys, ctrl ControlResponse) (out *bool) {
 	}
 	control := ctrl.HTTPRequest.Title
 	measurementBody := string(response.Body)
-	measurement := measurexlite.WebGetTitle(measurementBody)
+	measurement := measurexlite.WebGetTitleString(measurementBody)
 	if measurement == "" {
 		return
 	}

diff --git a/internal/experiment/webconnectivitylte/analysishttpdiff.go b/internal/experiment/webconnectivitylte/analysishttpdiff.go
@@ -231,7 +231,7 @@ func (tk *TestKeys) httpDiffTitleMatch(
 	}
 	control := ctrl.Title
 	measurementBody := string(response.Body)
-	measurement := measurexlite.WebGetTitle(measurementBody)
+	measurement := measurexlite.WebGetTitleString(measurementBody)
 	if control == "" || measurement == "" {
 		return
 	}

diff --git a/internal/measurexlite/web.go b/internal/measurexlite/web.go
@@ -6,12 +6,15 @@ package measurexlite
 
 import "regexp"
 
-// WebGetTitle returns the title or an empty string.
-func WebGetTitle(measurementBody string) string {
-	// MK used {1,128} but we're making it larger here to get longer titles
-	// e.g. <http://www.isa.gov.il/Pages/default.aspx>'s one
-	re := regexp.MustCompile(`(?i)<title>([^<]{1,512})</title>`)
-	v := re.FindStringSubmatch(measurementBody)
+// webTitleRegexp is the regexp to extract the title
+//
+// MK used {1,128} but we're making it larger here to get longer titles
+// e.g. <http://www.isa.gov.il/Pages/default.aspx>'s one
+var webTitleRegexp = regexp.MustCompile(`(?i)<title>([^<]{1,512})</title>`)
+
+// WebGetTitleString returns the title or an empty string.
+func WebGetTitleString(measurementBody string) string {
+	v := webTitleRegexp.FindStringSubmatch(measurementBody)
 	if len(v) < 2 {
 		return ""
 	}

diff --git a/internal/measurexlite/web_test.go b/internal/measurexlite/web_test.go
@@ -54,7 +54,7 @@ func TestWebGetTitle(t *testing.T) {
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotOut := WebGetTitle(tt.args.body)
+			gotOut := WebGetTitleString(tt.args.body)
 			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
 				t.Fatal(diff)
 			}

diff --git a/internal/minipipeline/doc.go b/internal/minipipeline/doc.go
@@ -0,0 +1,3 @@
+// Package minipipeline implements a minimal data processing pipeline used
+// to analyze local measurements collected by OONI Probe.
+package minipipeline