Skip to content

Commit

Permalink
some more progress
Browse files Browse the repository at this point in the history
  • Loading branch information
bassosimone committed Nov 24, 2023
1 parent 6aff4f0 commit 94f9fd7
Show file tree
Hide file tree
Showing 17 changed files with 437 additions and 119 deletions.
25 changes: 25 additions & 0 deletions internal/cmd/pipeline/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package main

import (
"os"

"github.com/ooni/probe-cli/v3/internal/must"
"github.com/ooni/probe-cli/v3/internal/pipeline"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)

func main() {
rawMeasurement := must.ReadFile(os.Args[1])
var canonicalMeasurement pipeline.CanonicalMeasurement
must.UnmarshalJSON(rawMeasurement, &canonicalMeasurement)

db := pipeline.NewDB()
runtimex.Try0(db.Ingest(&canonicalMeasurement))

must.WriteFile("db.json", must.MarshalJSON(db), 0600)

ax := &pipeline.Analysis{}
ax.ComputeAllValues(db)

must.WriteFile("ax.json", must.MarshalJSON(ax), 0600)
}
1 change: 1 addition & 0 deletions internal/experiment/webconnectivitylte/cleartextflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) error {
tcpDialer := trace.NewDialerWithoutResolver(t.Logger)
tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address)
t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...)
t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...)
if err != nil {
// TODO(bassosimone): document why we're adding a request to the heap here
t.TestKeys.PrependRequests(newArchivalHTTPRequestResultWithError(
Expand Down
1 change: 1 addition & 0 deletions internal/experiment/webconnectivitylte/secureflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) error {
tcpDialer := trace.NewDialerWithoutResolver(t.Logger)
tcpConn, err := tcpDialer.DialContext(tcpCtx, "tcp", t.Address)
t.TestKeys.AppendTCPConnectResults(trace.TCPConnects()...)
t.TestKeys.AppendNetworkEvents(trace.NetworkEvents()...)
if err != nil {
// TODO(bassosimone): document why we're adding a request to the heap here
if t.PrioSelector != nil {
Expand Down
26 changes: 21 additions & 5 deletions internal/pipeline/analysiscore.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,24 @@ type Analysis struct {

// DNSExperimentFailure is a backward-compatibility value that contains the
// failure obtained when using getaddrinfo for the URL's domain
DNSExperimentFailure optional.Value[*string]
DNSExperimentFailure optional.Value[Failure]

// HTTPDiffBodyProportionFactor is the ratio of the two final bodies.
HTTPDiffBodyProportionFactor optional.Value[float64]

// HTTPDiffTitleMatch indicates whether the titles have common words in them.
HTTPDiffTitleMatch optional.Value[bool]

// HTTPDiffStatusCodeMatch indicates whether the status code matches taking into
// account some false positives that may arise.
HTTPDiffStatusCodeMatch optional.Value[bool]

// HTTPDiffUncommonHeadersMatch indicates whether uncommon headers match.
HTTPDiffUncommonHeadersMatch optional.Value[bool]

// HTTPExperimentFailure is a backward-compatibility value that contains the
// failure obtained for the final HTTP request made by the probe
HTTPExperimentFailure optional.Value[*string]
HTTPExperimentFailure optional.Value[Failure]

// HTTPUnexpectedFailure contains all the endpoint transaction IDs where
// the TH succeded while the probe failed to fetch a response
Expand All @@ -54,8 +67,8 @@ type Analysis struct {
TLSUnexpectedFailure []int64
}

// ComputeAll computes all the analysis flags using the DB.
func (ax *Analysis) ComputeAll(db *DB) {
// ComputeAllValues computes all the analysis values using the DB.
func (ax *Analysis) ComputeAllValues(db *DB) {
// DNS
ax.ComputeDNSExperimentFailure(db)
ax.ComputeDNSBogon(db)
Expand All @@ -75,5 +88,8 @@ func (ax *Analysis) ComputeAll(db *DB) {
ax.ComputeHTTPExperimentFailure(db)

// HTTP (diff)

ax.ComputeHTTPDiffBodyProportionFactor(db)
ax.ComputeHTTPDiffStatusCodeMatch(db)
ax.ComputeHTTPDiffUncommonHeadersMatch(db)
ax.ComputeHTTPDiffTitleMatch(db)
}
42 changes: 21 additions & 21 deletions internal/pipeline/analysisdns.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ import (

// ComputeDNSExperimentFailure computes DNSExperimentFailure.
func (ax *Analysis) ComputeDNSExperimentFailure(db *DB) {
for _, entry := range db.dnsByTxID {
for _, entry := range db.DNSByTxID {
// skip queries not for the original hostname
if db.urlHostname != entry.QueryHostname {
if db.URLHostname != entry.QueryHostname {
continue
}

Expand All @@ -20,7 +20,7 @@ func (ax *Analysis) ComputeDNSExperimentFailure(db *DB) {
}

// skip successful cases
if entry.Failure == nil {
if entry.Failure == "" {
continue
}

Expand All @@ -32,7 +32,7 @@ func (ax *Analysis) ComputeDNSExperimentFailure(db *DB) {

// ComputeDNSUnexpectedBogon computes DNSUnexpectedBogon.
func (ax *Analysis) ComputeDNSBogon(db *DB) {
for _, entry := range db.webByTxID {
for _, entry := range db.WebByTxID {
// skip all the entries without a bogon
if !entry.IPAddressIsBogon {
continue
Expand All @@ -52,24 +52,24 @@ func (ax *Analysis) ComputeDNSBogon(db *DB) {
// ComputeDNSUnexpectedFailure computes DNSUnexpectedFailure.
func (ax *Analysis) ComputeDNSUnexpectedFailure(db *DB) {
// we cannot run this algorithm if the control failed or returned no IP addresses.
if db.thDNSFailure != nil {
if db.THDNSFailure != "" {
return
}

// we cannot run this algorithm if the control returned no IP addresses.
if len(db.thDNSAddrs) <= 0 {
if len(db.THDNSAddrs) <= 0 {
return
}

// inspect DNS lookup results
for _, entry := range db.dnsByTxID {
for _, entry := range db.DNSByTxID {
// skip cases without failures
if entry.Failure == nil {
if entry.Failure == "" {
continue
}

// skip cases that query the wrong domain name
if entry.QueryHostname != db.urlHostname {
if entry.QueryHostname != db.URLHostname {
continue
}

Expand All @@ -82,7 +82,7 @@ func (ax *Analysis) ComputeDNSUnexpectedFailure(db *DB) {
}

// skip cases where there's no IPv6 addresses for a domain
if entry.QueryType == "AAAA" && *entry.Failure == netxlite.FailureDNSNoAnswer {
if entry.QueryType == "AAAA" && entry.Failure == netxlite.FailureDNSNoAnswer {
continue
}

Expand All @@ -93,24 +93,24 @@ func (ax *Analysis) ComputeDNSUnexpectedFailure(db *DB) {

func (ax *Analysis) dnsDiffHelper(db *DB, fx func(db *DB, entry *DNSObservation)) {
// we cannot run this algorithm if the control failed or returned no IP addresses.
if db.thDNSFailure != nil {
if db.THDNSFailure != "" {
return
}

// we cannot run this algorithm if the control returned no IP addresses.
if len(db.thDNSAddrs) <= 0 {
if len(db.THDNSAddrs) <= 0 {
return
}

// inspect DNS lookup results
for _, entry := range db.dnsByTxID {
for _, entry := range db.DNSByTxID {
// skip cases witht failures
if entry.Failure != nil {
if entry.Failure != "" {
continue
}

// skip cases that query the wrong domain name
if entry.QueryHostname != db.urlHostname {
if entry.QueryHostname != db.URLHostname {
continue
}

Expand All @@ -131,7 +131,7 @@ func (ax *Analysis) ComputeDNSUnexpectedAddr(db *DB) {
state[addr] |= OriginProbe
}

for addr := range db.thDNSAddrs {
for addr := range db.THDNSAddrs {
state[addr] |= OriginTH
}

Expand All @@ -155,7 +155,7 @@ func (ax *Analysis) ComputeDNSUnexpectedAddrASN(db *DB) {
}
}

for addr := range db.thDNSAddrs {
for addr := range db.THDNSAddrs {
if asn, _, err := geoipx.LookupASN(addr); err == nil {
state[int64(asn)] |= OriginTH
}
Expand All @@ -177,7 +177,7 @@ func (ax *Analysis) ComputeDNSWithTLSHandshakeFailureAddr(db *DB) {
for _, addr := range dns.IPAddrs {

// find the corresponding endpoint measurement
for _, epnt := range db.webByTxID {
for _, epnt := range db.WebByTxID {

// skip entries related to a different address
if epnt.IPAddress != addr {
Expand All @@ -190,12 +190,12 @@ func (ax *Analysis) ComputeDNSWithTLSHandshakeFailureAddr(db *DB) {
}

// skip entries where the handshake succeded
if epnt.TLSHandshakeFailure.Unwrap() == nil {
if epnt.TLSHandshakeFailure.Unwrap() == "" {
continue
}

// find the related TH measurement
thEpnt, good := db.thEpntByEpnt[epnt.Endpoint]
thEpnt, good := db.THEpntByEpnt[epnt.Endpoint]

// skip cases where there's no TH entry
if !good {
Expand All @@ -208,7 +208,7 @@ func (ax *Analysis) ComputeDNSWithTLSHandshakeFailureAddr(db *DB) {
}

// skip cases where the TH's handshake also failed
if thEpnt.TLSHandshakeFailure.Unwrap() != nil {
if thEpnt.TLSHandshakeFailure.Unwrap() != "" {
continue
}

Expand Down
18 changes: 9 additions & 9 deletions internal/pipeline/analysishttpcore.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@ package pipeline

import "github.com/ooni/probe-cli/v3/internal/optional"

func (ax *Analysis) httpExperimentFailureHelper(db *DB, fx func(txId int64, probeFailure *string)) {
func (ax *Analysis) httpExperimentFailureHelper(db *DB, fx func(txId int64, probeFailure Failure)) {
// skip if there's no final request
if db.webFinalRequest.IsNone() {
if db.WebFinalRequest.IsNone() {
return
}
probeFR := db.webFinalRequest.Unwrap()
probeFR := db.WebFinalRequest.Unwrap()

// skip if the HTTP failure is not defined (bug?)
if probeFR.HTTPFailure.IsNone() {
Expand All @@ -17,15 +17,15 @@ func (ax *Analysis) httpExperimentFailureHelper(db *DB, fx func(txId int64, prob
// skip if the final request succeded
// TODO(bassosimone): say that the probe succeeds and the TH fails, then what?
probeFailure := probeFR.HTTPFailure.Unwrap()
if probeFailure == nil {
if probeFailure == "" {
return
}

// skip if the final request is not defined for the TH
if db.thWeb.IsNone() {
if db.THWeb.IsNone() {
return
}
thFR := db.thWeb.Unwrap()
thFR := db.THWeb.Unwrap()

// skip if the failure is not defined for the TH
if thFR.HTTPFailure.IsNone() {
Expand All @@ -34,7 +34,7 @@ func (ax *Analysis) httpExperimentFailureHelper(db *DB, fx func(txId int64, prob

// skip if also the TH's HTTP request failed
thFailure := thFR.HTTPFailure.Unwrap()
if thFailure != nil {
if thFailure != "" {
return
}

Expand All @@ -44,14 +44,14 @@ func (ax *Analysis) httpExperimentFailureHelper(db *DB, fx func(txId int64, prob

// ComputeHTTPUnexpectedFailure computes HTTPUnexpectedFailure.
func (ax *Analysis) ComputeHTTPUnexpectedFailure(db *DB) {
ax.httpExperimentFailureHelper(db, func(txId int64, probeFailure *string) {
ax.httpExperimentFailureHelper(db, func(txId int64, probeFailure Failure) {
ax.HTTPUnexpectedFailure = append(ax.HTTPUnexpectedFailure, txId)
})
}

// ComputeHTTPExperimentFailure computes HTTPExperimentFailure.
func (ax *Analysis) ComputeHTTPExperimentFailure(db *DB) {
ax.httpExperimentFailureHelper(db, func(txId int64, probeFailure *string) {
ax.httpExperimentFailureHelper(db, func(txId int64, probeFailure Failure) {
ax.HTTPExperimentFailure = optional.Some(probeFailure)
})
}
Loading

0 comments on commit 94f9fd7

Please sign in to comment.