feat(enginenetx): introduce beacons policy (#1302)

The beacons policy generates tactics sending on the wire SNI values that
should be okay for specific domains.

This policy also uses a fallback policy to generate more standard
tactics using the DNS, but those tactics have lower priority meaning
that we try using beacons first.

When there is no beacon for a domain, this policy immediately falls back
to the underlying fallback policy.

Part of ooni/probe#2531

internal/enginenetx/beacons.go

@@ -0,0 +1,248 @@
+package enginenetx
+
+import (
+	"context"
+	"math/rand"
+	"net"
+	"time"
+)
+
+// BeaconsPolicy is a policy where we use beacons for communicating
+// with the OONI backend, i.e., api.ooni.io.
+//
+// A beacon is an IP address that can route traffic from and to
+// the OONI backend and accepts any SNI.
+//
+// The zero value is invalid; please, init MANDATORY fields.
+type BeaconsPolicy struct {
+	// Fallback is the MANDATORY fallback policy.
+	Fallback HTTPSDialerPolicy
+}
+
+var _ HTTPSDialerPolicy = &BeaconsPolicy{}
+
+// LookupTactics implements HTTPSDialerPolicy.
+func (p *BeaconsPolicy) LookupTactics(ctx context.Context, domain, port string) <-chan *HTTPSDialerTactic {
+	out := make(chan *HTTPSDialerTactic)
+
+	go func() {
+		defer close(out)
+		index := 0
+
+		// emit beacons related tactics first
+		for tx := range p.tacticsForDomain(domain, port) {
+			tx.InitialDelay = happyEyeballsDelay(index)
+			index += 1
+			out <- tx
+		}
+
+		// now emit tactics using the DNS
+		for tx := range p.Fallback.LookupTactics(ctx, domain, port) {
+			tx.InitialDelay = happyEyeballsDelay(index)
+			index += 1
+			out <- tx
+		}
+	}()
+
+	return out
+}
+
+func (p *BeaconsPolicy) tacticsForDomain(domain, port string) <-chan *HTTPSDialerTactic {
+	out := make(chan *HTTPSDialerTactic)
+
+	go func() {
+		defer close(out)
+
+		// we currently only have beacons for api.ooni.io
+		if domain != "api.ooni.io" {
+			return
+		}
+
+		snis := p.beaconsDomains()
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+		r.Shuffle(len(snis), func(i, j int) {
+			snis[i], snis[j] = snis[j], snis[i]
+		})
+
+		ipAddrs := p.beaconsAddrs()
+
+		for _, ipAddr := range ipAddrs {
+			for _, sni := range snis {
+				out <- &HTTPSDialerTactic{
+					Endpoint:       net.JoinHostPort(ipAddr, port),
+					InitialDelay:   0,
+					SNI:            sni,
+					VerifyHostname: domain,
+				}
+			}
+		}
+	}()
+
+	return out
+}
+
+func (p *BeaconsPolicy) beaconsAddrs() (out []string) {
+	return append(
+		out,
+		"162.55.247.208",
+	)
+}
+
+func (p *BeaconsPolicy) beaconsDomains() (out []string) {
+	// See https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/issues/40273
+	return append(
+		out,
+		"adtm.spreadshirts.net",
+		"alb.reddit.com",
+		"a.loveholidays.com",
+		"api.giphy.com",
+		"api.nextgen.guardianapps.co.uk",
+		"api.trademe.co.nz",
+		"app.launchdarkly.com",
+		"apps.voxmedia.com",
+		"assets0.uswitch.com",
+		"assets.boots.com",
+		"assets.dunelm.com",
+		"assets.guim.co.uk",
+		"assets.hearstapps.com",
+		"assets-jpcust.jwpsrv.com",
+		"assets.nymag.com",
+		"assets.thecut.com",
+		"atreseries.atresmedia.com",
+		"cdn.bfldr.com",
+		"cdn.concert.io",
+		"cdn.contentful.com",
+		"cdn.ketchjs.com",
+		"cdn.laredoute.com",
+		"cdn.polyfill.io",
+		"cdn.speedcurve.com",
+		"cdn.sstatic.net",
+		"cdn.taboola.com",
+		"client.grubstreet.com",
+		"client.nymag.com",
+		"client-registry.mutinycdn.com",
+		"client.thecut.com",
+		"client.thestrategist.co.uk",
+		"client.vulture.com",
+		"compote.slate.com",
+		"concertads-configs.vox-cdn.com",
+		"contributions.guardianapis.com",
+		"display.bidder.taboola.com",
+		"edgemesh.webflow.io",
+		"embed.api.video",
+		"epsf.ticketmaster.com",
+		"fastly.com",
+		"fastly.jsdelivr.net",
+		"fast.ssqt.io",
+		"fast.wistia.com",
+		"fdyn.pubwise.io",
+		"fonts.nymag.com",
+		"foursquare.com",
+		"frend-assets.freetls.fastly.net",
+		"f.vimeocdn.com",
+		"github.githubassets.com",
+		"global.ketchcdn.com",
+		"helpersng.taboola.com",
+		"hips.hearstapps.com",
+		"i.guimcode.co.uk",
+		"i.guim.co.uk",
+		"i.insider.com",
+		"images.mutinycdn.com",
+		"images.taboola.com",
+		"interactive.guim.co.uk",
+		"i.vimeocdn.com",
+		"js-agent.newrelic.com",
+		"js.sentry-cdn.com",
+		"linktr.ee",
+		"login.nine.com.au",
+		"lux.speedcurve.com",
+		"martech.condenastdigital.com",
+		"media0.giphy.com",
+		"media1.giphy.com",
+		"media2.giphy.com",
+		"media3.giphy.com",
+		"media.giphy.com",
+		"media.newyorker.com",
+		"media.wired.com",
+		"mparticle.weather.com",
+		"mv.outbrain.com",
+		"newrelic.com",
+		"next.ticketmaster.com",
+		"nm.realtyninja.com",
+		"pingback.giphy.com",
+		"pips.taboola.com",
+		"pitchfork.com",
+		"pixel.condenastdigital.com",
+		"player.ex.co",
+		"pm-widget.taboola.com",
+		"polyfill.io",
+		"prd.jwpltx.com",
+		"pyxis.nymag.com",
+		"rapid-cdn.yottaa.com",
+		"rtd-tm.everesttech.net",
+		"s1.ticketm.net",
+		"s3-media0.fl.yelpcdn.com",
+		"slate.com",
+		"sourcepoint.theguardian.com",
+		"ssl.p.jwpcdn.com",
+		"sstc.dunelm.com",
+		"static.ads-twitter.com",
+		"static.filestackapi.com",
+		"static.klaviyo.com",
+		"static.theguardian.com",
+		"static-tracking.klaviyo.com",
+		"s.w-x.co",
+		"trademe.tmcdn.co.nz",
+		"trc.taboola.com",
+		"t.seenthis.se",
+		"uploads.guim.co.uk",
+		"video.seenthis.se",
+		"vidstat.taboola.com",
+		"vod.api.video",
+		"vulcan.condenastdigital.com",
+		"widget.perfectmarket.com",
+		"www.allure.com",
+		"www.amazeelabs.com",
+		"www.architecturaldigest.com",
+		"www.blackpepper.co.nz",
+		"www.bonappetit.com",
+		"www.cntraveler.com",
+		"www.drupal.org",
+		"www.dunelm.com",
+		"www.epicurious.com",
+		"www.fastly.com",
+		"www.filestack.com",
+		"www.giphy.com",
+		"www.glamour.com",
+		"www.gq.com",
+		"www.insider.com",
+		"www.jimdo.com",
+		"www.loveholidays.com",
+		"www.madeiramadeira.com.br",
+		"www.newrelic.com",
+		"www.newyorker.com",
+		"www.pronovias.com",
+		"www.redditstatic.com",
+		"www.rvu.co.uk",
+		"www.self.com",
+		"www.shazam.com",
+		"www.shondaland.com",
+		"www.split.io",
+		"www.spreadgroup.com",
+		"www.spreadshirt.com",
+		"www.taboola.com",
+		"www.teenvogue.com",
+		"www.thecut.com",
+		"www.theguardian.com",
+		"www.them.us",
+		"www.ticketmaster.com",
+		"www.trademe.co.nz",
+		"www.vanityfair.com",
+		"www.vogue.com",
+		"www.wikihow.com",
+		"www.wired.com",
+		"www.yelp.com",
+		"x.giphy.com",
+		"yelp.com",
+	)
+}

internal/enginenetx/beacons_test.go

@@ -0,0 +1,128 @@
+package enginenetx
+
+import (
+	"context"
+	"errors"
+	"net"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/mocks"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+func TestBeaconsPolicy(t *testing.T) {
+	t.Run("for domains for which we don't have beacons and DNS failure", func(t *testing.T) {
+		expected := errors.New("mocked error")
+		policy := &BeaconsPolicy{
+			Fallback: &HTTPSDialerNullPolicy{
+				Logger: model.DiscardLogger,
+				Resolver: &mocks.Resolver{
+					MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+						return nil, expected
+					},
+				},
+			},
+		}
+
+		ctx := context.Background()
+		tactics := policy.LookupTactics(ctx, "www.example.com", "443")
+
+		var count int
+		for range tactics {
+			count++
+		}
+
+		if count != 0 {
+			t.Fatal("expected to see zero tactics")
+		}
+	})
+
+	t.Run("for domains for which we don't have beacons and DNS success", func(t *testing.T) {
+		policy := &BeaconsPolicy{
+			Fallback: &HTTPSDialerNullPolicy{
+				Logger: model.DiscardLogger,
+				Resolver: &mocks.Resolver{
+					MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+						return []string{"93.184.216.34"}, nil
+					},
+				},
+			},
+		}
+
+		ctx := context.Background()
+		tactics := policy.LookupTactics(ctx, "www.example.com", "443")
+
+		var count int
+		for tactic := range tactics {
+			count++
+
+			host, port, err := net.SplitHostPort(tactic.Endpoint)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if port != "443" {
+				t.Fatal("the port should always be 443")
+			}
+			if host != "93.184.216.34" {
+				t.Fatal("the host should always be 93.184.216.34")
+			}
+
+			if tactic.SNI != "www.example.com" {
+				t.Fatal("the SNI field should always be like `www.example.com`")
+			}
+
+			if tactic.VerifyHostname != "www.example.com" {
+				t.Fatal("the VerifyHostname field should always be like `www.example.com`")
+			}
+		}
+
+		if count != 1 {
+			t.Fatal("expected to see one tactic")
+		}
+	})
+
+	t.Run("for the api.ooni.io domain", func(t *testing.T) {
+		expected := errors.New("mocked error")
+		policy := &BeaconsPolicy{
+			Fallback: &HTTPSDialerNullPolicy{
+				Logger: model.DiscardLogger,
+				Resolver: &mocks.Resolver{
+					MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+						return nil, expected
+					},
+				},
+			},
+		}
+
+		ctx := context.Background()
+		tactics := policy.LookupTactics(ctx, "api.ooni.io", "443")
+
+		var count int
+		for tactic := range tactics {
+			count++
+
+			host, port, err := net.SplitHostPort(tactic.Endpoint)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if port != "443" {
+				t.Fatal("the port should always be 443")
+			}
+			if host != "162.55.247.208" {
+				t.Fatal("the host should always be 162.55.247.208")
+			}
+
+			if tactic.SNI == "api.ooni.io" {
+				t.Fatal("we should not see the `api.ooni.io` SNI on the wire")
+			}
+
+			if tactic.VerifyHostname != "api.ooni.io" {
+				t.Fatal("the VerifyHostname field should always be like `api.ooni.io`")
+			}
+		}
+
+		if count <= 0 {
+			t.Fatal("expected to see at least one tactic")
+		}
+	})
+}

internal/enginenetx/httpsdialernull.go

@@ -37,6 +37,13 @@ func (p *HTTPSDialerNullPolicy) LookupTactics(
 		// make sure we close the output channel when done
 		defer close(out)
 
+		// Do not even start the DNS lookup if the context has already been canceled, which
+		// happens if some policy running before us had successfully connected
+		if err := ctx.Err(); err != nil {
+			p.Logger.Debugf("HTTPSDialerNullPolicy: LookupTactics: %s", err.Error())
+			return
+		}
+
 		// See https://github.com/ooni/probe-cli/pull/1295#issuecomment-1731243994 for context
 		// on why here we MUST make sure we short-circuit IP addresses.
 		resoWithShortCircuit := &netxlite.ResolverShortCircuitIPAddr{Resolver: p.Resolver}