diff --git a/internal/enginenetx/beacons.go b/internal/enginenetx/beacons.go new file mode 100644 index 0000000000..db52e112ef --- /dev/null +++ b/internal/enginenetx/beacons.go @@ -0,0 +1,248 @@ +package enginenetx + +import ( + "context" + "math/rand" + "net" + "time" +) + +// BeaconsPolicy is a policy where we use beacons for communicating +// with the OONI backend, i.e., api.ooni.io. +// +// A beacon is an IP address that can route traffic from and to +// the OONI backend and accepts any SNI. +// +// The zero value is invalid; please, init MANDATORY fields. +type BeaconsPolicy struct { + // Fallback is the MANDATORY fallback policy. + Fallback HTTPSDialerPolicy +} + +var _ HTTPSDialerPolicy = &BeaconsPolicy{} + +// LookupTactics implements HTTPSDialerPolicy. +func (p *BeaconsPolicy) LookupTactics(ctx context.Context, domain, port string) <-chan *HTTPSDialerTactic { + out := make(chan *HTTPSDialerTactic) + + go func() { + defer close(out) + index := 0 + + // emit beacons related tactics first + for tx := range p.tacticsForDomain(domain, port) { + tx.InitialDelay = happyEyeballsDelay(index) + index += 1 + out <- tx + } + + // now emit tactics using the DNS + for tx := range p.Fallback.LookupTactics(ctx, domain, port) { + tx.InitialDelay = happyEyeballsDelay(index) + index += 1 + out <- tx + } + }() + + return out +} + +func (p *BeaconsPolicy) tacticsForDomain(domain, port string) <-chan *HTTPSDialerTactic { + out := make(chan *HTTPSDialerTactic) + + go func() { + defer close(out) + + // we currently only have beacons for api.ooni.io + if domain != "api.ooni.io" { + return + } + + snis := p.beaconsDomains() + r := rand.New(rand.NewSource(time.Now().UnixNano())) + r.Shuffle(len(snis), func(i, j int) { + snis[i], snis[j] = snis[j], snis[i] + }) + + ipAddrs := p.beaconsAddrs() + + for _, ipAddr := range ipAddrs { + for _, sni := range snis { + out <- &HTTPSDialerTactic{ + Endpoint: net.JoinHostPort(ipAddr, port), + InitialDelay: 0, + SNI: sni, + VerifyHostname: domain, + } + } + } + }() + + return out +} + +func (p *BeaconsPolicy) beaconsAddrs() (out []string) { + return append( + out, + "162.55.247.208", + ) +} + +func (p *BeaconsPolicy) beaconsDomains() (out []string) { + // See https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/issues/40273 + return append( + out, + "adtm.spreadshirts.net", + "alb.reddit.com", + "a.loveholidays.com", + "api.giphy.com", + "api.nextgen.guardianapps.co.uk", + "api.trademe.co.nz", + "app.launchdarkly.com", + "apps.voxmedia.com", + "assets0.uswitch.com", + "assets.boots.com", + "assets.dunelm.com", + "assets.guim.co.uk", + "assets.hearstapps.com", + "assets-jpcust.jwpsrv.com", + "assets.nymag.com", + "assets.thecut.com", + "atreseries.atresmedia.com", + "cdn.bfldr.com", + "cdn.concert.io", + "cdn.contentful.com", + "cdn.ketchjs.com", + "cdn.laredoute.com", + "cdn.polyfill.io", + "cdn.speedcurve.com", + "cdn.sstatic.net", + "cdn.taboola.com", + "client.grubstreet.com", + "client.nymag.com", + "client-registry.mutinycdn.com", + "client.thecut.com", + "client.thestrategist.co.uk", + "client.vulture.com", + "compote.slate.com", + "concertads-configs.vox-cdn.com", + "contributions.guardianapis.com", + "display.bidder.taboola.com", + "edgemesh.webflow.io", + "embed.api.video", + "epsf.ticketmaster.com", + "fastly.com", + "fastly.jsdelivr.net", + "fast.ssqt.io", + "fast.wistia.com", + "fdyn.pubwise.io", + "fonts.nymag.com", + "foursquare.com", + "frend-assets.freetls.fastly.net", + "f.vimeocdn.com", + "github.githubassets.com", + "global.ketchcdn.com", + "helpersng.taboola.com", + "hips.hearstapps.com", + "i.guimcode.co.uk", + "i.guim.co.uk", + "i.insider.com", + "images.mutinycdn.com", + "images.taboola.com", + "interactive.guim.co.uk", + "i.vimeocdn.com", + "js-agent.newrelic.com", + "js.sentry-cdn.com", + "linktr.ee", + "login.nine.com.au", + "lux.speedcurve.com", + "martech.condenastdigital.com", + "media0.giphy.com", + "media1.giphy.com", + "media2.giphy.com", + "media3.giphy.com", + "media.giphy.com", + "media.newyorker.com", + "media.wired.com", + "mparticle.weather.com", + "mv.outbrain.com", + "newrelic.com", + "next.ticketmaster.com", + "nm.realtyninja.com", + "pingback.giphy.com", + "pips.taboola.com", + "pitchfork.com", + "pixel.condenastdigital.com", + "player.ex.co", + "pm-widget.taboola.com", + "polyfill.io", + "prd.jwpltx.com", + "pyxis.nymag.com", + "rapid-cdn.yottaa.com", + "rtd-tm.everesttech.net", + "s1.ticketm.net", + "s3-media0.fl.yelpcdn.com", + "slate.com", + "sourcepoint.theguardian.com", + "ssl.p.jwpcdn.com", + "sstc.dunelm.com", + "static.ads-twitter.com", + "static.filestackapi.com", + "static.klaviyo.com", + "static.theguardian.com", + "static-tracking.klaviyo.com", + "s.w-x.co", + "trademe.tmcdn.co.nz", + "trc.taboola.com", + "t.seenthis.se", + "uploads.guim.co.uk", + "video.seenthis.se", + "vidstat.taboola.com", + "vod.api.video", + "vulcan.condenastdigital.com", + "widget.perfectmarket.com", + "www.allure.com", + "www.amazeelabs.com", + "www.architecturaldigest.com", + "www.blackpepper.co.nz", + "www.bonappetit.com", + "www.cntraveler.com", + "www.drupal.org", + "www.dunelm.com", + "www.epicurious.com", + "www.fastly.com", + "www.filestack.com", + "www.giphy.com", + "www.glamour.com", + "www.gq.com", + "www.insider.com", + "www.jimdo.com", + "www.loveholidays.com", + "www.madeiramadeira.com.br", + "www.newrelic.com", + "www.newyorker.com", + "www.pronovias.com", + "www.redditstatic.com", + "www.rvu.co.uk", + "www.self.com", + "www.shazam.com", + "www.shondaland.com", + "www.split.io", + "www.spreadgroup.com", + "www.spreadshirt.com", + "www.taboola.com", + "www.teenvogue.com", + "www.thecut.com", + "www.theguardian.com", + "www.them.us", + "www.ticketmaster.com", + "www.trademe.co.nz", + "www.vanityfair.com", + "www.vogue.com", + "www.wikihow.com", + "www.wired.com", + "www.yelp.com", + "x.giphy.com", + "yelp.com", + ) +} diff --git a/internal/enginenetx/beacons_test.go b/internal/enginenetx/beacons_test.go new file mode 100644 index 0000000000..56bf537d5c --- /dev/null +++ b/internal/enginenetx/beacons_test.go @@ -0,0 +1,128 @@ +package enginenetx + +import ( + "context" + "errors" + "net" + "testing" + + "github.com/ooni/probe-cli/v3/internal/mocks" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func TestBeaconsPolicy(t *testing.T) { + t.Run("for domains for which we don't have beacons and DNS failure", func(t *testing.T) { + expected := errors.New("mocked error") + policy := &BeaconsPolicy{ + Fallback: &HTTPSDialerNullPolicy{ + Logger: model.DiscardLogger, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return nil, expected + }, + }, + }, + } + + ctx := context.Background() + tactics := policy.LookupTactics(ctx, "www.example.com", "443") + + var count int + for range tactics { + count++ + } + + if count != 0 { + t.Fatal("expected to see zero tactics") + } + }) + + t.Run("for domains for which we don't have beacons and DNS success", func(t *testing.T) { + policy := &BeaconsPolicy{ + Fallback: &HTTPSDialerNullPolicy{ + Logger: model.DiscardLogger, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return []string{"93.184.216.34"}, nil + }, + }, + }, + } + + ctx := context.Background() + tactics := policy.LookupTactics(ctx, "www.example.com", "443") + + var count int + for tactic := range tactics { + count++ + + host, port, err := net.SplitHostPort(tactic.Endpoint) + if err != nil { + t.Fatal(err) + } + if port != "443" { + t.Fatal("the port should always be 443") + } + if host != "93.184.216.34" { + t.Fatal("the host should always be 93.184.216.34") + } + + if tactic.SNI != "www.example.com" { + t.Fatal("the SNI field should always be like `www.example.com`") + } + + if tactic.VerifyHostname != "www.example.com" { + t.Fatal("the VerifyHostname field should always be like `www.example.com`") + } + } + + if count != 1 { + t.Fatal("expected to see one tactic") + } + }) + + t.Run("for the api.ooni.io domain", func(t *testing.T) { + expected := errors.New("mocked error") + policy := &BeaconsPolicy{ + Fallback: &HTTPSDialerNullPolicy{ + Logger: model.DiscardLogger, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return nil, expected + }, + }, + }, + } + + ctx := context.Background() + tactics := policy.LookupTactics(ctx, "api.ooni.io", "443") + + var count int + for tactic := range tactics { + count++ + + host, port, err := net.SplitHostPort(tactic.Endpoint) + if err != nil { + t.Fatal(err) + } + if port != "443" { + t.Fatal("the port should always be 443") + } + if host != "162.55.247.208" { + t.Fatal("the host should always be 162.55.247.208") + } + + if tactic.SNI == "api.ooni.io" { + t.Fatal("we should not see the `api.ooni.io` SNI on the wire") + } + + if tactic.VerifyHostname != "api.ooni.io" { + t.Fatal("the VerifyHostname field should always be like `api.ooni.io`") + } + } + + if count <= 0 { + t.Fatal("expected to see at least one tactic") + } + }) +} diff --git a/internal/enginenetx/httpsdialernull.go b/internal/enginenetx/httpsdialernull.go index f9d88e4c29..001be25dcb 100644 --- a/internal/enginenetx/httpsdialernull.go +++ b/internal/enginenetx/httpsdialernull.go @@ -37,6 +37,13 @@ func (p *HTTPSDialerNullPolicy) LookupTactics( // make sure we close the output channel when done defer close(out) + // Do not even start the DNS lookup if the context has already been canceled, which + // happens if some policy running before us had successfully connected + if err := ctx.Err(); err != nil { + p.Logger.Debugf("HTTPSDialerNullPolicy: LookupTactics: %s", err.Error()) + return + } + // See https://github.com/ooni/probe-cli/pull/1295#issuecomment-1731243994 for context // on why here we MUST make sure we short-circuit IP addresses. resoWithShortCircuit := &netxlite.ResolverShortCircuitIPAddr{Resolver: p.Resolver} diff --git a/internal/enginenetx/httpsdialernull_test.go b/internal/enginenetx/httpsdialernull_test.go new file mode 100644 index 0000000000..37640797e8 --- /dev/null +++ b/internal/enginenetx/httpsdialernull_test.go @@ -0,0 +1,69 @@ +package enginenetx + +import ( + "context" + "testing" + + "github.com/ooni/probe-cli/v3/internal/mocks" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func TestHTTPSDialerNullPolicy(t *testing.T) { + t.Run("LookupTactics with canceled context", func(t *testing.T) { + var called int + + policy := &HTTPSDialerNullPolicy{ + Logger: &mocks.Logger{ + MockDebugf: func(format string, v ...interface{}) { + called++ + }, + }, + Resolver: &mocks.Resolver{}, // empty so we crash if we hit the resolver + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately cancel! + + tactics := policy.LookupTactics(ctx, "www.example.com", "443") + + var count int + for range tactics { + count++ + } + + if count != 0 { + t.Fatal("expected to see no tactic") + } + if called != 1 { + t.Fatal("did not call Debugf") + } + }) + + t.Run("we short circuit IP addresses", func(t *testing.T) { + policy := &HTTPSDialerNullPolicy{ + Logger: model.DiscardLogger, + Resolver: &mocks.Resolver{}, // empty so we crash if we hit the resolver + } + + tactics := policy.LookupTactics(context.Background(), "130.192.91.211", "443") + + var count int + for tactic := range tactics { + count++ + + if tactic.Endpoint != "130.192.91.211:443" { + t.Fatal("invalid endpoint") + } + if tactic.SNI != "130.192.91.211" { + t.Fatal("invalid SNI") + } + if tactic.VerifyHostname != "130.192.91.211" { + t.Fatal("invalid VerifyHostname") + } + } + + if count != 1 { + t.Fatal("expected to see just one tactic") + } + }) +} diff --git a/internal/enginenetx/network.go b/internal/enginenetx/network.go index 4da83c6a20..cef2e72ffb 100644 --- a/internal/enginenetx/network.go +++ b/internal/enginenetx/network.go @@ -135,8 +135,10 @@ func NewNetwork( // newHTTPSDialerPolicy contains the logic to select the [HTTPSDialerPolicy] to use. func newHTTPSDialerPolicy(kvStore model.KeyValueStore, logger model.Logger, resolver model.Resolver) HTTPSDialerPolicy { - // the fallback policy we're using is the "null" policy - fallback := &HTTPSDialerNullPolicy{logger, resolver} + // create a composed fallback TLS dialer policy + fallback := &BeaconsPolicy{ + Fallback: &HTTPSDialerNullPolicy{logger, resolver}, + } // make sure we honor a user-provided policy policy, err := NewHTTPSDialerStaticPolicy(kvStore, fallback)