From 6ef3fba5da3e3aa03de69af11ef6c8931fe10bcb Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 19 Sep 2023 13:57:47 +0200 Subject: [PATCH] feat(enginenetx): support HTTP and HTTPS proxies (#1282) This diff completes the work we have been doing for a few days now and provides HTTP and HTTPS proxy support, in addition to SOCKS5 support, for the engine-specific network. We did this work in the context of https://github.com/ooni/probe/issues/2531 and https://github.com/ooni/probe/issues/1955. BTW, the fact that tests used `measurexlite` and tracing is very nice here. It means the idea to write `measurexlite` based on context and tracing was good and could be used beyond its original design goals. --- internal/enginenetx/http.go | 10 +- internal/enginenetx/http_test.go | 265 ++++++++++++++++++++++++++++--- internal/netemx/qaenv.go | 10 +- internal/netxlite/maybeproxy.go | 2 + 4 files changed, 259 insertions(+), 28 deletions(-) diff --git a/internal/enginenetx/http.go b/internal/enginenetx/http.go index e4d56df5bb..f8af2c04da 100644 --- a/internal/enginenetx/http.go +++ b/internal/enginenetx/http.go @@ -48,13 +48,13 @@ func NewHTTPTransport( resolver model.Resolver, ) *HTTPTransport { dialer := netxlite.NewDialerWithResolver(logger, resolver) - dialer = netxlite.MaybeWrapWithProxyDialer(dialer, proxyURL) handshaker := netxlite.NewTLSHandshakerStdlib(logger) tlsDialer := netxlite.NewTLSDialer(dialer, handshaker) - // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport - // function, but we can probably avoid using it, given that this code is - // not using tracing and does not care about those quirks. - txp := netxlite.NewHTTPTransport(logger, dialer, tlsDialer) + txp := netxlite.NewHTTPTransportWithOptions( + logger, dialer, tlsDialer, + netxlite.HTTPTransportOptionDisableCompression(false), + netxlite.HTTPTransportOptionProxyURL(proxyURL), // nil implies "no proxy" + ) txp = bytecounter.WrapHTTPTransport(txp, counter) return &HTTPTransport{txp} } diff --git a/internal/enginenetx/http_test.go b/internal/enginenetx/http_test.go index c3604cd6d0..68f75ddfc8 100644 --- a/internal/enginenetx/http_test.go +++ b/internal/enginenetx/http_test.go @@ -1,34 +1,263 @@ -package enginenetx +package enginenetx_test import ( + "context" + "net" + "net/http" + "net/url" "testing" + "time" + "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/enginenetx" + "github.com/ooni/probe-cli/v3/internal/measurexlite" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netemx" "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/testingsocks5" + "github.com/ooni/probe-cli/v3/internal/testingx" ) -func TestHTTPTransport(t *testing.T) { +func TestHTTPTransportWAI(t *testing.T) { + t.Run("is WAI when not using any proxy", func(t *testing.T) { + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() - // TODO(bassosimone): we should replace this integration test with netemx - // as soon as we can sever the hard link between netxlite and this pkg - t.Run("is working as intended", func(t *testing.T) { - txp := NewHTTPTransport( - bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger)) - client := txp.NewHTTPClient() - resp, err := client.Get("https://www.google.com/robots.txt") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatal("unexpected status code") - } + env.Do(func() { + txp := enginenetx.NewHTTPTransport( + bytecounter.New(), + model.DiscardLogger, + nil, + netxlite.NewStdlibResolver(model.DiscardLogger), + ) + client := txp.NewHTTPClient() + resp, err := client.Get("https://www.example.com/") + if err != nil { + t.Fatal(err) + } + t.Logf("%+v", resp) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal("unexpected status code") + } + }) + }) + + t.Run("is WAI when using a SOCKS5 proxy", func(t *testing.T) { + // create internet measurement scenario + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + + // create a proxy using the client's TCP/IP stack + proxy := testingsocks5.MustNewServer( + log.Log, + &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}, + &net.TCPAddr{ + IP: net.ParseIP(env.ClientStack.IPAddress()), + Port: 9050, + }, + ) + defer proxy.Close() + + env.Do(func() { + txp := enginenetx.NewHTTPTransport( + bytecounter.New(), + model.DiscardLogger, + &url.URL{ + Scheme: "socks5", + Host: net.JoinHostPort(env.ClientStack.IPAddress(), "9050"), + Path: "/", + }, + netxlite.NewStdlibResolver(model.DiscardLogger), + ) + client := txp.NewHTTPClient() + + // To make sure we're connecting to the expected endpoint, we're going to use + // measurexlite and tracing to observe the destination endpoints + trace := measurexlite.NewTrace(0, time.Now()) + ctx := netxlite.ContextWithTrace(context.Background(), trace) + + // create request using the above context + // + // Implementation note: we cannot use HTTPS with netem here as explained + // by the https://github.com/ooni/probe/issues/2536 issue. + req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + t.Logf("%+v", resp) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal("unexpected status code") + } + + // make sure that we only connected to the SOCKS5 proxy + tcpConnects := trace.TCPConnects() + if len(tcpConnects) <= 0 { + t.Fatal("expected at least one TCP connect") + } + for idx, entry := range tcpConnects { + t.Logf("%d: %+v", idx, entry) + if entry.IP != env.ClientStack.IPAddress() { + t.Fatal("unexpected IP address") + } + if entry.Port != 9050 { + t.Fatal("unexpected port") + } + } + }) + }) + + t.Run("is WAI when using an HTTP proxy", func(t *testing.T) { + // create internet measurement scenario + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + + // create a proxy using the client's TCP/IP stack + proxy := testingx.MustNewHTTPServerEx( + &net.TCPAddr{IP: net.ParseIP(env.ClientStack.IPAddress()), Port: 8080}, + env.ClientStack, + testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{ + Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}), + ) + defer proxy.Close() + + env.Do(func() { + txp := enginenetx.NewHTTPTransport( + bytecounter.New(), + model.DiscardLogger, + &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(env.ClientStack.IPAddress(), "8080"), + Path: "/", + }, + netxlite.NewStdlibResolver(model.DiscardLogger), + ) + client := txp.NewHTTPClient() + + // To make sure we're connecting to the expected endpoint, we're going to use + // measurexlite and tracing to observe the destination endpoints + trace := measurexlite.NewTrace(0, time.Now()) + ctx := netxlite.ContextWithTrace(context.Background(), trace) + + // create request using the above context + // + // Implementation note: we cannot use HTTPS with netem here as explained + // by the https://github.com/ooni/probe/issues/2536 issue. + req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + t.Logf("%+v", resp) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal("unexpected status code") + } + + // make sure that we only connected to the HTTP proxy + tcpConnects := trace.TCPConnects() + if len(tcpConnects) <= 0 { + t.Fatal("expected at least one TCP connect") + } + for idx, entry := range tcpConnects { + t.Logf("%d: %+v", idx, entry) + if entry.IP != env.ClientStack.IPAddress() { + t.Fatal("unexpected IP address") + } + if entry.Port != 8080 { + t.Fatal("unexpected port") + } + } + }) + }) + + t.Run("is WAI when using an HTTPS proxy", func(t *testing.T) { + // create internet measurement scenario + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + + // create a proxy using the client's TCP/IP stack + proxy := testingx.MustNewHTTPServerTLSEx( + &net.TCPAddr{IP: net.ParseIP(env.ClientStack.IPAddress()), Port: 4443}, + env.ClientStack, + testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{ + Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}), + env.ClientStack, + ) + defer proxy.Close() + + env.Do(func() { + txp := enginenetx.NewHTTPTransport( + bytecounter.New(), + model.DiscardLogger, + &url.URL{ + Scheme: "https", + Host: net.JoinHostPort(env.ClientStack.IPAddress(), "4443"), + Path: "/", + }, + netxlite.NewStdlibResolver(model.DiscardLogger), + ) + client := txp.NewHTTPClient() + + // To make sure we're connecting to the expected endpoint, we're going to use + // measurexlite and tracing to observe the destination endpoints + trace := measurexlite.NewTrace(0, time.Now()) + ctx := netxlite.ContextWithTrace(context.Background(), trace) + + // create request using the above context + // + // Implementation note: we cannot use HTTPS with netem here as explained + // by the https://github.com/ooni/probe/issues/2536 issue. + req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + t.Logf("%+v", resp) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal("unexpected status code") + } + + // make sure that we only connected to the HTTPS proxy + tcpConnects := trace.TCPConnects() + if len(tcpConnects) <= 0 { + t.Fatal("expected at least one TCP connect") + } + for idx, entry := range tcpConnects { + t.Logf("%d: %+v", idx, entry) + if entry.IP != env.ClientStack.IPAddress() { + t.Fatal("unexpected IP address") + } + if entry.Port != 4443 { + t.Fatal("unexpected port") + } + } + }) }) t.Run("NewHTTPClient returns a client with a cookie jar", func(t *testing.T) { - txp := NewHTTPTransport( - bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger)) + txp := enginenetx.NewHTTPTransport( + bytecounter.New(), + model.DiscardLogger, + nil, + netxlite.NewStdlibResolver(model.DiscardLogger), + ) client := txp.NewHTTPClient() if client.Jar == nil { t.Fatal("expected non-nil cookie jar") diff --git a/internal/netemx/qaenv.go b/internal/netemx/qaenv.go index 0e03536516..17e1299696 100644 --- a/internal/netemx/qaenv.go +++ b/internal/netemx/qaenv.go @@ -140,8 +140,8 @@ type QAEnv struct { // clientNICWrapper is the OPTIONAL wrapper for the client NIC. clientNICWrapper netem.LinkNICWrapper - // clientStack is the client stack to use. - clientStack *netem.UNetStack + // ClientStack is the client stack to use. + ClientStack *netem.UNetStack // closables contains all entities where we have to take care of closing. closables []io.Closer @@ -197,7 +197,7 @@ func MustNewQAEnv(options ...QAEnvOption) *QAEnv { env := &QAEnv{ baseLogger: config.logger, clientNICWrapper: config.clientNICWrapper, - clientStack: nil, + ClientStack: nil, closables: []io.Closer{}, emulateAndroidGetaddrinfo: &atomic.Bool{}, ispResolverConfig: netem.NewDNSConfig(), @@ -208,7 +208,7 @@ func MustNewQAEnv(options ...QAEnvOption) *QAEnv { } // create all the required internals - env.clientStack = env.mustNewClientStack(config) + env.ClientStack = env.mustNewClientStack(config) env.closables = append(env.closables, env.mustNewNetStacks(config)...) return env @@ -306,7 +306,7 @@ func (env *QAEnv) EmulateAndroidGetaddrinfo(value bool) { // Do executes the given function such that [netxlite] code uses the // underlying clientStack rather than ordinary networking code. func (env *QAEnv) Do(function func()) { - var stack netem.UnderlyingNetwork = env.clientStack + var stack netem.UnderlyingNetwork = env.ClientStack if env.emulateAndroidGetaddrinfo.Load() { stack = &androidStack{stack} } diff --git a/internal/netxlite/maybeproxy.go b/internal/netxlite/maybeproxy.go index edab70b19b..ec5f6cc8fb 100644 --- a/internal/netxlite/maybeproxy.go +++ b/internal/netxlite/maybeproxy.go @@ -22,6 +22,8 @@ type proxyDialer struct { // MaybeWrapWithProxyDialer returns the original dialer if the proxyURL is nil // and otherwise returns a wrapped dialer that implements proxying. +// +// Deprecated: do not use this function in new code. func MaybeWrapWithProxyDialer(dialer model.Dialer, proxyURL *url.URL) model.Dialer { if proxyURL == nil { return dialer