From 829b1b018d308508b3d1bbf1212ec04e5c0c07e3 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 15 Sep 2023 15:56:32 +0200 Subject: [PATCH] feat(testingx): introduce more comprehensive HTTP(S) proxy (#1274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want more comprehensive testing of how we use proxies during the bootstrap. Tests should encompass both SOCKS5 and HTTP(S) proxies. Tests should support using both the host network and using netem. This diff starts paving the way for improving proxy testing, by introducing in `./internal/testingx/httpproxy.go` an HTTP(S) proxy implementation supporting both the case of HTTP over HTTP(S) proxy (where you use the host header) and HTTPS over HTTP(S) proxy (where the client issues a `CONNECT` request to the remote endpoint). The previous implementation (in `./internal/testingx/httptestx.go`) is still there, for now. We used the previous implementation, which only supported the host header, as a starting point for writing the new one. More in detail, this diff introduces the new proxy and its tests. Because testing the proxy functionality is a bit complex, I chose to use a separate package and also write tests for the tests. Obviously, we're still using `netxlite` as the underlying library, so there is some circularity in testing, but `netxlite` also has its own tests, so we should probably be fine. The separate package with tests is `./internal/testingproxy`. While working on this package, I realized the need to forward the CA used by the proxy. This happens in `./internal/testingproxy/hosthttps.go`. If we do not do this, we see the following failure: ``` === RUN TestWorkingAsIntended/fetching_https://www.example.com/_using_the_host_network_and_an_HTTPS_proxy 2023/09/15 15:28:39 debug > GET https://www.example.com/ 2023/09/15 15:28:39 debug > 2023/09/15 15:28:39 dialerWithAssertions: verified that the network is tcp as expected 2023/09/15 15:28:39 dialerWithAssertions: verified that the address is 127.0.0.1 as expected 2023/09/15 15:28:39 debug dial 127.0.0.1:61772/tcp... 2023/09/15 15:28:39 debug dial_address 127.0.0.1:61772/tcp... 2023/09/15 15:28:39 debug dial_address 127.0.0.1:61772/tcp... ok in 212.083µs 2023/09/15 15:28:39 debug dial 127.0.0.1:61772/tcp... ok in 222.208µs 2023/09/15 15:28:39 debug tls {sni=127.0.0.1 next=[]}... 2023/09/15 15:28:39 debug tls {sni=127.0.0.1 next=[]}... ssl_unknown_authority in 3.687875ms [...] ``` In other words, the TLS config we're using does not know the proxy CA. So, in this diff you also see the required work to forward the proxy CA, including the related netem changes in https://github.com/ooni/netem/pull/38. The reference issue is https://github.com/ooni/probe/issues/2531. --- go.mod | 2 +- go.sum | 4 +- internal/testingproxy/dialer.go | 49 +++++++++ internal/testingproxy/doc.go | 2 + internal/testingproxy/hosthttp.go | 74 +++++++++++++ internal/testingproxy/hosthttps.go | 83 ++++++++++++++ internal/testingproxy/httputils.go | 52 +++++++++ internal/testingproxy/httputils_test.go | 122 +++++++++++++++++++++ internal/testingproxy/qa_test.go | 19 ++++ internal/testingproxy/testcase.go | 26 +++++ internal/testingx/httpproxy.go | 138 ++++++++++++++++++++++++ internal/testingx/httpproxy_test.go | 19 ++++ internal/testingx/httptestx.go | 16 +++ internal/testingx/tlsx.go | 9 ++ script/nocopyreadall.bash | 6 ++ 15 files changed, 618 insertions(+), 3 deletions(-) create mode 100644 internal/testingproxy/dialer.go create mode 100644 internal/testingproxy/doc.go create mode 100644 internal/testingproxy/hosthttp.go create mode 100644 internal/testingproxy/hosthttps.go create mode 100644 internal/testingproxy/httputils.go create mode 100644 internal/testingproxy/httputils_test.go create mode 100644 internal/testingproxy/qa_test.go create mode 100644 internal/testingproxy/testcase.go create mode 100644 internal/testingx/httpproxy.go create mode 100644 internal/testingx/httpproxy_test.go diff --git a/go.mod b/go.mod index 509c8e649a..a196f65ae2 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/mitchellh/go-wordwrap v1.0.1 github.com/montanaflynn/stats v0.7.1 github.com/ooni/go-libtor v1.1.8 - github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3 + github.com/ooni/netem v0.0.0-20230915101649-ab0dc13be014 github.com/ooni/oocrypto v0.5.3 github.com/ooni/oohttp v0.6.3 github.com/ooni/probe-assets v0.18.0 diff --git a/go.sum b/go.sum index d6efac959d..cc20af397b 100644 --- a/go.sum +++ b/go.sum @@ -483,8 +483,8 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= github.com/ooni/go-libtor v1.1.8 h1:Wo3V3DVTxl5vZdxtQakqYP+DAHx7pPtAFSl1bnAa08w= github.com/ooni/go-libtor v1.1.8/go.mod h1:q1YyLwRD9GeMyeerVvwc0vJ2YgwDLTp2bdVcrh/JXyI= -github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3 h1:zpTbzNzpo00cKbjLLnWMKjZeGLdoNC81vMiBDiur7NU= -github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3/go.mod h1:3LJOzTIu2O4ADDJN2ILG4ViJOqyH/u9fKY8QT2Rma8Y= +github.com/ooni/netem v0.0.0-20230915101649-ab0dc13be014 h1:4kOSV4D6mwrdoNUkAbGz1XoFUPcjsuNlLhZMc2CoHGg= +github.com/ooni/netem v0.0.0-20230915101649-ab0dc13be014/go.mod h1:3LJOzTIu2O4ADDJN2ILG4ViJOqyH/u9fKY8QT2Rma8Y= github.com/ooni/oocrypto v0.5.3 h1:CAb0Ze6q/EWD1PRGl9KqpzMfkut4O3XMaiKYsyxrWOs= github.com/ooni/oocrypto v0.5.3/go.mod h1:HjEQ5pQBl6btcWgAsKKq1tFo8CfBrZu63C/vPAUGIDk= github.com/ooni/oohttp v0.6.3 h1:MHydpeAPU/LSDSI/hIFJwZm4afBhd2Yo+rNxxFdeMCY= diff --git a/internal/testingproxy/dialer.go b/internal/testingproxy/dialer.go new file mode 100644 index 0000000000..0469fa250e --- /dev/null +++ b/internal/testingproxy/dialer.go @@ -0,0 +1,49 @@ +package testingproxy + +import ( + "context" + "fmt" + "log" + "net" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// dialerWithAssertions ensures that we're dialing with the proxy address. +type dialerWithAssertions struct { + // ExpectAddress is the expected IP address to dial + ExpectAddress string + + // Dialer is the underlying dialer to use + Dialer model.Dialer +} + +var _ model.Dialer = &dialerWithAssertions{} + +// CloseIdleConnections implements model.Dialer. +func (d *dialerWithAssertions) CloseIdleConnections() { + d.Dialer.CloseIdleConnections() +} + +// DialContext implements model.Dialer. +func (d *dialerWithAssertions) DialContext(ctx context.Context, network string, address string) (net.Conn, error) { + // make sure the network is tcp + const expectNetwork = "tcp" + runtimex.Assert( + network == expectNetwork, + fmt.Sprintf("dialerWithAssertions: expected %s, got %s", expectNetwork, network), + ) + log.Printf("dialerWithAssertions: verified that the network is %s as expected", expectNetwork) + + // make sure the IP address is the expected one + ipAddr, _ := runtimex.Try2(net.SplitHostPort(address)) + runtimex.Assert( + ipAddr == d.ExpectAddress, + fmt.Sprintf("dialerWithAssertions: expected %s, got %s", d.ExpectAddress, ipAddr), + ) + log.Printf("dialerWithAssertions: verified that the address is %s as expected", d.ExpectAddress) + + // now that we're sure we're using the proxy, we can actually dial + return d.Dialer.DialContext(ctx, network, address) +} diff --git a/internal/testingproxy/doc.go b/internal/testingproxy/doc.go new file mode 100644 index 0000000000..b7560cc81b --- /dev/null +++ b/internal/testingproxy/doc.go @@ -0,0 +1,2 @@ +// Package testingproxy contains shared test cases for the proxies. +package testingproxy diff --git a/internal/testingproxy/hosthttp.go b/internal/testingproxy/hosthttp.go new file mode 100644 index 0000000000..cb3b1773a3 --- /dev/null +++ b/internal/testingproxy/hosthttp.go @@ -0,0 +1,74 @@ +package testingproxy + +import ( + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/testingx" +) + +// WithHostNetworkHTTPProxyAndURL returns a [TestCase] where: +// +// - we fetch a URL; +// +// - using the host network; +// +// - and an HTTP proxy. +// +// Because this [TestCase] uses the host network, it does not run in -short mode. +func WithHostNetworkHTTPProxyAndURL(URL string) TestCase { + return &hostNetworkTestCaseWithHTTP{ + TargetURL: URL, + } +} + +type hostNetworkTestCaseWithHTTP struct { + TargetURL string +} + +var _ TestCase = &hostNetworkTestCaseWithHTTP{} + +// Name implements TestCase. +func (tc *hostNetworkTestCaseWithHTTP) Name() string { + return fmt.Sprintf("fetching %s using the host network and an HTTP proxy", tc.TargetURL) +} + +// Run implements TestCase. +func (tc *hostNetworkTestCaseWithHTTP) Run(t *testing.T) { + // create an instance of Netx where the underlying network is nil, + // which means we're using the host's network + netx := &netxlite.Netx{Underlying: nil} + + // create the proxy server using the host network + proxyServer := testingx.MustNewHTTPServer(testingx.NewHTTPProxyHandler(log.Log, netx)) + defer proxyServer.Close() + + //log.SetLevel(log.DebugLevel) + + // create an HTTP client configured to use the given proxy + // + // note how we use a dialer that asserts that we're using the proxy IP address + // rather than the host address, so we're sure that we're using the proxy + dialer := &dialerWithAssertions{ + ExpectAddress: "127.0.0.1", + Dialer: netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)), + } + tlsDialer := netxlite.NewTLSDialer(dialer, netxlite.NewTLSHandshakerStdlib(log.Log)) + txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, + netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL)))) + client := &http.Client{Transport: txp} + defer client.CloseIdleConnections() + + // get the homepage and assert we're getting a succesful response + httpCheckResponse(t, client, tc.TargetURL) +} + +// Short implements TestCase. +func (tc *hostNetworkTestCaseWithHTTP) Short() bool { + return false +} diff --git a/internal/testingproxy/hosthttps.go b/internal/testingproxy/hosthttps.go new file mode 100644 index 0000000000..67be7cb417 --- /dev/null +++ b/internal/testingproxy/hosthttps.go @@ -0,0 +1,83 @@ +package testingproxy + +import ( + "crypto/tls" + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/testingx" +) + +// WithHostNetworkHTTPWithTLSProxyAndURL returns a [TestCase] where: +// +// - we fetch a URL; +// +// - using the host network; +// +// - and an HTTPS proxy. +// +// Because this [TestCase] uses the host network, it does not run in -short mode. +func WithHostNetworkHTTPWithTLSProxyAndURL(URL string) TestCase { + return &hostNetworkTestCaseWithHTTPWithTLS{ + TargetURL: URL, + } +} + +type hostNetworkTestCaseWithHTTPWithTLS struct { + TargetURL string +} + +var _ TestCase = &hostNetworkTestCaseWithHTTPWithTLS{} + +// Name implements TestCase. +func (tc *hostNetworkTestCaseWithHTTPWithTLS) Name() string { + return fmt.Sprintf("fetching %s using the host network and an HTTPS proxy", tc.TargetURL) +} + +// Run implements TestCase. +func (tc *hostNetworkTestCaseWithHTTPWithTLS) Run(t *testing.T) { + // create an instance of Netx where the underlying network is nil, + // which means we're using the host's network + netx := &netxlite.Netx{Underlying: nil} + + // create the proxy server using the host network + proxyServer := testingx.MustNewHTTPServerTLS(testingx.NewHTTPProxyHandler(log.Log, netx)) + defer proxyServer.Close() + + //log.SetLevel(log.DebugLevel) + + // extend the default cert pool with the proxy's own CA + pool := netxlite.NewMozillaCertPool() + pool.AddCert(proxyServer.CACert) + tlsConfig := &tls.Config{RootCAs: pool} + + // create an HTTP client configured to use the given proxy + // + // note how we use a dialer that asserts that we're using the proxy IP address + // rather than the host address, so we're sure that we're using the proxy + dialer := &dialerWithAssertions{ + ExpectAddress: "127.0.0.1", + Dialer: netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)), + } + tlsDialer := netxlite.NewTLSDialerWithConfig( + dialer, netxlite.NewTLSHandshakerStdlib(log.Log), + tlsConfig, + ) + txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, + netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL)))) + client := &http.Client{Transport: txp} + defer client.CloseIdleConnections() + + // get the homepage and assert we're getting a succesful response + httpCheckResponse(t, client, tc.TargetURL) +} + +// Short implements TestCase. +func (tc *hostNetworkTestCaseWithHTTPWithTLS) Short() bool { + return false +} diff --git a/internal/testingproxy/httputils.go b/internal/testingproxy/httputils.go new file mode 100644 index 0000000000..f377754ed0 --- /dev/null +++ b/internal/testingproxy/httputils.go @@ -0,0 +1,52 @@ +package testingproxy + +import "net/http" + +type httpClient interface { + Get(URL string) (*http.Response, error) +} + +type httpClientMock struct { + MockGet func(URL string) (*http.Response, error) +} + +var _ httpClient = &httpClientMock{} + +// Get implements httpClient. +func (c *httpClientMock) Get(URL string) (*http.Response, error) { + return c.MockGet(URL) +} + +type httpTestingT interface { + Logf(format string, v ...any) + Fatal(v ...any) +} + +type httpTestingTMock struct { + MockLogf func(format string, v ...any) + MockFatal func(v ...any) +} + +var _ httpTestingT = &httpTestingTMock{} + +// Fatal implements httpTestingT. +func (t *httpTestingTMock) Fatal(v ...any) { + t.MockFatal(v...) +} + +// Logf implements httpTestingT. +func (t *httpTestingTMock) Logf(format string, v ...any) { + t.MockLogf(format, v...) +} + +func httpCheckResponse(t httpTestingT, client httpClient, targetURL string) { + resp, err := client.Get(targetURL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + t.Logf("%+v", resp) + if resp.StatusCode != 200 { + t.Fatal("invalid status code") + } +} diff --git a/internal/testingproxy/httputils_test.go b/internal/testingproxy/httputils_test.go new file mode 100644 index 0000000000..a01c34cac9 --- /dev/null +++ b/internal/testingproxy/httputils_test.go @@ -0,0 +1,122 @@ +package testingproxy + +import ( + "bytes" + "errors" + "io" + "net/http" + "testing" +) + +func TestHTTPClientMock(t *testing.T) { + t.Run("for Get", func(t *testing.T) { + expected := errors.New("mocked error") + c := &httpClientMock{ + MockGet: func(URL string) (*http.Response, error) { + return nil, expected + }, + } + resp, err := c.Get("https://www.google.com/") + if !errors.Is(err, expected) { + t.Fatal("unexpected error") + } + if resp != nil { + t.Fatal("expected nil response") + } + }) +} + +func TestHTTPTestingTMock(t *testing.T) { + t.Run("for Fatal", func(t *testing.T) { + var called bool + mt := &httpTestingTMock{ + MockFatal: func(v ...any) { + called = true + }, + } + mt.Fatal("antani") + if !called { + t.Fatal("not called") + } + }) + + t.Run("for Logf", func(t *testing.T) { + var called bool + mt := &httpTestingTMock{ + MockLogf: func(format string, v ...any) { + called = true + }, + } + mt.Logf("antani %v", "mascetti") + if !called { + t.Fatal("not called") + } + }) +} + +func TestHTTPCheckResponseHandlesFailures(t *testing.T) { + type testcase struct { + name string + mclient httpClient + expectLog bool + } + + testcases := []testcase{{ + name: "when HTTP round trip fails", + mclient: &httpClientMock{ + MockGet: func(URL string) (*http.Response, error) { + return nil, io.EOF + }, + }, + expectLog: false, + }, { + name: "with unexpected status code", + mclient: &httpClientMock{ + MockGet: func(URL string) (*http.Response, error) { + resp := &http.Response{ + StatusCode: 404, + Body: io.NopCloser(bytes.NewReader(nil)), + } + return resp, nil + }, + }, + expectLog: true, + }} + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + + // prepare for capturing what happened + var ( + calledLogf bool + calledFatal bool + ) + mt := &httpTestingTMock{ + MockLogf: func(format string, v ...any) { + calledLogf = true + }, + MockFatal: func(v ...any) { + calledFatal = true + panic(v) + }, + } + + // make sure we handle the panic and check what happened + defer func() { + result := recover() + if result == nil { + t.Fatal("did not panic") + } + if !calledFatal { + t.Fatal("did not actually call t.Fatal") + } + if tc.expectLog != calledLogf { + t.Fatal("tc.expectLog is", tc.expectLog, "but calledLogf is", calledLogf) + } + }() + + // invoke the function we're testing + httpCheckResponse(mt, tc.mclient, "https://www.google.com/") + }) + } +} diff --git a/internal/testingproxy/qa_test.go b/internal/testingproxy/qa_test.go new file mode 100644 index 0000000000..3e4019068a --- /dev/null +++ b/internal/testingproxy/qa_test.go @@ -0,0 +1,19 @@ +package testingproxy_test + +import ( + "testing" + + "github.com/ooni/probe-cli/v3/internal/testingproxy" +) + +func TestWorkingAsIntended(t *testing.T) { + for _, testCase := range testingproxy.AllTestCases { + short := testCase.Short() + if !short && testing.Short() { + t.Skip("skip test in short mode") + } + t.Run(testCase.Name(), func(t *testing.T) { + testCase.Run(t) + }) + } +} diff --git a/internal/testingproxy/testcase.go b/internal/testingproxy/testcase.go new file mode 100644 index 0000000000..8892795a67 --- /dev/null +++ b/internal/testingproxy/testcase.go @@ -0,0 +1,26 @@ +package testingproxy + +import "testing" + +// TestCase is a test case implemented by this package. +type TestCase interface { + // Name returns the test case name. + Name() string + + // Run runs the test case. + Run(t *testing.T) + + // Short returns whether this is a short test. + Short() bool +} + +// AllTestCases contains all the test cases. +var AllTestCases = []TestCase{ + // host network and HTTP proxy + WithHostNetworkHTTPProxyAndURL("http://www.example.com/"), + WithHostNetworkHTTPProxyAndURL("https://www.example.com/"), + + // host network and HTTPS proxy + WithHostNetworkHTTPWithTLSProxyAndURL("http://www.example.com/"), + WithHostNetworkHTTPWithTLSProxyAndURL("https://www.example.com/"), +} diff --git a/internal/testingx/httpproxy.go b/internal/testingx/httpproxy.go new file mode 100644 index 0000000000..a66c060d75 --- /dev/null +++ b/internal/testingx/httpproxy.go @@ -0,0 +1,138 @@ +package testingx + +import ( + "io" + "net/http" + "sync" + + "github.com/ooni/probe-cli/v3/internal/logx" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// HTTPProxyHandlerNetx abstracts [*netxlite.Netx] for the [*HTTPProxyHandler]. +type HTTPProxyHandlerNetx interface { + // NewDialerWithResolver creates a new dialer using the given resolver and logger. + NewDialerWithResolver(dl model.DebugLogger, r model.Resolver, w ...model.DialerWrapper) model.Dialer + + // NewHTTPTransportStdlib creates a new HTTP transport using the stdlib. + NewHTTPTransportStdlib(dl model.DebugLogger) model.HTTPTransport + + // NewStdlibResolver creates a new resolver that tries to use the getaddrinfo libc call. + NewStdlibResolver(logger model.DebugLogger) model.Resolver +} + +// httpProxyHandler is an HTTP/HTTPS proxy. +type httpProxyHandler struct { + // Logger is the logger to use. + Logger model.Logger + + // Netx is the network to use. + Netx HTTPProxyHandlerNetx +} + +// NewHTTPProxyHandler constructs a new [*HTTPProxyHandler]. +func NewHTTPProxyHandler(logger model.Logger, netx HTTPProxyHandlerNetx) http.Handler { + return &httpProxyHandler{ + Logger: &logx.PrefixLogger{ + Prefix: "PROXY: ", + Logger: logger, + }, + Netx: netx, + } +} + +// ServeHTTP implements http.Handler. +func (ph *httpProxyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + ph.Logger.Infof("request: %+v", req) + + switch req.Method { + case http.MethodConnect: + ph.connect(rw, req) + + case http.MethodGet: + ph.get(rw, req) + + default: + rw.WriteHeader(http.StatusNotImplemented) + } +} + +func (ph *httpProxyHandler) connect(rw http.ResponseWriter, req *http.Request) { + resolver := ph.Netx.NewStdlibResolver(ph.Logger) + dialer := ph.Netx.NewDialerWithResolver(ph.Logger, resolver) + + sconn, err := dialer.DialContext(req.Context(), "tcp", req.Host) + if err != nil { + rw.WriteHeader(http.StatusBadGateway) + return + } + + hijacker := rw.(http.Hijacker) + cconn, buffered := runtimex.Try2(hijacker.Hijack()) + runtimex.Assert(buffered.Reader.Buffered() <= 0, "data before finishing HTTP handshake") + + _, _ = cconn.Write([]byte("HTTP/1.1 200 Ok\r\n\r\n")) + + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + _, _ = io.Copy(sconn, cconn) + }() + + wg.Add(1) + go func() { + defer wg.Done() + _, _ = io.Copy(cconn, sconn) + }() + + wg.Wait() +} + +func (ph *httpProxyHandler) get(rw http.ResponseWriter, req *http.Request) { + // reject requests that already visited the proxy and requests we cannot route + if req.Host == "" || req.Header.Get("Via") != "" { + rw.WriteHeader(http.StatusBadRequest) + return + } + + // clone the request before modifying it + req = req.Clone(req.Context()) + + // include proxy header to prevent sending requests to ourself + req.Header.Add("Via", "testingx/0.1.0") + + // fix: "http: Request.RequestURI can't be set in client requests" + req.RequestURI = "" + + // fix: `http: unsupported protocol scheme ""` + req.URL.Host = req.Host + + // fix: "http: no Host in request URL" + req.URL.Scheme = "http" + + ph.Logger.Debugf("sending request: %s", req) + + // create HTTP client using netx + txp := ph.Netx.NewHTTPTransportStdlib(ph.Logger) + + // obtain response + resp, err := txp.RoundTrip(req) + if err != nil { + ph.Logger.Warnf("request failed: %s", err.Error()) + rw.WriteHeader(http.StatusBadGateway) + return + } + + // write response + rw.WriteHeader(resp.StatusCode) + for key, values := range resp.Header { + for _, value := range values { + rw.Header().Add(key, value) + } + } + + // write response body + _, _ = io.Copy(rw, resp.Body) +} diff --git a/internal/testingx/httpproxy_test.go b/internal/testingx/httpproxy_test.go new file mode 100644 index 0000000000..56fb6c5058 --- /dev/null +++ b/internal/testingx/httpproxy_test.go @@ -0,0 +1,19 @@ +package testingx_test + +import ( + "testing" + + "github.com/ooni/probe-cli/v3/internal/testingproxy" +) + +func TestHTTPProxyHandler(t *testing.T) { + for _, testCase := range testingproxy.AllTestCases { + short := testCase.Short() + if !short && testing.Short() { + t.Skip("skip test in short mode") + } + t.Run(testCase.Name(), func(t *testing.T) { + testCase.Run(t) + }) + } +} diff --git a/internal/testingx/httptestx.go b/internal/testingx/httptestx.go index 61e4dd21d0..fb7480d4c7 100644 --- a/internal/testingx/httptestx.go +++ b/internal/testingx/httptestx.go @@ -21,20 +21,35 @@ import ( // transitioning the code from that struct to this one. type HTTPServer struct { // Config contains the server started by the constructor. + // + // This field also exists in the [*net/http/httptest.Server] struct. Config *http.Server // Listener is the underlying [net.Listener]. + // + // This field also exists in the [*net/http/httptest.Server] struct. Listener net.Listener // TLS contains the TLS configuration used by the constructor, or nil // if you constructed a server that does not use TLS. + // + // This field also exists in the [*net/http/httptest.Server] struct. TLS *tls.Config // URL is the base URL used by the server. + // + // This field also exists in the [*net/http/httptest.Server] struct. URL string // X509CertPool is the X.509 cert pool we're using or nil. + // + // This field is an extension that is not present in the httptest package. X509CertPool *x509.CertPool + + // CACert is the CA used by this server. + // + // This field is an extension that is not present in the httptest package. + CACert *x509.Certificate } // MustNewHTTPServer is morally equivalent to [httptest.NewHTTPServer]. @@ -79,6 +94,7 @@ func mustNewHTTPServer( switch !tlsConfig.IsNone() { case true: baseURL.Scheme = "https" + srv.CACert = tlsConfig.Unwrap().CACert() srv.TLS = tlsConfig.Unwrap().ServerTLSConfig() srv.Config.TLSConfig = srv.TLS srv.X509CertPool = runtimex.Try1(tlsConfig.Unwrap().DefaultCertPool()) diff --git a/internal/testingx/tlsx.go b/internal/testingx/tlsx.go index efcdc676ba..296493bfdb 100644 --- a/internal/testingx/tlsx.go +++ b/internal/testingx/tlsx.go @@ -25,6 +25,10 @@ import ( // // Use the former when you're using netem; the latter when using the stdlib. type TLSMITMProvider interface { + // CACert returns the CA certificate used by the server, which + // allows you to add to an existing [*x509.CertPool]. + CACert() *x509.Certificate + // DefaultCertPool returns the default cert pool to use. DefaultCertPool() (*x509.CertPool, error) @@ -43,6 +47,11 @@ type netemTLSMITMProvider struct { cfg *netem.TLSMITMConfig } +// CACert implements TLSMITMProvider. +func (p *netemTLSMITMProvider) CACert() *x509.Certificate { + return p.cfg.Cert +} + // DefaultCertPool implements TLSMITMProvider. func (p *netemTLSMITMProvider) DefaultCertPool() (*x509.CertPool, error) { return p.cfg.CertPool() diff --git a/script/nocopyreadall.bash b/script/nocopyreadall.bash index 8180292d87..29bf5de6e4 100755 --- a/script/nocopyreadall.bash +++ b/script/nocopyreadall.bash @@ -33,6 +33,12 @@ for file in $(find . -type f -name \*.go); do continue fi + if [ "$file" = "./internal/testingx/httpproxy.go" ]; then + # We're allowed to use ReadAll and Copy in this file because + # it's code that we only use for testing purposes. + continue + fi + if [ "$file" = "./internal/testingx/httptestx.go" ]; then # We're allowed to use ReadAll and Copy in this file because # it's code that we only use for testing purposes.