-
Notifications
You must be signed in to change notification settings - Fork 54
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(testingx): introduce more comprehensive HTTP(S) proxy (#1274)
We want more comprehensive testing of how we use proxies during the bootstrap. Tests should encompass both SOCKS5 and HTTP(S) proxies. Tests should support using both the host network and using netem. This diff starts paving the way for improving proxy testing, by introducing in `./internal/testingx/httpproxy.go` an HTTP(S) proxy implementation supporting both the case of HTTP over HTTP(S) proxy (where you use the host header) and HTTPS over HTTP(S) proxy (where the client issues a `CONNECT` request to the remote endpoint). The previous implementation (in `./internal/testingx/httptestx.go`) is still there, for now. We used the previous implementation, which only supported the host header, as a starting point for writing the new one. More in detail, this diff introduces the new proxy and its tests. Because testing the proxy functionality is a bit complex, I chose to use a separate package and also write tests for the tests. Obviously, we're still using `netxlite` as the underlying library, so there is some circularity in testing, but `netxlite` also has its own tests, so we should probably be fine. The separate package with tests is `./internal/testingproxy`. While working on this package, I realized the need to forward the CA used by the proxy. This happens in `./internal/testingproxy/hosthttps.go`. If we do not do this, we see the following failure: ``` === RUN TestWorkingAsIntended/fetching_https://www.example.com/_using_the_host_network_and_an_HTTPS_proxy 2023/09/15 15:28:39 debug > GET https://www.example.com/ 2023/09/15 15:28:39 debug > 2023/09/15 15:28:39 dialerWithAssertions: verified that the network is tcp as expected 2023/09/15 15:28:39 dialerWithAssertions: verified that the address is 127.0.0.1 as expected 2023/09/15 15:28:39 debug dial 127.0.0.1:61772/tcp... 2023/09/15 15:28:39 debug dial_address 127.0.0.1:61772/tcp... 2023/09/15 15:28:39 debug dial_address 127.0.0.1:61772/tcp... ok in 212.083µs 2023/09/15 15:28:39 debug dial 127.0.0.1:61772/tcp... ok in 222.208µs 2023/09/15 15:28:39 debug tls {sni=127.0.0.1 next=[]}... 2023/09/15 15:28:39 debug tls {sni=127.0.0.1 next=[]}... ssl_unknown_authority in 3.687875ms [...] ``` In other words, the TLS config we're using does not know the proxy CA. So, in this diff you also see the required work to forward the proxy CA, including the related netem changes in ooni/netem#38. The reference issue is ooni/probe#2531.
1 parent
f30f250
commit 829b1b0
Showing
15 changed files
with
618 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
package testingproxy | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"log" | ||
"net" | ||
|
||
"github.com/ooni/probe-cli/v3/internal/model" | ||
"github.com/ooni/probe-cli/v3/internal/runtimex" | ||
) | ||
|
||
// dialerWithAssertions ensures that we're dialing with the proxy address. | ||
type dialerWithAssertions struct { | ||
// ExpectAddress is the expected IP address to dial | ||
ExpectAddress string | ||
|
||
// Dialer is the underlying dialer to use | ||
Dialer model.Dialer | ||
} | ||
|
||
var _ model.Dialer = &dialerWithAssertions{} | ||
|
||
// CloseIdleConnections implements model.Dialer. | ||
func (d *dialerWithAssertions) CloseIdleConnections() { | ||
d.Dialer.CloseIdleConnections() | ||
} | ||
|
||
// DialContext implements model.Dialer. | ||
func (d *dialerWithAssertions) DialContext(ctx context.Context, network string, address string) (net.Conn, error) { | ||
// make sure the network is tcp | ||
const expectNetwork = "tcp" | ||
runtimex.Assert( | ||
network == expectNetwork, | ||
fmt.Sprintf("dialerWithAssertions: expected %s, got %s", expectNetwork, network), | ||
) | ||
log.Printf("dialerWithAssertions: verified that the network is %s as expected", expectNetwork) | ||
|
||
// make sure the IP address is the expected one | ||
ipAddr, _ := runtimex.Try2(net.SplitHostPort(address)) | ||
runtimex.Assert( | ||
ipAddr == d.ExpectAddress, | ||
fmt.Sprintf("dialerWithAssertions: expected %s, got %s", d.ExpectAddress, ipAddr), | ||
) | ||
log.Printf("dialerWithAssertions: verified that the address is %s as expected", d.ExpectAddress) | ||
|
||
// now that we're sure we're using the proxy, we can actually dial | ||
return d.Dialer.DialContext(ctx, network, address) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
// Package testingproxy contains shared test cases for the proxies. | ||
package testingproxy |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
package testingproxy | ||
|
||
import ( | ||
"fmt" | ||
"net/http" | ||
"net/url" | ||
"testing" | ||
|
||
"github.com/apex/log" | ||
"github.com/ooni/probe-cli/v3/internal/netxlite" | ||
"github.com/ooni/probe-cli/v3/internal/runtimex" | ||
"github.com/ooni/probe-cli/v3/internal/testingx" | ||
) | ||
|
||
// WithHostNetworkHTTPProxyAndURL returns a [TestCase] where: | ||
// | ||
// - we fetch a URL; | ||
// | ||
// - using the host network; | ||
// | ||
// - and an HTTP proxy. | ||
// | ||
// Because this [TestCase] uses the host network, it does not run in -short mode. | ||
func WithHostNetworkHTTPProxyAndURL(URL string) TestCase { | ||
return &hostNetworkTestCaseWithHTTP{ | ||
TargetURL: URL, | ||
} | ||
} | ||
|
||
type hostNetworkTestCaseWithHTTP struct { | ||
TargetURL string | ||
} | ||
|
||
var _ TestCase = &hostNetworkTestCaseWithHTTP{} | ||
|
||
// Name implements TestCase. | ||
func (tc *hostNetworkTestCaseWithHTTP) Name() string { | ||
return fmt.Sprintf("fetching %s using the host network and an HTTP proxy", tc.TargetURL) | ||
} | ||
|
||
// Run implements TestCase. | ||
func (tc *hostNetworkTestCaseWithHTTP) Run(t *testing.T) { | ||
// create an instance of Netx where the underlying network is nil, | ||
// which means we're using the host's network | ||
netx := &netxlite.Netx{Underlying: nil} | ||
|
||
// create the proxy server using the host network | ||
proxyServer := testingx.MustNewHTTPServer(testingx.NewHTTPProxyHandler(log.Log, netx)) | ||
defer proxyServer.Close() | ||
|
||
//log.SetLevel(log.DebugLevel) | ||
|
||
// create an HTTP client configured to use the given proxy | ||
// | ||
// note how we use a dialer that asserts that we're using the proxy IP address | ||
// rather than the host address, so we're sure that we're using the proxy | ||
dialer := &dialerWithAssertions{ | ||
ExpectAddress: "127.0.0.1", | ||
Dialer: netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)), | ||
} | ||
tlsDialer := netxlite.NewTLSDialer(dialer, netxlite.NewTLSHandshakerStdlib(log.Log)) | ||
txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, | ||
netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL)))) | ||
client := &http.Client{Transport: txp} | ||
defer client.CloseIdleConnections() | ||
|
||
// get the homepage and assert we're getting a succesful response | ||
httpCheckResponse(t, client, tc.TargetURL) | ||
} | ||
|
||
// Short implements TestCase. | ||
func (tc *hostNetworkTestCaseWithHTTP) Short() bool { | ||
return false | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,83 @@ | ||
package testingproxy | ||
|
||
import ( | ||
"crypto/tls" | ||
"fmt" | ||
"net/http" | ||
"net/url" | ||
"testing" | ||
|
||
"github.com/apex/log" | ||
"github.com/ooni/probe-cli/v3/internal/netxlite" | ||
"github.com/ooni/probe-cli/v3/internal/runtimex" | ||
"github.com/ooni/probe-cli/v3/internal/testingx" | ||
) | ||
|
||
// WithHostNetworkHTTPWithTLSProxyAndURL returns a [TestCase] where: | ||
// | ||
// - we fetch a URL; | ||
// | ||
// - using the host network; | ||
// | ||
// - and an HTTPS proxy. | ||
// | ||
// Because this [TestCase] uses the host network, it does not run in -short mode. | ||
func WithHostNetworkHTTPWithTLSProxyAndURL(URL string) TestCase { | ||
return &hostNetworkTestCaseWithHTTPWithTLS{ | ||
TargetURL: URL, | ||
} | ||
} | ||
|
||
type hostNetworkTestCaseWithHTTPWithTLS struct { | ||
TargetURL string | ||
} | ||
|
||
var _ TestCase = &hostNetworkTestCaseWithHTTPWithTLS{} | ||
|
||
// Name implements TestCase. | ||
func (tc *hostNetworkTestCaseWithHTTPWithTLS) Name() string { | ||
return fmt.Sprintf("fetching %s using the host network and an HTTPS proxy", tc.TargetURL) | ||
} | ||
|
||
// Run implements TestCase. | ||
func (tc *hostNetworkTestCaseWithHTTPWithTLS) Run(t *testing.T) { | ||
// create an instance of Netx where the underlying network is nil, | ||
// which means we're using the host's network | ||
netx := &netxlite.Netx{Underlying: nil} | ||
|
||
// create the proxy server using the host network | ||
proxyServer := testingx.MustNewHTTPServerTLS(testingx.NewHTTPProxyHandler(log.Log, netx)) | ||
defer proxyServer.Close() | ||
|
||
//log.SetLevel(log.DebugLevel) | ||
|
||
// extend the default cert pool with the proxy's own CA | ||
pool := netxlite.NewMozillaCertPool() | ||
pool.AddCert(proxyServer.CACert) | ||
tlsConfig := &tls.Config{RootCAs: pool} | ||
|
||
// create an HTTP client configured to use the given proxy | ||
// | ||
// note how we use a dialer that asserts that we're using the proxy IP address | ||
// rather than the host address, so we're sure that we're using the proxy | ||
dialer := &dialerWithAssertions{ | ||
ExpectAddress: "127.0.0.1", | ||
Dialer: netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)), | ||
} | ||
tlsDialer := netxlite.NewTLSDialerWithConfig( | ||
dialer, netxlite.NewTLSHandshakerStdlib(log.Log), | ||
tlsConfig, | ||
) | ||
txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, | ||
netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL)))) | ||
client := &http.Client{Transport: txp} | ||
defer client.CloseIdleConnections() | ||
|
||
// get the homepage and assert we're getting a succesful response | ||
httpCheckResponse(t, client, tc.TargetURL) | ||
} | ||
|
||
// Short implements TestCase. | ||
func (tc *hostNetworkTestCaseWithHTTPWithTLS) Short() bool { | ||
return false | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
package testingproxy | ||
|
||
import "net/http" | ||
|
||
type httpClient interface { | ||
Get(URL string) (*http.Response, error) | ||
} | ||
|
||
type httpClientMock struct { | ||
MockGet func(URL string) (*http.Response, error) | ||
} | ||
|
||
var _ httpClient = &httpClientMock{} | ||
|
||
// Get implements httpClient. | ||
func (c *httpClientMock) Get(URL string) (*http.Response, error) { | ||
return c.MockGet(URL) | ||
} | ||
|
||
type httpTestingT interface { | ||
Logf(format string, v ...any) | ||
Fatal(v ...any) | ||
} | ||
|
||
type httpTestingTMock struct { | ||
MockLogf func(format string, v ...any) | ||
MockFatal func(v ...any) | ||
} | ||
|
||
var _ httpTestingT = &httpTestingTMock{} | ||
|
||
// Fatal implements httpTestingT. | ||
func (t *httpTestingTMock) Fatal(v ...any) { | ||
t.MockFatal(v...) | ||
} | ||
|
||
// Logf implements httpTestingT. | ||
func (t *httpTestingTMock) Logf(format string, v ...any) { | ||
t.MockLogf(format, v...) | ||
} | ||
|
||
func httpCheckResponse(t httpTestingT, client httpClient, targetURL string) { | ||
resp, err := client.Get(targetURL) | ||
if err != nil { | ||
t.Fatal(err) | ||
} | ||
defer resp.Body.Close() | ||
t.Logf("%+v", resp) | ||
if resp.StatusCode != 200 { | ||
t.Fatal("invalid status code") | ||
} | ||
} |
Oops, something went wrong.