feat(testingproxy): test HTTP(S) proxies using netem

internal/netxlite/httpfactory.go

@@ -1,6 +1,7 @@
 package netxlite
 
 import (
+	"crypto/tls"
 	"net/url"
 
 	oohttp "github.com/ooni/oohttp"
@@ -94,3 +95,16 @@ func HTTPTransportOptionDisableCompression(value bool) HTTPTransportOption {
 		txp.DisableCompression = value
 	}
 }
+
+// HTTPTransportOptionTLSClientConfig configures the .TLSClientConfig field,
+// which otherwise is nil, to imply using the default config.
+//
+// TODO(https://github.com/ooni/probe/issues/2536): using the default config breaks
+// tests using netem and this option is the workaround we're using to address
+// this limitation. Future releases MIGHT use a different technique and, as such,
+// we MAY remove this option when we don't need it anymore.
+func HTTPTransportOptionTLSClientConfig(config *tls.Config) HTTPTransportOption {
+	return func(txp *oohttp.Transport) {
+		txp.TLSClientConfig = config
+	}
+}

internal/testingproxy/netemhttp.go

@@ -0,0 +1,136 @@
+package testingproxy
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/netem"
+	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+// WithNetemHTTPProxyAndURL returns a [TestCase] where:
+//
+// - we fetch a URL;
+//
+// - using the github.com/ooni.netem;
+//
+// - and an HTTP proxy.
+//
+// Because this [TestCase] uses netem, it also runs in -short mode.
+func WithNetemHTTPProxyAndURL(URL string) TestCase {
+	return &netemTestCaseWithHTTP{
+		TargetURL: URL,
+	}
+}
+
+type netemTestCaseWithHTTP struct {
+	TargetURL string
+}
+
+var _ TestCase = &netemTestCaseWithHTTP{}
+
+// Name implements TestCase.
+func (tc *netemTestCaseWithHTTP) Name() string {
+	return fmt.Sprintf("fetching %s using netem and an HTTP proxy", tc.TargetURL)
+}
+
+// Run implements TestCase.
+func (tc *netemTestCaseWithHTTP) Run(t *testing.T) {
+	topology := runtimex.Try1(netem.NewStarTopology(log.Log))
+	defer topology.Close()
+
+	const (
+		wwwIPAddr    = "93.184.216.34"
+		proxyIPAddr  = "10.0.0.1"
+		clientIPAddr = "10.0.0.2"
+	)
+
+	// create:
+	//
+	// - a www stack modeling www.example.com
+	//
+	// - a proxy stack
+	//
+	// - a client stack
+	//
+	// Note that www.example.com's IP address is also the resolver used by everyone
+	wwwStack := runtimex.Try1(topology.AddHost(wwwIPAddr, wwwIPAddr, &netem.LinkConfig{}))
+	proxyStack := runtimex.Try1(topology.AddHost(proxyIPAddr, wwwIPAddr, &netem.LinkConfig{}))
+	clientStack := runtimex.Try1(topology.AddHost(clientIPAddr, wwwIPAddr, &netem.LinkConfig{}))
+
+	// configure the wwwStack as the DNS resolver with proper configuration
+	dnsConfig := netem.NewDNSConfig()
+	dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr)
+	dnsServer := runtimex.Try1(netem.NewDNSServer(log.Log, wwwStack, wwwIPAddr, dnsConfig))
+	defer dnsServer.Close()
+
+	// configure the wwwStack to respond to HTTP requests on port 80
+	wwwServer80 := testingx.MustNewHTTPServerEx(
+		&net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 80},
+		wwwStack,
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("Bonsoir, Elliot!\r\n"))
+		}),
+	)
+	defer wwwServer80.Close()
+
+	// configure the wwwStack to respond to HTTPS requests on port 443
+	wwwServer443 := testingx.MustNewHTTPServerTLSEx(
+		&net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 443},
+		wwwStack,
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("Bonsoir, Elliot!\r\n"))
+		}),
+		wwwStack,
+	)
+	defer wwwServer443.Close()
+
+	// configure the proxyStack to implement the HTTP proxy on port 8080
+	proxyServer := testingx.MustNewHTTPServerEx(
+		&net.TCPAddr{IP: net.ParseIP(proxyIPAddr), Port: 8080},
+		proxyStack,
+		testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{
+			Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: proxyStack}}),
+	)
+	defer proxyServer.Close()
+
+	// create the netx instance for the client
+	netx := &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: clientStack}}
+
+	//log.SetLevel(log.DebugLevel)
+
+	// create an HTTP client configured to use the given proxy
+	//
+	// note how we use a dialer that asserts that we're using the proxy IP address
+	// rather than the host address, so we're sure that we're using the proxy
+	dialer := &dialerWithAssertions{
+		ExpectAddress: proxyIPAddr,
+		Dialer:        netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)),
+	}
+	tlsDialer := netxlite.NewTLSDialer(dialer, netx.NewTLSHandshakerStdlib(log.Log))
+	txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer,
+		netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL))),
+
+		// TODO(https://github.com/ooni/probe/issues/2536)
+		netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{
+			RootCAs: runtimex.Try1(clientStack.DefaultCertPool()),
+		}),
+	)
+	client := &http.Client{Transport: txp}
+	defer client.CloseIdleConnections()
+
+	// get the homepage and assert we're getting a succesful response
+	httpCheckResponse(t, client, tc.TargetURL)
+}
+
+// Short implements TestCase.
+func (tc *netemTestCaseWithHTTP) Short() bool {
+	return true
+}

internal/testingproxy/netemhttps.go

@@ -0,0 +1,137 @@
+package testingproxy
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/netem"
+	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+// WithNetemHTTPWithTLSProxyAndURL returns a [TestCase] where:
+//
+// - we fetch a URL;
+//
+// - using the github.com/ooni.netem;
+//
+// - and an HTTPS proxy.
+//
+// Because this [TestCase] uses netem, it also runs in -short mode.
+func WithNetemHTTPWithTLSProxyAndURL(URL string) TestCase {
+	return &netemTestCaseWithHTTPWithTLS{
+		TargetURL: URL,
+	}
+}
+
+type netemTestCaseWithHTTPWithTLS struct {
+	TargetURL string
+}
+
+var _ TestCase = &netemTestCaseWithHTTPWithTLS{}
+
+// Name implements TestCase.
+func (tc *netemTestCaseWithHTTPWithTLS) Name() string {
+	return fmt.Sprintf("fetching %s using netem and an HTTPS proxy", tc.TargetURL)
+}
+
+// Run implements TestCase.
+func (tc *netemTestCaseWithHTTPWithTLS) Run(t *testing.T) {
+	topology := runtimex.Try1(netem.NewStarTopology(log.Log))
+	defer topology.Close()
+
+	const (
+		wwwIPAddr    = "93.184.216.34"
+		proxyIPAddr  = "10.0.0.1"
+		clientIPAddr = "10.0.0.2"
+	)
+
+	// create:
+	//
+	// - a www stack modeling www.example.com
+	//
+	// - a proxy stack
+	//
+	// - a client stack
+	//
+	// Note that www.example.com's IP address is also the resolver used by everyone
+	wwwStack := runtimex.Try1(topology.AddHost(wwwIPAddr, wwwIPAddr, &netem.LinkConfig{}))
+	proxyStack := runtimex.Try1(topology.AddHost(proxyIPAddr, wwwIPAddr, &netem.LinkConfig{}))
+	clientStack := runtimex.Try1(topology.AddHost(clientIPAddr, wwwIPAddr, &netem.LinkConfig{}))
+
+	// configure the wwwStack as the DNS resolver with proper configuration
+	dnsConfig := netem.NewDNSConfig()
+	dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr)
+	dnsServer := runtimex.Try1(netem.NewDNSServer(log.Log, wwwStack, wwwIPAddr, dnsConfig))
+	defer dnsServer.Close()
+
+	// configure the wwwStack to respond to HTTP requests on port 80
+	wwwServer80 := testingx.MustNewHTTPServerEx(
+		&net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 80},
+		wwwStack,
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("Bonsoir, Elliot!\r\n"))
+		}),
+	)
+	defer wwwServer80.Close()
+
+	// configure the wwwStack to respond to HTTPS requests on port 443
+	wwwServer443 := testingx.MustNewHTTPServerTLSEx(
+		&net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 443},
+		wwwStack,
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("Bonsoir, Elliot!\r\n"))
+		}),
+		wwwStack,
+	)
+	defer wwwServer443.Close()
+
+	// configure the proxyStack to implement the HTTP proxy on port 8080
+	proxyServer := testingx.MustNewHTTPServerTLSEx(
+		&net.TCPAddr{IP: net.ParseIP(proxyIPAddr), Port: 80443},
+		proxyStack,
+		testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{
+			Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: proxyStack}}),
+		proxyStack,
+	)
+	defer proxyServer.Close()
+
+	// crete the netx instance for the client
+	netx := &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: clientStack}}
+
+	//log.SetLevel(log.DebugLevel)
+
+	// create an HTTP client configured to use the given proxy
+	//
+	// note how we use a dialer that asserts that we're using the proxy IP address
+	// rather than the host address, so we're sure that we're using the proxy
+	dialer := &dialerWithAssertions{
+		ExpectAddress: proxyIPAddr,
+		Dialer:        netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)),
+	}
+	tlsDialer := netxlite.NewTLSDialer(dialer, netx.NewTLSHandshakerStdlib(log.Log))
+	txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer,
+		netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL))),
+
+		// TODO(https://github.com/ooni/probe/issues/2536)
+		netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{
+			RootCAs: runtimex.Try1(clientStack.DefaultCertPool()),
+		}),
+	)
+	client := &http.Client{Transport: txp}
+	defer client.CloseIdleConnections()
+
+	// get the homepage and assert we're getting a succesful response
+	httpCheckResponse(t, client, tc.TargetURL)
+}
+
+// Short implements TestCase.
+func (tc *netemTestCaseWithHTTPWithTLS) Short() bool {
+	return true
+}

internal/testingproxy/testcase.go

@@ -23,4 +23,12 @@ var AllTestCases = []TestCase{
 	// host network and HTTPS proxy
 	WithHostNetworkHTTPWithTLSProxyAndURL("http://www.example.com/"),
 	WithHostNetworkHTTPWithTLSProxyAndURL("https://www.example.com/"),
+
+	// with netem and HTTP proxy
+	WithNetemHTTPProxyAndURL("http://www.example.com/"),
+	WithNetemHTTPProxyAndURL("https://www.example.com/"),
+
+	// with netem and HTTPS proxy
+	WithNetemHTTPWithTLSProxyAndURL("http://www.example.com/"),
+	WithNetemHTTPWithTLSProxyAndURL("https://www.example.com/"),
 }

internal/testingx/httpproxy.go

@@ -67,10 +67,12 @@ func (ph *httpProxyHandler) connect(rw http.ResponseWriter, req *http.Request) {
 		rw.WriteHeader(http.StatusBadGateway)
 		return
 	}
+	defer sconn.Close()
 
 	hijacker := rw.(http.Hijacker)
 	cconn, buffered := runtimex.Try2(hijacker.Hijack())
 	runtimex.Assert(buffered.Reader.Buffered() <= 0, "data before finishing HTTP handshake")
+	defer cconn.Close()
 
 	_, _ = cconn.Write([]byte("HTTP/1.1 200 Ok\r\n\r\n"))
 
@@ -116,6 +118,7 @@ func (ph *httpProxyHandler) get(rw http.ResponseWriter, req *http.Request) {
 
 	// create HTTP client using netx
 	txp := ph.Netx.NewHTTPTransportStdlib(ph.Logger)
+	defer txp.CloseIdleConnections()
 
 	// obtain response
 	resp, err := txp.RoundTrip(req)