x

internal/enginenetx/http.go

@@ -48,13 +48,13 @@ func NewHTTPTransport(
 	resolver model.Resolver,
 ) *HTTPTransport {
 	dialer := netxlite.NewDialerWithResolver(logger, resolver)
+	dialer = netxlite.MaybeWrapWithProxyDialer(dialer, proxyURL)
 	handshaker := netxlite.NewTLSHandshakerStdlib(logger)
 	tlsDialer := netxlite.NewTLSDialer(dialer, handshaker)
-	txp := netxlite.NewHTTPTransportWithOptions(
-		logger, dialer, tlsDialer,
-		netxlite.HTTPTransportOptionDisableCompression(false),
-		netxlite.HTTPTransportOptionProxyURL(proxyURL), // nil implies "no proxy"
-	)
+	// TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport
+	// function, but we can probably avoid using it, given that this code is
+	// not using tracing and does not care about those quirks.
+	txp := netxlite.NewHTTPTransport(logger, dialer, tlsDialer)
 	txp = bytecounter.WrapHTTPTransport(txp, counter)
 	return &HTTPTransport{txp}
 }

internal/enginenetx/http_test.go

@@ -1,50 +1,33 @@
-package enginenetx_test
+package enginenetx
 
 import (
 	"testing"
 
 	"github.com/ooni/probe-cli/v3/internal/bytecounter"
-	"github.com/ooni/probe-cli/v3/internal/enginenetx"
 	"github.com/ooni/probe-cli/v3/internal/model"
-	"github.com/ooni/probe-cli/v3/internal/netemx"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
 )
 
 func TestHTTPTransport(t *testing.T) {
 
-	t.Run("the HTTPTransport is working as intended", func(t *testing.T) {
-		env := netemx.MustNewScenario(netemx.InternetScenario)
-		defer env.Close()
-
-		env.Do(func() {
-			txp := enginenetx.NewHTTPTransport(
-				bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
-			client := txp.NewHTTPClient()
-			resp, err := client.Get("https://www.example.com/")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer resp.Body.Close()
-			if resp.StatusCode != 200 {
-				t.Fatal("unexpected status code")
-			}
-		})
-	})
-
-	t.Run("we can use a socks5 proxy", func(t *testing.T) {
-		panic("not implemented")
-	})
-
-	t.Run("we can use an HTTP proxy", func(t *testing.T) {
-		panic("not implemented")
-	})
-
-	t.Run("we can use an HTTPS proxy", func(t *testing.T) {
-		panic("not implemented")
+	// TODO(bassosimone): we should replace this integration test with netemx
+	// as soon as we can sever the hard link between netxlite and this pkg
+	t.Run("is working as intended", func(t *testing.T) {
+		txp := NewHTTPTransport(
+			bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
+		client := txp.NewHTTPClient()
+		resp, err := client.Get("https://www.google.com/robots.txt")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != 200 {
+			t.Fatal("unexpected status code")
+		}
 	})
 
 	t.Run("NewHTTPClient returns a client with a cookie jar", func(t *testing.T) {
-		txp := enginenetx.NewHTTPTransport(
+		txp := NewHTTPTransport(
 			bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
 		client := txp.NewHTTPClient()
 		if client.Jar == nil {

internal/netemx/scenario.go

@@ -223,7 +223,7 @@ func MustNewScenario(config []*ScenarioDomainAddresses) *QAEnv {
 				opts = append(opts, QAEnvOptionNetStack(addr,
 					&HTTPCleartextServerFactory{
 						Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
-							return testingx.NewHTTPProxyHandler(env.Logger(), &netxlite.Netx{
+							return testingx.HTTPHandlerProxy(env.Logger(), &netxlite.Netx{
 								Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: stack}})
 						}),
 						Ports: []int{80},

internal/netxlite/tls.go

@@ -255,18 +255,18 @@ func (h *tlsHandshakerLogger) Handshake(
 	ctx context.Context, conn net.Conn, config *tls.Config,
 ) (net.Conn, tls.ConnectionState, error) {
 	h.DebugLogger.Debugf(
-		"tls_handshake {sni=%s next=%+v}...", config.ServerName, config.NextProtos)
+		"tls {sni=%s next=%+v}...", config.ServerName, config.NextProtos)
 	start := time.Now()
 	tlsconn, state, err := h.TLSHandshaker.Handshake(ctx, conn, config)
 	elapsed := time.Since(start)
 	if err != nil {
 		h.DebugLogger.Debugf(
-			"tls_handshake {sni=%s next=%+v}... %s in %s", config.ServerName,
+			"tls {sni=%s next=%+v}... %s in %s", config.ServerName,
 			config.NextProtos, err, elapsed)
 		return nil, tls.ConnectionState{}, err
 	}
 	h.DebugLogger.Debugf(
-		"tls_handshake {sni=%s next=%+v}... ok in %s {next=%s cipher=%s v=%s}",
+		"tls {sni=%s next=%+v}... ok in %s {next=%s cipher=%s v=%s}",
 		config.ServerName, config.NextProtos, elapsed, state.NegotiatedProtocol,
 		TLSCipherSuiteString(state.CipherSuite),
 		TLSVersionString(state.Version))

internal/testingx/httptestx.go

@@ -3,10 +3,12 @@ package testingx
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"io"
 	"net"
 	"net/http"
 	"net/url"
 
+	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/ooni/probe-cli/v3/internal/optional"
 	"github.com/ooni/probe-cli/v3/internal/runtimex"
 )
@@ -174,3 +176,70 @@ func httpHandlerHijack(w http.ResponseWriter, r *http.Request, policy string) {
 		// nothing
 	}
 }
+
+// TODO(bassosimone): eventually we may want to have a model type
+// that models the equivalent of [netxlite.Netx].
+
+// HTTPHandlerProxyNetx is [netxlite.Netx] as seen by [HTTPHandlerProxy].
+type HTTPHandlerProxyNetx interface {
+	NewHTTPTransportStdlib(logger model.DebugLogger) model.HTTPTransport
+}
+
+// HTTPHandlerProxy is a handler implementing an HTTP proxy using the host header
+// to determine who to connect to. We additionally use the via header to avoid sending
+// requests to ourself. Please, note that we designed this proxy ONLY to be used for
+// testing purposes and that it's rather simplistic.
+func HTTPHandlerProxy(logger model.Logger, netx HTTPHandlerProxyNetx) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		// reject requests that already visited the proxy and requests we cannot route
+		if req.Host == "" || req.Header.Get("Via") != "" {
+			rw.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		// be explicit about not supporting request bodies
+		if req.Method != http.MethodGet {
+			rw.WriteHeader(http.StatusNotImplemented)
+			return
+		}
+
+		// clone the request before modifying it
+		req = req.Clone(req.Context())
+
+		// include proxy header to prevent sending requests to ourself
+		req.Header.Add("Via", "testingx/0.1.0")
+
+		// fix: "http: Request.RequestURI can't be set in client requests"
+		req.RequestURI = ""
+
+		// fix: `http: unsupported protocol scheme ""`
+		req.URL.Host = req.Host
+
+		// fix: "http: no Host in request URL"
+		req.URL.Scheme = "http"
+
+		logger.Debugf("PROXY: sending request: %s", req)
+
+		// create HTTP client using netx
+		txp := netx.NewHTTPTransportStdlib(logger)
+
+		// obtain response
+		resp, err := txp.RoundTrip(req)
+		if err != nil {
+			logger.Warnf("PROXY: request failed: %s", err.Error())
+			rw.WriteHeader(http.StatusBadGateway)
+			return
+		}
+
+		// write response
+		rw.WriteHeader(resp.StatusCode)
+		for key, values := range resp.Header {
+			for _, value := range values {
+				rw.Header().Add(key, value)
+			}
+		}
+
+		// write response body
+		_, _ = io.Copy(rw, resp.Body)
+	})
+}