y

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/mitchellh/go-wordwrap v1.0.1
 	github.com/montanaflynn/stats v0.7.1
 	github.com/ooni/go-libtor v1.1.8
-	github.com/ooni/netem v0.0.0-20230915101649-ab0dc13be014
+	github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3
 	github.com/ooni/oocrypto v0.5.3
 	github.com/ooni/oohttp v0.6.3
 	github.com/ooni/probe-assets v0.18.0

go.sum

@@ -483,8 +483,8 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl
 github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
 github.com/ooni/go-libtor v1.1.8 h1:Wo3V3DVTxl5vZdxtQakqYP+DAHx7pPtAFSl1bnAa08w=
 github.com/ooni/go-libtor v1.1.8/go.mod h1:q1YyLwRD9GeMyeerVvwc0vJ2YgwDLTp2bdVcrh/JXyI=
-github.com/ooni/netem v0.0.0-20230915101649-ab0dc13be014 h1:4kOSV4D6mwrdoNUkAbGz1XoFUPcjsuNlLhZMc2CoHGg=
-github.com/ooni/netem v0.0.0-20230915101649-ab0dc13be014/go.mod h1:3LJOzTIu2O4ADDJN2ILG4ViJOqyH/u9fKY8QT2Rma8Y=
+github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3 h1:zpTbzNzpo00cKbjLLnWMKjZeGLdoNC81vMiBDiur7NU=
+github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3/go.mod h1:3LJOzTIu2O4ADDJN2ILG4ViJOqyH/u9fKY8QT2Rma8Y=
 github.com/ooni/oocrypto v0.5.3 h1:CAb0Ze6q/EWD1PRGl9KqpzMfkut4O3XMaiKYsyxrWOs=
 github.com/ooni/oocrypto v0.5.3/go.mod h1:HjEQ5pQBl6btcWgAsKKq1tFo8CfBrZu63C/vPAUGIDk=
 github.com/ooni/oohttp v0.6.3 h1:MHydpeAPU/LSDSI/hIFJwZm4afBhd2Yo+rNxxFdeMCY=

internal/enginenetx/http.go

@@ -48,13 +48,13 @@ func NewHTTPTransport(
 	resolver model.Resolver,
 ) *HTTPTransport {
 	dialer := netxlite.NewDialerWithResolver(logger, resolver)
+	dialer = netxlite.MaybeWrapWithProxyDialer(dialer, proxyURL)
 	handshaker := netxlite.NewTLSHandshakerStdlib(logger)
 	tlsDialer := netxlite.NewTLSDialer(dialer, handshaker)
-	txp := netxlite.NewHTTPTransportWithOptions(
-		logger, dialer, tlsDialer,
-		netxlite.HTTPTransportOptionDisableCompression(false),
-		netxlite.HTTPTransportOptionProxyURL(proxyURL), // nil implies "no proxy"
-	)
+	// TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport
+	// function, but we can probably avoid using it, given that this code is
+	// not using tracing and does not care about those quirks.
+	txp := netxlite.NewHTTPTransport(logger, dialer, tlsDialer)
 	txp = bytecounter.WrapHTTPTransport(txp, counter)
 	return &HTTPTransport{txp}
 }

internal/enginenetx/http_test.go

@@ -1,50 +1,33 @@
-package enginenetx_test
+package enginenetx
 
 import (
 	"testing"
 
 	"github.com/ooni/probe-cli/v3/internal/bytecounter"
-	"github.com/ooni/probe-cli/v3/internal/enginenetx"
 	"github.com/ooni/probe-cli/v3/internal/model"
-	"github.com/ooni/probe-cli/v3/internal/netemx"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
 )
 
 func TestHTTPTransport(t *testing.T) {
 
-	t.Run("the HTTPTransport is working as intended", func(t *testing.T) {
-		env := netemx.MustNewScenario(netemx.InternetScenario)
-		defer env.Close()
-
-		env.Do(func() {
-			txp := enginenetx.NewHTTPTransport(
-				bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
-			client := txp.NewHTTPClient()
-			resp, err := client.Get("https://www.example.com/")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer resp.Body.Close()
-			if resp.StatusCode != 200 {
-				t.Fatal("unexpected status code")
-			}
-		})
-	})
-
-	t.Run("we can use a socks5 proxy", func(t *testing.T) {
-		panic("not implemented")
-	})
-
-	t.Run("we can use an HTTP proxy", func(t *testing.T) {
-		panic("not implemented")
-	})
-
-	t.Run("we can use an HTTPS proxy", func(t *testing.T) {
-		panic("not implemented")
+	// TODO(bassosimone): we should replace this integration test with netemx
+	// as soon as we can sever the hard link between netxlite and this pkg
+	t.Run("is working as intended", func(t *testing.T) {
+		txp := NewHTTPTransport(
+			bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
+		client := txp.NewHTTPClient()
+		resp, err := client.Get("https://www.google.com/robots.txt")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != 200 {
+			t.Fatal("unexpected status code")
+		}
 	})
 
 	t.Run("NewHTTPClient returns a client with a cookie jar", func(t *testing.T) {
-		txp := enginenetx.NewHTTPTransport(
+		txp := NewHTTPTransport(
 			bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
 		client := txp.NewHTTPClient()
 		if client.Jar == nil {

internal/netemx/scenario.go

@@ -223,7 +223,7 @@ func MustNewScenario(config []*ScenarioDomainAddresses) *QAEnv {
 				opts = append(opts, QAEnvOptionNetStack(addr,
 					&HTTPCleartextServerFactory{
 						Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
-							return testingx.NewHTTPProxyHandler(env.Logger(), &netxlite.Netx{
+							return testingx.HTTPHandlerProxy(env.Logger(), &netxlite.Netx{
 								Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: stack}})
 						}),
 						Ports: []int{80},

internal/netxlite/httpfactory.go

@@ -1,7 +1,6 @@
 package netxlite
 
 import (
-	"crypto/tls"
 	"net/url"
 
 	oohttp "github.com/ooni/oohttp"
@@ -95,14 +94,3 @@ func HTTPTransportOptionDisableCompression(value bool) HTTPTransportOption {
 		txp.DisableCompression = value
 	}
 }
-
-// HTTPTransportOptionTLSClientConfig configures the .TLSClientConfig field, which
-// otherwise is left nil, meaning we're using the crypto/tls or ooni/ootls defaults
-// including the default cert pool. Because leaving the default .TLSClientConfig
-// has implications when dialing TLS connections over an HTTP proxy, be aware that
-// this default value could change in a future release of ooni/probe-cli.
-func HTTPTransportOptionTLSClientConfig(config *tls.Config) HTTPTransportOption {
-	return func(txp *oohttp.Transport) {
-		txp.TLSClientConfig = config
-	}
-}

internal/netxlite/tls.go

@@ -255,18 +255,18 @@ func (h *tlsHandshakerLogger) Handshake(
 	ctx context.Context, conn net.Conn, config *tls.Config,
 ) (net.Conn, tls.ConnectionState, error) {
 	h.DebugLogger.Debugf(
-		"tls_handshake {sni=%s next=%+v}...", config.ServerName, config.NextProtos)
+		"tls {sni=%s next=%+v}...", config.ServerName, config.NextProtos)
 	start := time.Now()
 	tlsconn, state, err := h.TLSHandshaker.Handshake(ctx, conn, config)
 	elapsed := time.Since(start)
 	if err != nil {
 		h.DebugLogger.Debugf(
-			"tls_handshake {sni=%s next=%+v}... %s in %s", config.ServerName,
+			"tls {sni=%s next=%+v}... %s in %s", config.ServerName,
 			config.NextProtos, err, elapsed)
 		return nil, tls.ConnectionState{}, err
 	}
 	h.DebugLogger.Debugf(
-		"tls_handshake {sni=%s next=%+v}... ok in %s {next=%s cipher=%s v=%s}",
+		"tls {sni=%s next=%+v}... ok in %s {next=%s cipher=%s v=%s}",
 		config.ServerName, config.NextProtos, elapsed, state.NegotiatedProtocol,
 		TLSCipherSuiteString(state.CipherSuite),
 		TLSVersionString(state.Version))