Skip to content

Commit

Permalink
feat(enginenetx): support HTTP and HTTPS proxies (#1282)
Browse files Browse the repository at this point in the history 
This diff completes the work we have been doing for a few days now and
provides HTTP and HTTPS proxy support, in addition to SOCKS5 support,
for the engine-specific network.

We did this work in the context of
ooni/probe#2531 and
ooni/probe#1955.

BTW, the fact that tests used `measurexlite` and tracing is very nice
here. It means the idea to write `measurexlite` based on context and
tracing was good and could be used beyond its original design goals.
  • Loading branch information
@bassosimone
bassosimone authored Sep 19, 2023
1 parent d0ea69d commit 6ef3fba
Show file tree
Hide file tree
Showing 4 changed files with 259 additions and 28 deletions.
10 changes: 5 additions & 5 deletions internal/enginenetx/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@ func NewHTTPTransport(
resolver model.Resolver,
) *HTTPTransport {
dialer := netxlite.NewDialerWithResolver(logger, resolver)
dialer = netxlite.MaybeWrapWithProxyDialer(dialer, proxyURL)
handshaker := netxlite.NewTLSHandshakerStdlib(logger)
tlsDialer := netxlite.NewTLSDialer(dialer, handshaker)
// TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport
// function, but we can probably avoid using it, given that this code is
// not using tracing and does not care about those quirks.
txp := netxlite.NewHTTPTransport(logger, dialer, tlsDialer)
txp := netxlite.NewHTTPTransportWithOptions(
logger, dialer, tlsDialer,
netxlite.HTTPTransportOptionDisableCompression(false),
netxlite.HTTPTransportOptionProxyURL(proxyURL), // nil implies "no proxy"
)
txp = bytecounter.WrapHTTPTransport(txp, counter)
return &HTTPTransport{txp}
}
265 changes: 247 additions & 18 deletions internal/enginenetx/http_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,263 @@
package enginenetx
package enginenetx_test

import (
"context"
"net"
"net/http"
"net/url"
"testing"
"time"

"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/bytecounter"
"github.com/ooni/probe-cli/v3/internal/enginenetx"
"github.com/ooni/probe-cli/v3/internal/measurexlite"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netemx"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/testingsocks5"
"github.com/ooni/probe-cli/v3/internal/testingx"
)

func TestHTTPTransport(t *testing.T) {
func TestHTTPTransportWAI(t *testing.T) {
t.Run("is WAI when not using any proxy", func(t *testing.T) {
env := netemx.MustNewScenario(netemx.InternetScenario)
defer env.Close()

// TODO(bassosimone): we should replace this integration test with netemx
// as soon as we can sever the hard link between netxlite and this pkg
t.Run("is working as intended", func(t *testing.T) {
txp := NewHTTPTransport(
bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
client := txp.NewHTTPClient()
resp, err := client.Get("https://www.google.com/robots.txt")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatal("unexpected status code")
}
env.Do(func() {
txp := enginenetx.NewHTTPTransport(
bytecounter.New(),
model.DiscardLogger,
nil,
netxlite.NewStdlibResolver(model.DiscardLogger),
)
client := txp.NewHTTPClient()
resp, err := client.Get("https://www.example.com/")
if err != nil {
t.Fatal(err)
}
t.Logf("%+v", resp)
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatal("unexpected status code")
}
})
})

t.Run("is WAI when using a SOCKS5 proxy", func(t *testing.T) {
// create internet measurement scenario
env := netemx.MustNewScenario(netemx.InternetScenario)
defer env.Close()

// create a proxy using the client's TCP/IP stack
proxy := testingsocks5.MustNewServer(
log.Log,
&netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}},
&net.TCPAddr{
IP: net.ParseIP(env.ClientStack.IPAddress()),
Port: 9050,
},
)
defer proxy.Close()

env.Do(func() {
txp := enginenetx.NewHTTPTransport(
bytecounter.New(),
model.DiscardLogger,
&url.URL{
Scheme: "socks5",
Host: net.JoinHostPort(env.ClientStack.IPAddress(), "9050"),
Path: "/",
},
netxlite.NewStdlibResolver(model.DiscardLogger),
)
client := txp.NewHTTPClient()

// To make sure we're connecting to the expected endpoint, we're going to use
// measurexlite and tracing to observe the destination endpoints
trace := measurexlite.NewTrace(0, time.Now())
ctx := netxlite.ContextWithTrace(context.Background(), trace)

// create request using the above context
//
// Implementation note: we cannot use HTTPS with netem here as explained
// by the https://github.com/ooni/probe/issues/2536 issue.
req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil)
if err != nil {
t.Fatal(err)
}

resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
t.Logf("%+v", resp)
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatal("unexpected status code")
}

// make sure that we only connected to the SOCKS5 proxy
tcpConnects := trace.TCPConnects()
if len(tcpConnects) <= 0 {
t.Fatal("expected at least one TCP connect")
}
for idx, entry := range tcpConnects {
t.Logf("%d: %+v", idx, entry)
if entry.IP != env.ClientStack.IPAddress() {
t.Fatal("unexpected IP address")
}
if entry.Port != 9050 {
t.Fatal("unexpected port")
}
}
})
})

t.Run("is WAI when using an HTTP proxy", func(t *testing.T) {
// create internet measurement scenario
env := netemx.MustNewScenario(netemx.InternetScenario)
defer env.Close()

// create a proxy using the client's TCP/IP stack
proxy := testingx.MustNewHTTPServerEx(
&net.TCPAddr{IP: net.ParseIP(env.ClientStack.IPAddress()), Port: 8080},
env.ClientStack,
testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{
Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}),
)
defer proxy.Close()

env.Do(func() {
txp := enginenetx.NewHTTPTransport(
bytecounter.New(),
model.DiscardLogger,
&url.URL{
Scheme: "http",
Host: net.JoinHostPort(env.ClientStack.IPAddress(), "8080"),
Path: "/",
},
netxlite.NewStdlibResolver(model.DiscardLogger),
)
client := txp.NewHTTPClient()

// To make sure we're connecting to the expected endpoint, we're going to use
// measurexlite and tracing to observe the destination endpoints
trace := measurexlite.NewTrace(0, time.Now())
ctx := netxlite.ContextWithTrace(context.Background(), trace)

// create request using the above context
//
// Implementation note: we cannot use HTTPS with netem here as explained
// by the https://github.com/ooni/probe/issues/2536 issue.
req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil)
if err != nil {
t.Fatal(err)
}

resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
t.Logf("%+v", resp)
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatal("unexpected status code")
}

// make sure that we only connected to the HTTP proxy
tcpConnects := trace.TCPConnects()
if len(tcpConnects) <= 0 {
t.Fatal("expected at least one TCP connect")
}
for idx, entry := range tcpConnects {
t.Logf("%d: %+v", idx, entry)
if entry.IP != env.ClientStack.IPAddress() {
t.Fatal("unexpected IP address")
}
if entry.Port != 8080 {
t.Fatal("unexpected port")
}
}
})
})

t.Run("is WAI when using an HTTPS proxy", func(t *testing.T) {
// create internet measurement scenario
env := netemx.MustNewScenario(netemx.InternetScenario)
defer env.Close()

// create a proxy using the client's TCP/IP stack
proxy := testingx.MustNewHTTPServerTLSEx(
&net.TCPAddr{IP: net.ParseIP(env.ClientStack.IPAddress()), Port: 4443},
env.ClientStack,
testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{
Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}),
env.ClientStack,
)
defer proxy.Close()

env.Do(func() {
txp := enginenetx.NewHTTPTransport(
bytecounter.New(),
model.DiscardLogger,
&url.URL{
Scheme: "https",
Host: net.JoinHostPort(env.ClientStack.IPAddress(), "4443"),
Path: "/",
},
netxlite.NewStdlibResolver(model.DiscardLogger),
)
client := txp.NewHTTPClient()

// To make sure we're connecting to the expected endpoint, we're going to use
// measurexlite and tracing to observe the destination endpoints
trace := measurexlite.NewTrace(0, time.Now())
ctx := netxlite.ContextWithTrace(context.Background(), trace)

// create request using the above context
//
// Implementation note: we cannot use HTTPS with netem here as explained
// by the https://github.com/ooni/probe/issues/2536 issue.
req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil)
if err != nil {
t.Fatal(err)
}

resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
t.Logf("%+v", resp)
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatal("unexpected status code")
}

// make sure that we only connected to the HTTPS proxy
tcpConnects := trace.TCPConnects()
if len(tcpConnects) <= 0 {
t.Fatal("expected at least one TCP connect")
}
for idx, entry := range tcpConnects {
t.Logf("%d: %+v", idx, entry)
if entry.IP != env.ClientStack.IPAddress() {
t.Fatal("unexpected IP address")
}
if entry.Port != 4443 {
t.Fatal("unexpected port")
}
}
})
})

t.Run("NewHTTPClient returns a client with a cookie jar", func(t *testing.T) {
txp := NewHTTPTransport(
bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
txp := enginenetx.NewHTTPTransport(
bytecounter.New(),
model.DiscardLogger,
nil,
netxlite.NewStdlibResolver(model.DiscardLogger),
)
client := txp.NewHTTPClient()
if client.Jar == nil {
t.Fatal("expected non-nil cookie jar")
Expand Down
10 changes: 5 additions & 5 deletions internal/netemx/qaenv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,8 @@ type QAEnv struct {
// clientNICWrapper is the OPTIONAL wrapper for the client NIC.
clientNICWrapper netem.LinkNICWrapper

// clientStack is the client stack to use.
clientStack *netem.UNetStack
// ClientStack is the client stack to use.
ClientStack *netem.UNetStack

// closables contains all entities where we have to take care of closing.
closables []io.Closer
Expand Down Expand Up @@ -197,7 +197,7 @@ func MustNewQAEnv(options ...QAEnvOption) *QAEnv {
env := &QAEnv{
baseLogger: config.logger,
clientNICWrapper: config.clientNICWrapper,
clientStack: nil,
ClientStack: nil,
closables: []io.Closer{},
emulateAndroidGetaddrinfo: &atomic.Bool{},
ispResolverConfig: netem.NewDNSConfig(),
Expand All @@ -208,7 +208,7 @@ func MustNewQAEnv(options ...QAEnvOption) *QAEnv {
}

// create all the required internals
env.clientStack = env.mustNewClientStack(config)
env.ClientStack = env.mustNewClientStack(config)
env.closables = append(env.closables, env.mustNewNetStacks(config)...)

return env
Expand Down Expand Up @@ -306,7 +306,7 @@ func (env *QAEnv) EmulateAndroidGetaddrinfo(value bool) {
// Do executes the given function such that [netxlite] code uses the
// underlying clientStack rather than ordinary networking code.
func (env *QAEnv) Do(function func()) {
var stack netem.UnderlyingNetwork = env.clientStack
var stack netem.UnderlyingNetwork = env.ClientStack
if env.emulateAndroidGetaddrinfo.Load() {
stack = &androidStack{stack}
}
Expand Down
2 changes: 2 additions & 0 deletions internal/netxlite/maybeproxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ type proxyDialer struct {

// MaybeWrapWithProxyDialer returns the original dialer if the proxyURL is nil
// and otherwise returns a wrapped dialer that implements proxying.
//
// Deprecated: do not use this function in new code.
func MaybeWrapWithProxyDialer(dialer model.Dialer, proxyURL *url.URL) model.Dialer {
if proxyURL == nil {
return dialer
Expand Down

0 comments on commit 6ef3fba

Please sign in to comment.