diff --git a/internal/erroror/erroror.go b/internal/erroror/erroror.go
new file mode 100644
index 000000000..833272ee8
--- /dev/null
+++ b/internal/erroror/erroror.go
@@ -0,0 +1,8 @@
+// Package erroror contains code to represent an error or a value.
+package erroror
+
+// Value represents an error or a value.
+type Value[Type any] struct {
+ Err error
+ Value Type
+}
diff --git a/internal/httpclientx/DESIGN.md b/internal/httpclientx/DESIGN.md
new file mode 100644
index 000000000..af22ad1fe
--- /dev/null
+++ b/internal/httpclientx/DESIGN.md
@@ -0,0 +1,418 @@
+# ./internal/httpclientx
+
+This package aims to replace previously existing packages for interact with
+backend services controlled either by us or by third parties.
+
+As of 2024-04-22, these packages are:
+
+* `./internal/httpx`, which is currently deprecated;
+
+* `./internal/httpapi`, which aims to automatically generate swaggers from input
+and output messages and implements support for falling back.
+
+The rest of this document explains the requirements and describes the design.
+
+## Table of Contents
+
+- [Requirements](#requirements)
+- [Design](#design)
+ - [Overlapped Operations](#overlapped-operations)
+ - [Extensibility](#extensibility)
+ - [Functionality Comparison](#functionality-comparison)
+ - [GetJSON](#getjson)
+ - [GetRaw](#getraw)
+ - [GetXML](#getxml)
+ - [PostJSON](#postjson)
+- [Nil Safety](#nil-safety)
+- [Refactoring Plan](#refactoring-plan)
+- [Limitations and Future Work](#limitations-and-future-work)
+
+## Requirements
+
+We want this new package to:
+
+1. Implement common access patterns (GET with JSON body, GET with XML body, GET with
+raw body, and POST with JSON request and response bodies).
+
+2. Optionally log request and response bodies.
+
+3. Factor common code for accessing such services.
+
+4. Take advantage of Go generics to automatically marshal and unmarshal without
+having to write specific functions for each request/response pair.
+
+5. Support some kind of fallback policy like `httpapi` because we used this
+for test helpers and, while, as of today, we're mapping serveral TH domain names
+to a single IP address, it might still be useful to fallback.
+
+6. Provide an easy way for the caller to know whether an HTTP request failed
+and, if so, with which status code, which is needed to intercept `401` responses
+to take the appropriate logging-in actions.
+
+7. Make the design extensible such that re-adding unused functionality (such
+as cloud fronting) does not require us to refactor the code much.
+
+8. Functional equivalent with existing packages (modulo the existing
+functionality that is not relevant anymore).
+
+Non goals:
+
+1. automatically generate swaggers from the Go representation of API invocation
+for these reasons: (1) the OONI backend swagger is only for documentational
+purposes and it is not always in sync with reality; (2) by doing that, we obtained
+overly complex code, which hampers maintenance.
+
+2. implementing algorithms such as logging in to the OONI backend and requesting
+tokens, which should be the responsibility of another package.
+
+3. implementing cloud fronting, which is not used anymore.
+
+## Design
+
+This package supports the following operations:
+
+```Go
+type Config struct {
+ Client model.HTTPClient
+ Logger model.Logger
+ UserAgent string
+}
+
+func GetJSON[Output any](ctx context.Context, URL string, config *Config) (Output, error)
+
+func GetRaw(ctx context.Context, URL string, config *Config) ([]byte, error)
+
+func GetXML[Output any](ctx context.Context, URL string, config *Config) (Output, error)
+
+func PostJSON[Input, Output any](ctx context.Context, URL string, input Input, config *Config) (Output, error)
+```
+
+(The `*Config` is the last argument because it is handy to create it inline when calling
+and having it last reduces readability the least.)
+
+These operations implement all the actions listed in the first requirement.
+
+The `Config` struct allows to add new optional fields to implement new functionality without
+changing the API and with minimal code refactoring efforts (seventh requirement).
+
+We're using generics to automate marshaling and umarshaling (requirement four).
+
+The internal implementation is such that we reuse code to avoid code duplication,
+thus addressing also requirement three.
+
+Additionally, whenever a call fails with a non-200 status code, the return
+value can be converted to the following type using `errors.As`:
+
+```Go
+type ErrRequestFailed struct {
+ StatusCode int
+}
+
+func (err *ErrRequestFailed) Error() string
+```
+
+Therefore, we have a way to know why a request failed (requirement six).
+
+To avoid logging bodies, one just needs to pass `model.DiscardLogger` as the
+`logger` (thus fulfilling requirement two).
+
+### Overlapped Operations
+
+The code at `./internal/httpapi` performs sequential function calls. This design
+does not interact well with the `enginenetx` package and its dial tactics. A better
+strategy is to allow calls to be overlapped. This means that, if the `enginenetx`
+is busy trying tactics for a given API endpoint, we eventually try to use the
+subsequent (semantically-equivalent) endpoint after a given time, without waiting
+for the first endpoint to complete.
+
+We allow for overlapped operations by defining these constructors:
+
+```Go
+func NewOverlappedGetJSON[Output any](config *Config) *Overlapped[Output]
+
+func NewOverlappedGetRaw(config *Config) *Overlapped[[]byte]
+
+func NewOverlappedGetXML[Output any](config *Config) *Overlapped[Output]
+
+func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Overlapped[Output]
+```
+
+They all construct the same `*Overlapped` struct, which looks like this:
+
+```Go
+type Overlapped[Output any] struct {
+ RunFunc func(ctx context.Context, URL string) (Output, error)
+
+ ScheduleInterval time.Duration
+}
+```
+
+The constructor configures `RunFunc` to invoke the call corresponding to the construct
+name (i.e., `NewOverlappedGetXML` configures `RunFunc` to run `GetXML`).
+
+Then, we define the following method:
+
+```Go
+func (ovx *Overlapped[Output]) Run(ctx context.Context, URLs ...string) (Output, error)
+```
+
+This method starts N goroutines to issue the API calls with each URL. (A classic example is for
+the URLs to be `https://0.th.ooni.org/`, `https://1.th.ooni.org/` and so on.)
+
+By default, `ScheduleInterval` is 15 seconds. If the first URL does not provide a result
+within 15 seconds, we try the second one. That is, every 15 seconds, we will attempt using
+another URL, until there's a successful response or we run out of URLs.
+
+As soon as we have a successful response, we cancel all the other pending operations
+that may exist. Once all operations have terminated, we return to the caller.
+
+### Extensibility
+
+We use the `Config` object to package common settings. Thus adding a new field (e.g., a
+custom `Host` header to implement cloud fronting), only means the following:
+
+1. Adding a new OPTIONAL field to `Config`.
+
+2. Honoring this field inside the internal implementation.
+
+_Et voilà_, this should allow for minimal efforts API upgrades.
+
+### Functionality Comparison
+
+This section compares side-by-side the operations performed by each implementation
+as of [probe-cli@7dab5a29812](https://github.com/ooni/probe-cli/tree/7dab5a29812) to
+show that they implement ~equivalent functionality. This should be the case, since
+the `httpxclientx` package is a refactoring of `httpapi`, which in turn contains code
+originally derived from `httpx`. Anyways, better to double check.
+
+#### GetJSON
+
+We compare to `httpapi.Call` and `httpx.GetJSONWithQuery`.
+
+| Operation | GetJSON | httpapi | httpx |
+| ------------------------- | ------- | ------- | ----- |
+| enforce a call timeout | NO | yes | NO |
+| parse base URL | NO | yes | yes |
+| join path and base URL | NO | yes | yes |
+| append query to URL | NO | yes | yes |
+| NewRequestWithContext | yes️ | yes | yes |
+| handle cloud front | NO | yes | yes |
+| set Authorization | yes | yes | yes |
+| set Accept | NO | yes | yes |
+| set User-Agent | yes ️ | yes | yes |
+| set Accept-Encoding gzip | yes️ | yes | NO |
+| (HTTPClient).Do() | yes | yes | yes |
+| defer resp.Body.Close() | yes | yes | yes |
+| handle gzip encoding | yes | yes | NO |
+| limit io.Reader | yes | yes | yes |
+| netxlite.ReadAllContext() | yes | yes | yes |
+| handle truncated body | NO | yes | NO |
+| log response body | yes | yes | yes |
+| handle non-200 response | ️ yes | yes* | yes* |
+| unmarshal JSON | yes | yes | yes |
+
+The `yes*` means that `httpapi` rejects responses with status codes `>= 400` (like cURL)
+while the new package only accepts status codes `== 200`. This difference should be of little
+practical significance for all the APIs we invoke and the new behavior is stricter.
+
+Regarding all the other cases for which `GetJSON` is marked as "NO":
+
+1. Enforcing a call timeout is better done just through the context like `httpx` does.
+
+2. `GetJSON` lets the caller completely manage the construction of the URL, so we do not need
+code to join together a base URL, possibly including a base path, a path, and a query (and we're
+introducing the new `./internal/urlx` package to handle this situation).
+
+3. `GetJSON` does not handle cloud fronting because we don't use it. The design where the
+`Config` contains mandatory and optional fields would still allow doing that easily.
+
+4. Setting the `Accept` header does not seem to matter in out context because we mostly
+call API for which there's no need for content negotiation.
+
+5. It's difficult to say whether a body size was exactly the amount specified for truncation
+or the body has been truncated. While this is a corner case, it seems perhaps wiser to let
+the caller try parsing the body and failing if it is indeed truncated.
+
+#### GetRaw
+
+Here we're comparing to `httpapi.Call` and `httpx.FetchResource`.
+
+| Operation | GetRaw | httpapi | httpx |
+| ------------------------- | ------- | ------- | ----- |
+| enforce a call timeout | NO | yes | NO |
+| parse base URL | NO | yes | yes |
+| join path and base URL | NO | yes | yes |
+| append query to URL | NO | yes | yes |
+| NewRequestWithContext | yes️ | yes | yes |
+| handle cloud front | NO | yes | yes |
+| set Authorization | yes | yes | yes |
+| set Accept | NO | yes | yes |
+| set User-Agent | yes ️ | yes | yes |
+| set Accept-Encoding gzip | yes️ | yes | NO |
+| (HTTPClient).Do() | yes | yes | yes |
+| defer resp.Body.Close() | yes | yes | yes |
+| handle gzip encoding | yes | yes | NO |
+| limit io.Reader | yes | yes | yes |
+| netxlite.ReadAllContext() | yes | yes | yes |
+| handle truncated body | NO | yes | NO |
+| log response body | yes | yes | yes |
+| handle non-200 response | ️ yes | yes* | yes |
+
+Here we can basically make equivalent remarks as those of the previous section.
+
+#### GetXML
+
+There's no direct equivalent of `GetXML` in `httpapi` and `httpx`. Therefore, when using these
+two APIs, the caller would need to fetch a raw body and then manually parse XML.
+
+| Operation | GetXML | httpapi | httpx |
+| ------------------------- | ------- | ------- | ----- |
+| enforce a call timeout | NO | N/A | N/A |
+| parse base URL | NO | N/A | N/A |
+| join path and base URL | NO | N/A | N/A |
+| append query to URL | NO | N/A | N/A |
+| NewRequestWithContext | yes️ | N/A | N/A |
+| handle cloud front | NO | N/A | N/A |
+| set Authorization | yes | N/A | N/A |
+| set Accept | NO | N/A | N/A |
+| set User-Agent | yes ️ | N/A | N/A |
+| set Accept-Encoding gzip | yes️ | N/A | N/A |
+| (HTTPClient).Do() | yes | N/A | N/A |
+| defer resp.Body.Close() | yes | N/A | N/A |
+| handle gzip encoding | yes | N/A | N/A |
+| limit io.Reader | yes | N/A | N/A |
+| netxlite.ReadAllContext() | yes | N/A | N/A |
+| handle truncated body | NO | N/A | N/A |
+| log response body | yes | N/A | N/A |
+| handle non-200 response | ️ yes | N/A | N/A |
+| unmarshal XML | yes | N/A | N/A |
+
+Because comparison is not possible, there is not much else to say.
+
+#### PostJSON
+
+Here we're comparing to `httpapi.Call` and `httpx.PostJSON`.
+
+| Operation | PostJSON | httpapi | httpx |
+| ------------------------- | -------- | ------- | ----- |
+| marshal JSON | yes | yes~ | yes |
+| log request body | yes | yes | yes |
+| enforce a call timeout | NO | yes | NO |
+| parse base URL | NO | yes | yes |
+| join path and base URL | NO | yes | yes |
+| append query to URL | NO | yes | yes |
+| NewRequestWithContext | yes️ | yes | yes |
+| handle cloud front | NO | yes | yes |
+| set Authorization | yes | yes | yes |
+| set Accept | NO | yes | yes |
+| set Content-Type | yes | yes | yes |
+| set User-Agent | yes ️ | yes | yes |
+| set Accept-Encoding gzip | yes️ | yes | NO |
+| (HTTPClient).Do() | yes | yes | yes |
+| defer resp.Body.Close() | yes | yes | yes |
+| handle gzip encoding | yes | yes | NO |
+| limit io.Reader | yes | yes | yes |
+| netxlite.ReadAllContext() | yes | yes | yes |
+| handle truncated body | NO | yes | NO |
+| log response body | yes | yes | yes |
+| handle non-200 response | ️ yes | yes* | yes* |
+| unmarshal JSON | yes | yes | yes |
+
+The `yes*` means that `httpapi` rejects responses with status codes `>= 400` (like cURL)
+while the new package only accepts status codes `== 200`. This difference should be of little
+practical significance for all the APIs we invoke and the new behavior is stricter.
+
+The `yes~` means that `httpapi` already receives a marshaled body from a higher-level API
+that is part of the same package, while in this package we marshal in `PostJSON`.
+
+## Nil Safety
+
+Consider the following code snippet:
+
+```Go
+resp, err := httpclientx.GetJSON[*APIResponse](ctx, URL, config)
+runtimex.Assert((resp == nil && err != nil) || (resp != nil && err == nil), "ouch")
+```
+
+Now, consider the case where `URL` refers to a server that returns `null` as the JSON
+answer, rather than returning a JSON object. The `encoding/json` package will accept the
+`null` value and unmarshal it into a `nil` pointer. So, `GetJSON` will return `nil` and
+`nil`, and the `runtimex.Assert` will fail.
+
+The `httpx` package did not have this issue because the usage pattern was:
+
+```Go
+var resp APIResponse
+err := apiClient.GetJSON(ctx, "/foobar", &resp) // where apiClient implements httpx.APIClient
+```
+
+In such a case, the `null` would have no effect and `resp` would be an empty response.
+
+However, it is still handy to return a value and an error, and it is the most commonly used
+pattern in Go and, as a result, in OONI Probe. So, what do we do?
+
+Well, here's the strategy:
+
+1. When sending pointers, slices, or maps in `PostJSON`, we return `ErrIsNil` if the pointer,
+slice, or map is `nil`, to avoid sending literal `null` to servers.
+
+2. `GetJSON`, `GetXML`, and `PostJSON` include checks after unmarshaling so that, if the API response
+type is a slice, pointer, or map, and it is `nil`, we also return `ErrIsNil`.
+
+Strictly speaking, it is still unclear to us whether this could happen with `GetXML` but we have
+decided to implements these checks for `GetXML` as well, just in case.
+
+## Refactoring Plan
+
+The overall goal is to replace usages of `httpapi` and `httpx` with usages of `httpclient`.
+
+The following packages use `httpapi`:
+
+1. `internal/experiment/webconnectivity`: uses `httpapi.SeqCaller` to chain calls to all
+the available test helpers, which we can replace with using `*Overlapped`;
+
+2. `internal/experiment/webconnectivitylte`: same as above;
+
+3. `internal/ooapi`: uses `httpapi` to define the OONI backend APIs for the purpose of
+generating and cross-validating swaggers, which is something we defined as a non-goal given
+that we never really managed to do it reliably, and it has only led to code complexity;
+
+4. `internal/probeservices`: uses `httpapi` to implement check-in and the main reason why
+this is the case is because it supports `"gzip"` encoding;
+
+The following packages use `httpx`:
+
+1. `internal/cmd/apitool`: this is just a byproduct of `probeservices.Client` embedding
+the `httpx.APIClientTemplate`, so this should really be easy to get rid of;
+
+2. `internal/enginelocate`: we're using `httpx` convenience functions to figure out
+the probe IP and we can easily replace these calls with `httpclientx`;
+
+3. `internal/oonirun`: uses `httpx` to fetch descriptors and can be easily replaced;
+
+4. `internal/probeservices`: uses `httpx` for most other calls.
+
+Based on the above information, it seems the easiest way to proceed is this:
+
+1. `internal/enginelocate`: replace `httpx` with `httpclientx`;
+
+2. `internal/oonirun`: replace `httpx` with `httpclientx`;
+
+3. `internal/probeservices`: replace the check-in implementation to use `httpclientx`
+instead of using the `httpapi` package;
+
+4. `internal/experiment/webconnectivity{,lte}`: replace the `httpapi.SeqCaller` usage
+with invocations of `*Overlapped`;
+
+5. remove the `httpapi` and `ooapi` packages, now unused;
+
+6. finish replacing `httpx` with `httpclientx` in `internal/probeservices`
+
+7. remove the `httpx` package.
+
+## Limitations and Future Work
+
+The current implementation of `*Overlapped` may cause us to do more work than needed in
+case the network is really slow and an attempt is slowly fetching the body. In such a case,
+starting a new attempt duplicates work. Handling this case does not seem straightforward
+currently, therefore, we will focus on this as part of future work.
diff --git a/internal/httpclientx/config.go b/internal/httpclientx/config.go
new file mode 100644
index 000000000..71bdeb616
--- /dev/null
+++ b/internal/httpclientx/config.go
@@ -0,0 +1,20 @@
+package httpclientx
+
+import "github.com/ooni/probe-cli/v3/internal/model"
+
+// Config contains configuration shared by [GetJSON], [GetXML], [GetRaw], and [PostJSON].
+//
+// The zero value is invalid; initialize the MANDATORY fields.
+type Config struct {
+ // Authorization contains the OPTIONAL Authorization header value to use.
+ Authorization string
+
+ // Client is the MANDATORY [model.HTTPClient] to use.
+ Client model.HTTPClient
+
+ // Logger is the MANDATORY [model.Logger] to use.
+ Logger model.Logger
+
+ // UserAgent is the MANDATORY User-Agent header value to use.
+ UserAgent string
+}
diff --git a/internal/httpclientx/getjson.go b/internal/httpclientx/getjson.go
new file mode 100644
index 000000000..fb0896656
--- /dev/null
+++ b/internal/httpclientx/getjson.go
@@ -0,0 +1,44 @@
+package httpclientx
+
+//
+// getjson.go - GET a JSON response.
+//
+
+import (
+ "context"
+ "encoding/json"
+)
+
+// GetJSON sends a GET request and reads a JSON response.
+//
+// Arguments:
+//
+// - ctx is the cancellable context;
+//
+// - URL is the URL to use;
+//
+// - config contains the config.
+//
+// This function either returns an error or a valid Output.
+func GetJSON[Output any](ctx context.Context, URL string, config *Config) (Output, error) {
+ return NewOverlappedGetJSON[Output](config).Run(ctx, URL)
+}
+
+func getJSON[Output any](ctx context.Context, URL string, config *Config) (Output, error) {
+ // read the raw body
+ rawrespbody, err := GetRaw(ctx, URL, config)
+
+ // handle the case of error
+ if err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // parse the response body as JSON
+ var output Output
+ if err := json.Unmarshal(rawrespbody, &output); err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // avoid returning nil pointers, maps, slices
+ return NilSafetyErrorIfNil(output)
+}
diff --git a/internal/httpclientx/getjson_test.go b/internal/httpclientx/getjson_test.go
new file mode 100644
index 000000000..bbc453c75
--- /dev/null
+++ b/internal/httpclientx/getjson_test.go
@@ -0,0 +1,290 @@
+package httpclientx
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+ "github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+type apiResponse struct {
+ Age int
+ Name string
+}
+
+func TestGetJSON(t *testing.T) {
+ t.Run("when GetRaw fails", func(t *testing.T) {
+ // create a server that RST connections
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is nil.
+ if resp != nil {
+ t.Fatal("expected nil response")
+ }
+ })
+
+ t.Run("when JSON parsing fails", func(t *testing.T) {
+ // create a server that returns an invalid JSON type
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte("[]"))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err.Error() != "json: cannot unmarshal array into Go value of type httpclientx.apiResponse" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is nil.
+ if resp != nil {
+ t.Fatal("expected nil response")
+ }
+ })
+
+ t.Run("on success", func(t *testing.T) {
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"Name": "simone", "Age": 41}`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK
+ expect := &apiResponse{Name: "simone", Age: 41}
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+}
+
+// This test ensures that GetJSON sets correct HTTP headers
+func TestGetJSONHeadersOkay(t *testing.T) {
+ var (
+ gotheaders http.Header
+ gotmu sync.Mutex
+ )
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // save the headers
+ gotmu.Lock()
+ gotheaders = r.Header
+ gotmu.Unlock()
+
+ // send a minimal 200 Ok response
+ w.WriteHeader(200)
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // send the request and receive the response
+ apiresp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{
+ Authorization: "scribai",
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // we do not expect to see an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // given the handler, we expect to see an empty structure here
+ if apiresp.Age != 0 || apiresp.Name != "" {
+ t.Fatal("expected empty response")
+ }
+
+ // make sure there are no data races
+ defer gotmu.Unlock()
+ gotmu.Lock()
+
+ // make sure we have sent the authorization header
+ if value := gotheaders.Get("Authorization"); value != "scribai" {
+ t.Fatal("unexpected Authorization value", value)
+ }
+
+ // now make sure we have sent user-agent
+ if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent {
+ t.Fatal("unexpected User-Agent value", value)
+ }
+
+ // now make sure we have sent accept-encoding
+ if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
+ t.Fatal("unexpected Accept-Encoding value", value)
+ }
+}
+
+// This test ensures GetJSON logs the response body at Debug level.
+func TestGetJSONLoggingOkay(t *testing.T) {
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"Name": "simone", "Age": 41}`))
+ }))
+ defer server.Close()
+
+ // instantiate a logger that collects logs
+ logger := &testingx.Logger{}
+
+ // invoke the API
+ resp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: logger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK
+ expect := &apiResponse{Name: "simone", Age: 41}
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // collect and verify the debug lines
+ debuglines := logger.DebugLines()
+ t.Log(debuglines)
+ if len(debuglines) != 1 {
+ t.Fatal("expected to see a single debug line")
+ }
+ if !strings.Contains(debuglines[0], "raw response body:") {
+ t.Fatal("did not see raw response body log line")
+ }
+}
+
+// TestGetJSONCorrectlyRejectsNilValues ensures we correctly reject nil values.
+func TestGetJSONCorrectlyRejectsNilValues(t *testing.T) {
+
+ t.Run("when unmarshaling into a map", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[map[string]string](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when unmarshaling into a struct pointer", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when unmarshaling into a slice", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetJSON[[]string](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+}
diff --git a/internal/httpclientx/getraw.go b/internal/httpclientx/getraw.go
new file mode 100644
index 000000000..193df282e
--- /dev/null
+++ b/internal/httpclientx/getraw.go
@@ -0,0 +1,36 @@
+package httpclientx
+
+//
+// getraw.go - GET a raw response.
+//
+
+import (
+ "context"
+ "net/http"
+)
+
+// GetRaw sends a GET request and reads a raw response.
+//
+// Arguments:
+//
+// - ctx is the cancellable context;
+//
+// - config is the config to use;
+//
+// - URL is the URL to use.
+//
+// This function either returns an error or a valid Output.
+func GetRaw(ctx context.Context, URL string, config *Config) ([]byte, error) {
+ return NewOverlappedGetRaw(config).Run(ctx, URL)
+}
+
+func getRaw(ctx context.Context, URL string, config *Config) ([]byte, error) {
+ // construct the request to use
+ req, err := http.NewRequestWithContext(ctx, "GET", URL, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ // get raw response body
+ return do(ctx, req, config)
+}
diff --git a/internal/httpclientx/getraw_test.go b/internal/httpclientx/getraw_test.go
new file mode 100644
index 000000000..28c03d410
--- /dev/null
+++ b/internal/httpclientx/getraw_test.go
@@ -0,0 +1,165 @@
+package httpclientx
+
+import (
+ "context"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+func TestGetRaw(t *testing.T) {
+ t.Run("when we cannot create a request", func(t *testing.T) {
+ // create API call config
+
+ rawrespbody, err := GetRaw(
+ context.Background(),
+ "\t", // <- invalid URL that we cannot parse
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ },
+ )
+
+ t.Log(rawrespbody)
+ t.Log(err)
+
+ if err.Error() != `parse "\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ if len(rawrespbody) != 0 {
+ t.Fatal("expected zero-length body")
+ }
+ })
+
+ t.Run("on success", func(t *testing.T) {
+ expected := []byte(`Bonsoir, Elliot`)
+
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(expected)
+ }))
+ defer server.Close()
+
+ rawrespbody, err := GetRaw(context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(rawrespbody)
+ t.Log(err)
+
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ if diff := cmp.Diff(expected, rawrespbody); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+}
+
+// This test ensures that GetRaw sets correct HTTP headers
+func TestGetRawHeadersOkay(t *testing.T) {
+ var (
+ gotheaders http.Header
+ gotmu sync.Mutex
+ )
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // save the headers
+ gotmu.Lock()
+ gotheaders = r.Header
+ gotmu.Unlock()
+
+ // send a minimal 200 Ok response
+ w.WriteHeader(200)
+ w.Write([]byte(``))
+ }))
+ defer server.Close()
+
+ // send the request and receive the response
+ rawresp, err := GetRaw(context.Background(), server.URL, &Config{
+ Authorization: "scribai",
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // we do not expect to see an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // make sure the raw response is exactly what we expect to receive
+ if diff := cmp.Diff([]byte(``), rawresp); diff != "" {
+ t.Fatal("unexpected raw response")
+ }
+
+ // make sure there are no data races
+ defer gotmu.Unlock()
+ gotmu.Lock()
+
+ // make sure we have sent the authorization header
+ if value := gotheaders.Get("Authorization"); value != "scribai" {
+ t.Fatal("unexpected Authorization value", value)
+ }
+
+ // now make sure we have sent user-agent
+ if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent {
+ t.Fatal("unexpected User-Agent value", value)
+ }
+
+ // now make sure we have sent accept-encoding
+ if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
+ t.Fatal("unexpected Accept-Encoding value", value)
+ }
+}
+
+// This test ensures GetRaw logs the response body at Debug level.
+func TestGetRawLoggingOkay(t *testing.T) {
+ expected := []byte(`Bonsoir, Elliot`)
+
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(expected)
+ }))
+ defer server.Close()
+
+ // instantiate a logger that collects logs
+ logger := &testingx.Logger{}
+
+ rawrespbody, err := GetRaw(context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: logger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(rawrespbody)
+ t.Log(err)
+
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ if diff := cmp.Diff(expected, rawrespbody); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // collect and verify the debug lines
+ debuglines := logger.DebugLines()
+ t.Log(debuglines)
+ if len(debuglines) != 1 {
+ t.Fatal("expected to see a single debug line")
+ }
+ if !strings.Contains(debuglines[0], "raw response body:") {
+ t.Fatal("did not see raw response body log line")
+ }
+}
diff --git a/internal/httpclientx/getxml.go b/internal/httpclientx/getxml.go
new file mode 100644
index 000000000..34ddac2ed
--- /dev/null
+++ b/internal/httpclientx/getxml.go
@@ -0,0 +1,48 @@
+package httpclientx
+
+//
+// getxml.go - GET an XML response.
+//
+
+import (
+ "context"
+ "encoding/xml"
+)
+
+// GetXML sends a GET request and reads an XML response.
+//
+// Arguments:
+//
+// - ctx is the cancellable context;
+//
+// - URL is the URL to use;
+//
+// - config is the config to use.
+//
+// This function either returns an error or a valid Output.
+func GetXML[Output any](ctx context.Context, URL string, config *Config) (Output, error) {
+ return NewOverlappedGetXML[Output](config).Run(ctx, URL)
+}
+
+func getXML[Output any](ctx context.Context, URL string, config *Config) (Output, error) {
+ // read the raw body
+ rawrespbody, err := GetRaw(ctx, URL, config)
+
+ // handle the case of error
+ if err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // parse the response body as JSON
+ var output Output
+ if err := xml.Unmarshal(rawrespbody, &output); err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // TODO(bassosimone): it's unclear to me whether output can be nil when unmarshaling
+ // XML input, since there is no "null" in XML. In any case, the code below checks for
+ // and avoids emitting nil, so I guess we should be fine here.
+
+ // avoid returning nil pointers, maps, slices
+ return NilSafetyErrorIfNil(output)
+}
diff --git a/internal/httpclientx/getxml_test.go b/internal/httpclientx/getxml_test.go
new file mode 100644
index 000000000..e7a85337d
--- /dev/null
+++ b/internal/httpclientx/getxml_test.go
@@ -0,0 +1,201 @@
+package httpclientx
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+ "github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+func TestGetXML(t *testing.T) {
+ t.Run("when GetRaw fails", func(t *testing.T) {
+ // create a server that RST connections
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is nil.
+ if resp != nil {
+ t.Fatal("expected nil response")
+ }
+ })
+
+ t.Run("when XML parsing fails", func(t *testing.T) {
+ // create a server that returns an invalid XML file
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte("[]"))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, io.EOF) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is nil.
+ if resp != nil {
+ t.Fatal("expected nil response")
+ }
+ })
+
+ t.Run("on success", func(t *testing.T) {
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`simone41`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK
+ expect := &apiResponse{Name: "simone", Age: 41}
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+}
+
+// This test ensures that GetXML sets correct HTTP headers
+func TestGetXMLHeadersOkay(t *testing.T) {
+ var (
+ gotheaders http.Header
+ gotmu sync.Mutex
+ )
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // save the headers
+ gotmu.Lock()
+ gotheaders = r.Header
+ gotmu.Unlock()
+
+ // send a minimal 200 Ok response
+ w.WriteHeader(200)
+ w.Write([]byte(``))
+ }))
+ defer server.Close()
+
+ // send the request and receive the response
+ apiresp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{
+ Authorization: "scribai",
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // we do not expect to see an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // given the handler, we expect to see an empty structure here
+ if apiresp.Age != 0 || apiresp.Name != "" {
+ t.Fatal("expected empty response")
+ }
+
+ // make sure there are no data races
+ defer gotmu.Unlock()
+ gotmu.Lock()
+
+ // make sure we have sent the authorization header
+ if value := gotheaders.Get("Authorization"); value != "scribai" {
+ t.Fatal("unexpected Authorization value", value)
+ }
+
+ // now make sure we have sent user-agent
+ if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent {
+ t.Fatal("unexpected User-Agent value", value)
+ }
+
+ // now make sure we have sent accept-encoding
+ if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
+ t.Fatal("unexpected Accept-Encoding value", value)
+ }
+}
+
+// This test ensures GetXML logs the response body at Debug level.
+func TestGetXMLLoggingOkay(t *testing.T) {
+ // create a server that returns a legit response
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`simone41`))
+ }))
+ defer server.Close()
+
+ // instantiate a logger that collects logs
+ logger := &testingx.Logger{}
+
+ // invoke the API
+ resp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: logger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK
+ expect := &apiResponse{Name: "simone", Age: 41}
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // collect and verify the debug lines
+ debuglines := logger.DebugLines()
+ t.Log(debuglines)
+ if len(debuglines) != 1 {
+ t.Fatal("expected to see a single debug line")
+ }
+ if !strings.Contains(debuglines[0], "raw response body:") {
+ t.Fatal("did not see raw response body log line")
+ }
+}
diff --git a/internal/httpclientx/httpclientx.go b/internal/httpclientx/httpclientx.go
new file mode 100644
index 000000000..a5e826e7d
--- /dev/null
+++ b/internal/httpclientx/httpclientx.go
@@ -0,0 +1,103 @@
+// Package httpclientx contains extensions to more easily invoke HTTP APIs.
+package httpclientx
+
+//
+// httpclientx.go - common code
+//
+
+import (
+ "compress/gzip"
+ "context"
+ "io"
+ "net/http"
+
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+// ErrRequestFailed indicates that an HTTP request status indicates failure.
+type ErrRequestFailed struct {
+ StatusCode int
+}
+
+var _ error = &ErrRequestFailed{}
+
+// Error returns the error as a string.
+//
+// The string returned by this error starts with the httpx prefix for backwards
+// compatibility with the legacy httpx package.
+func (err *ErrRequestFailed) Error() string {
+ return "httpx: request failed"
+}
+
+// zeroValue is a convenience function to return the zero value.
+func zeroValue[T any]() T {
+ return *new(T)
+}
+
+// newLimitReader is a wrapper for [io.LimitReader] that automatically
+// sets the maximum readable amount of bytes.
+func newLimitReader(r io.Reader) io.Reader {
+ return io.LimitReader(r, 1<<24)
+}
+
+// do is the internal function to finish preparing the request and getting a raw response.
+func do(ctx context.Context, req *http.Request, config *Config) ([]byte, error) {
+ // optionally assign authorization
+ if value := config.Authorization; value != "" {
+ req.Header.Set("Authorization", value)
+ }
+
+ // assign the user agent
+ req.Header.Set("User-Agent", config.UserAgent)
+
+ // say that we're accepting gzip encoded bodies
+ req.Header.Set("Accept-Encoding", "gzip")
+
+ // get the response
+ resp, err := config.Client.Do(req)
+
+ // handle the case of error
+ if err != nil {
+ return nil, err
+ }
+
+ // eventually close the response body
+ defer resp.Body.Close()
+
+ // Implementation note: here we choose to always read the response
+ // body before checking the status code because it helps a lot to log
+ // the response body received on failure when testing a backend
+
+ var baseReader io.Reader = resp.Body
+
+ // handle the case of gzip encoded body
+ if resp.Header.Get("Content-Encoding") == "gzip" {
+ gzreader, err := gzip.NewReader(baseReader)
+ if err != nil {
+ return nil, err
+ }
+ baseReader = gzreader
+ }
+
+ // protect against unreasonably large response bodies
+ limitReader := newLimitReader(baseReader)
+
+ // read the raw body
+ rawrespbody, err := netxlite.ReadAllContext(ctx, limitReader)
+
+ // handle the case of failure
+ if err != nil {
+ return nil, err
+ }
+
+ // log the response body for debugging purposes
+ config.Logger.Debugf("%s %s: raw response body: %s", req.Method, req.URL.String(), string(rawrespbody))
+
+ // handle the case of HTTP error
+ if resp.StatusCode != 200 {
+ return nil, &ErrRequestFailed{resp.StatusCode}
+ }
+
+ // make sure we replace a nil slice with an empty slice
+ return NilSafetyAvoidNilBytesSlice(rawrespbody), nil
+}
diff --git a/internal/httpclientx/httpclientx_test.go b/internal/httpclientx/httpclientx_test.go
new file mode 100644
index 000000000..26a7fbd46
--- /dev/null
+++ b/internal/httpclientx/httpclientx_test.go
@@ -0,0 +1,132 @@
+package httpclientx
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "errors"
+ "net/http"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+ "github.com/ooni/probe-cli/v3/internal/runtimex"
+ "github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+func TestGzipDecompression(t *testing.T) {
+ t.Run("we correctly handle gzip encoding", func(t *testing.T) {
+ expected := []byte(`Bonsoir, Elliot!!!`)
+
+ // create a server returning compressed content
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var buffer bytes.Buffer
+ writer := gzip.NewWriter(&buffer)
+ _ = runtimex.Try1(writer.Write(expected))
+ runtimex.Try0(writer.Close())
+ w.Header().Add("Content-Encoding", "gzip")
+ w.Write(buffer.Bytes())
+ }))
+ defer server.Close()
+
+ // make sure we can read it
+ respbody, err := GetRaw(context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(respbody)
+ t.Log(err)
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if diff := cmp.Diff(expected, respbody); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+
+ t.Run("we correctly handle the case where we cannot decode gzip", func(t *testing.T) {
+ expected := []byte(`Bonsoir, Elliot!!!`)
+
+ // create a server pretending to return compressed content
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Add("Content-Encoding", "gzip")
+ w.Write(expected)
+ }))
+ defer server.Close()
+
+ // attempt to get a response body
+ respbody, err := GetRaw(context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(respbody)
+ t.Log(err)
+
+ if err.Error() != "gzip: invalid header" {
+ t.Fatal(err)
+ }
+
+ if respbody != nil {
+ t.Fatal("expected nil response body")
+ }
+ })
+}
+
+func TestHTTPStatusCodeHandling(t *testing.T) {
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerBlockpage451())
+ defer server.Close()
+
+ respbody, err := GetRaw(context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(respbody)
+ t.Log(err)
+
+ if err.Error() != "httpx: request failed" {
+ t.Fatal(err)
+ }
+
+ if respbody != nil {
+ t.Fatal("expected nil response body")
+ }
+
+ var orig *ErrRequestFailed
+ if !errors.As(err, &orig) {
+ t.Fatal("not an *ErrRequestFailed instance")
+ }
+ if orig.StatusCode != 451 {
+ t.Fatal("unexpected status code", orig.StatusCode)
+ }
+}
+
+func TestHTTPReadBodyErrorsHandling(t *testing.T) {
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerResetWhileReadingBody())
+ defer server.Close()
+
+ respbody, err := GetRaw(context.Background(), server.URL, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(respbody)
+ t.Log(err)
+
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("expected ECONNRESET, got", err)
+ }
+
+ if respbody != nil {
+ t.Fatal("expected nil response body")
+ }
+}
diff --git a/internal/httpclientx/nilsafety.go b/internal/httpclientx/nilsafety.go
new file mode 100644
index 000000000..c73323cb1
--- /dev/null
+++ b/internal/httpclientx/nilsafety.go
@@ -0,0 +1,31 @@
+package httpclientx
+
+import (
+ "errors"
+ "reflect"
+)
+
+// ErrIsNil indicates that [NilSafetyErrorIfNil] was passed a nil value.
+var ErrIsNil = errors.New("nil map, pointer, or slice")
+
+// NilSafetyErrorIfNil returns [ErrIsNil] iff input is a nil map, struct, or slice.
+//
+// This mechanism prevents us from mistakenly sending to a server a literal JSON "null" and
+// protects us from attempting to process a literal JSON "null" from a server.
+func NilSafetyErrorIfNil[Type any](value Type) (Type, error) {
+ switch rv := reflect.ValueOf(value); rv.Kind() {
+ case reflect.Map, reflect.Pointer, reflect.Slice:
+ if rv.IsNil() {
+ return zeroValue[Type](), ErrIsNil
+ }
+ }
+ return value, nil
+}
+
+// NilSafetyAvoidNilBytesSlice replaces a nil bytes slice with an empty slice.
+func NilSafetyAvoidNilBytesSlice(input []byte) []byte {
+ if input == nil {
+ input = []byte{}
+ }
+ return input
+}
diff --git a/internal/httpclientx/nilsafety_test.go b/internal/httpclientx/nilsafety_test.go
new file mode 100644
index 000000000..85551266e
--- /dev/null
+++ b/internal/httpclientx/nilsafety_test.go
@@ -0,0 +1,122 @@
+package httpclientx
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func TestNilSafetyErrorIfNil(t *testing.T) {
+
+ // testcase is a test case implemented by this function.
+ type testcase struct {
+ name string
+ input any
+ err error
+ output any
+ }
+
+ cases := []testcase{{
+ name: "with a nil map",
+ input: func() any {
+ var v map[string]string
+ return v
+ }(),
+ err: ErrIsNil,
+ output: nil,
+ }, {
+ name: "with a non-nil but empty map",
+ input: make(map[string]string),
+ err: nil,
+ output: make(map[string]string),
+ }, {
+ name: "with a non-nil non-empty map",
+ input: map[string]string{"a": "b"},
+ err: nil,
+ output: map[string]string{"a": "b"},
+ }, {
+ name: "with a nil pointer",
+ input: func() any {
+ var v *apiRequest
+ return v
+ }(),
+ err: ErrIsNil,
+ output: nil,
+ }, {
+ name: "with a non-nil empty pointer",
+ input: &apiRequest{},
+ err: nil,
+ output: &apiRequest{},
+ }, {
+ name: "with a non-nil non-empty pointer",
+ input: &apiRequest{UserID: 11},
+ err: nil,
+ output: &apiRequest{UserID: 11},
+ }, {
+ name: "with a nil slice",
+ input: func() any {
+ var v []int
+ return v
+ }(),
+ err: ErrIsNil,
+ output: nil,
+ }, {
+ name: "with a non-nil empty slice",
+ input: []int{},
+ err: nil,
+ output: []int{},
+ }, {
+ name: "with a non-nil non-empty slice",
+ input: []int{44},
+ err: nil,
+ output: []int{44},
+ }}
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ output, err := NilSafetyErrorIfNil(tc.input)
+
+ switch {
+ case err == nil && tc.err == nil:
+ if diff := cmp.Diff(tc.output, output); diff != "" {
+ t.Fatal(diff)
+ }
+ return
+
+ case err != nil && tc.err == nil:
+ t.Fatal("expected", tc.err.Error(), "got", err.Error())
+ return
+
+ case err == nil && tc.err != nil:
+ t.Fatal("expected", tc.err.Error(), "got", err.Error())
+ return
+
+ case err != nil && tc.err != nil:
+ if err.Error() != tc.err.Error() {
+ t.Fatal("expected", tc.err.Error(), "got", err.Error())
+ }
+ return
+ }
+ })
+ }
+}
+
+func TestNilSafetyAvoidNilByteSlice(t *testing.T) {
+ t.Run("for nil byte slice", func(t *testing.T) {
+ output := NilSafetyAvoidNilBytesSlice(nil)
+ if output == nil {
+ t.Fatal("expected non-nil")
+ }
+ if len(output) != 0 {
+ t.Fatal("expected zero length")
+ }
+ })
+
+ t.Run("for non-nil byte slice", func(t *testing.T) {
+ expected := []byte{44}
+ output := NilSafetyAvoidNilBytesSlice(expected)
+ if diff := cmp.Diff(expected, output); diff != "" {
+ t.Fatal("not the same pointer")
+ }
+ })
+}
diff --git a/internal/httpclientx/overlapped.go b/internal/httpclientx/overlapped.go
new file mode 100644
index 000000000..a60e776fd
--- /dev/null
+++ b/internal/httpclientx/overlapped.go
@@ -0,0 +1,163 @@
+package httpclientx
+
+//
+// overlapped.go - overlapped operations.
+//
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/ooni/probe-cli/v3/internal/erroror"
+)
+
+// OverlappedDefaultScheduleInterval is the default schedule interval. After this interval
+// has elapsed for a URL without seeing a success, we will schedule the next URL.
+const OverlappedDefaultScheduleInterval = 15 * time.Second
+
+// Overlapped represents the possibility of overlapping HTTP calls for a set of
+// functionally equivalent URLs, such that we start a new call if the previous one
+// has failed to produce a result within the configured ScheduleInterval.
+//
+// # Limitations
+//
+// Under very bad networking conditions, [*Overlapped] would cause a new network
+// call to start while the previous one is still in progress and very slowly downloading
+// a response. A future implementation SHOULD probably account for this possibility.
+type Overlapped[Output any] struct {
+ // RunFunc is the MANDATORY function that fetches the given URL.
+ //
+ // This field is typically initialized by [NewOverlappedGetJSON], [NewOverlappedGetRaw],
+ // [NewOverlappedGetXML], or [NewOverlappedPostJSON] to be the proper function that
+ // makes sense for the operation that you requested with the constructor.
+ //
+ // If you set it manually, you MUST modify it before calling [*Overlapped.Run].
+ RunFunc func(ctx context.Context, URL string) (Output, error)
+
+ // ScheduleInterval is the MANDATORY scheduling interval.
+ //
+ // This field is typically initialized by [NewOverlappedGetJSON], [NewOverlappedGetRaw],
+ // [NewOverlappedGetXML], or [NewOverlappedPostJSON] to be [OverlappedDefaultScheduleInterval].
+ //
+ // If you set it manually, you MUST modify it before calling [*Overlapped.Run].
+ ScheduleInterval time.Duration
+}
+
+func newOverlappedWithFunc[Output any](fx func(context.Context, string) (Output, error)) *Overlapped[Output] {
+ return &Overlapped[Output]{
+ RunFunc: fx,
+ ScheduleInterval: OverlappedDefaultScheduleInterval,
+ }
+}
+
+// NewOverlappedGetJSON constructs a [*Overlapped] for calling [GetJSON] with multiple URLs.
+func NewOverlappedGetJSON[Output any](config *Config) *Overlapped[Output] {
+ return newOverlappedWithFunc(func(ctx context.Context, URL string) (Output, error) {
+ return getJSON[Output](ctx, URL, config)
+ })
+}
+
+// NewOverlappedGetRaw constructs a [*Overlapped] for calling [GetRaw] with multiple URLs.
+func NewOverlappedGetRaw(config *Config) *Overlapped[[]byte] {
+ return newOverlappedWithFunc(func(ctx context.Context, URL string) ([]byte, error) {
+ return getRaw(ctx, URL, config)
+ })
+}
+
+// NewOverlappedGetXML constructs a [*Overlapped] for calling [GetXML] with multiple URLs.
+func NewOverlappedGetXML[Output any](config *Config) *Overlapped[Output] {
+ return newOverlappedWithFunc(func(ctx context.Context, URL string) (Output, error) {
+ return getXML[Output](ctx, URL, config)
+ })
+}
+
+// NewOverlappedPostJSON constructs a [*Overlapped] for calling [PostJSON] with multiple URLs.
+func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Overlapped[Output] {
+ return newOverlappedWithFunc(func(ctx context.Context, URL string) (Output, error) {
+ return postJSON[Input, Output](ctx, URL, input, config)
+ })
+}
+
+// ErrGenericOverlappedFailure indicates that a generic [*Overlapped] failure occurred.
+var ErrGenericOverlappedFailure = errors.New("overlapped: generic failure")
+
+// Run runs the overlapped operations, returning the result of the first operation
+// that succeeds and otherwise returning an error describing what happened.
+//
+// # Limitations
+//
+// This implementation creates a new goroutine for each provided URL under the assumption that
+// the overall number of URLs is small. A future revision would address this issue.
+func (ovx *Overlapped[Output]) Run(ctx context.Context, URLs ...string) (Output, error) {
+ // create cancellable context for early cancellation
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ // construct channel for collecting the results
+ output := make(chan *erroror.Value[Output])
+
+ // schedule a measuring goroutine per URL.
+ for idx := 0; idx < len(URLs); idx++ {
+ go ovx.transact(ctx, idx, URLs[idx], output)
+ }
+
+ // we expect to see exactly a response for each goroutine
+ var (
+ firstOutput *Output
+ errorv []error
+ )
+ for idx := 0; idx < len(URLs); idx++ {
+ // get a result from one of the goroutines
+ result := <-output
+
+ // handle the error case
+ if result.Err != nil {
+ errorv = append(errorv, result.Err)
+ continue
+ }
+
+ // possibly record the first success
+ if firstOutput == nil {
+ firstOutput = &result.Value
+ }
+
+ // make sure we interrupt all the other goroutines
+ cancel()
+ }
+
+ // handle the case of success
+ if firstOutput != nil {
+ return *firstOutput, nil
+ }
+
+ // handle the case where there's no error
+ if len(errorv) <= 0 {
+ errorv = append(errorv, ErrGenericOverlappedFailure)
+ }
+
+ // return zero value and errors list
+ return *new(Output), errors.Join(errorv...)
+}
+
+// transact performs an HTTP transaction with the given URL and writes results to the output channel.
+func (ovx *Overlapped[Output]) transact(ctx context.Context, idx int, URL string, output chan<- *erroror.Value[Output]) {
+ // wait for our time to start
+ //
+ // add one nanosecond to make sure the delay is always positive
+ timer := time.NewTimer(time.Duration(idx)*ovx.ScheduleInterval + time.Nanosecond)
+ defer timer.Stop()
+ select {
+ case <-ctx.Done():
+ output <- &erroror.Value[Output]{Err: ctx.Err()}
+ return
+ case <-timer.C:
+ // fallthrough
+ }
+
+ // obtain the results
+ value, err := ovx.RunFunc(ctx, URL)
+
+ // emit the results
+ output <- &erroror.Value[Output]{Err: err, Value: value}
+}
diff --git a/internal/httpclientx/overlapped_test.go b/internal/httpclientx/overlapped_test.go
new file mode 100644
index 000000000..350823e9f
--- /dev/null
+++ b/internal/httpclientx/overlapped_test.go
@@ -0,0 +1,209 @@
+package httpclientx
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/apex/log"
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/must"
+ "github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+// Implementation note: because top-level functions such as GetRaw always use
+// an [*Overlapped], we do not necessarily need to test that each top-level constructor
+// are WAI; rather, we should focus on the mechanics of multiple URLs.
+
+func TestNewOverlappedPostJSONIsPerformingOverlappedCalls(t *testing.T) {
+
+ // Scenario:
+ //
+ // - 0.th.ooni.org is SNI blocked
+ // - 1.th.ooni.org is SNI blocked
+ // - 2.th.ooni.org is SNI blocked
+ // - 3.th.ooni.org WAIs
+
+ zeroTh := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer zeroTh.Close()
+
+ oneTh := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer oneTh.Close()
+
+ twoTh := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer twoTh.Close()
+
+ expectedResponse := &apiResponse{
+ Age: 41,
+ Name: "sbs",
+ }
+
+ threeTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer threeTh.Close()
+
+ // Create client configuration. We don't care much about the
+ // JSON requests and reponses being aligned to reality.
+
+ apiReq := &apiRequest{
+ UserID: 117,
+ }
+
+ overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{
+ Authorization: "", // not relevant for this test
+ Client: http.DefaultClient,
+ Logger: log.Log,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // make sure we set a low scheduling interval to make test faster
+ overlapped.ScheduleInterval = time.Second
+
+ // Now we issue the requests and check we're getting the correct response.
+
+ apiResp, err := overlapped.Run(
+ context.Background(),
+ zeroTh.URL,
+ oneTh.URL,
+ twoTh.URL,
+ threeTh.URL,
+ )
+
+ // we do not expect to see a failure because threeTh is WAI
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // compare response to expectation
+ if diff := cmp.Diff(expectedResponse, apiResp); diff != "" {
+ t.Fatal(diff)
+ }
+}
+
+func TestNewOverlappedPostJSONCancelsPendingCalls(t *testing.T) {
+
+ // Scenario:
+ //
+ // - 0.th.ooni.org is WAI but slow
+ // - 1.th.ooni.org is WAI
+ // - 2.th.ooni.org is WAI
+ // - 3.th.ooni.org is WAI
+
+ expectedResponse := &apiResponse{
+ Age: 41,
+ Name: "sbs",
+ }
+
+ slowwakeup := make(chan any)
+
+ zeroTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-slowwakeup
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer zeroTh.Close()
+
+ oneTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-slowwakeup
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer oneTh.Close()
+
+ twoTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-slowwakeup
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer twoTh.Close()
+
+ threeTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-slowwakeup
+ w.Write(must.MarshalJSON(expectedResponse))
+ }))
+ defer threeTh.Close()
+
+ // Create client configuration. We don't care much about the
+ // JSON requests and reponses being aligned to reality.
+
+ apiReq := &apiRequest{
+ UserID: 117,
+ }
+
+ overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{
+ Authorization: "", // not relevant for this test
+ Client: http.DefaultClient,
+ Logger: log.Log,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // make sure the schedule interval is high because we want
+ // all the goroutines but the first to be waiting for permission
+ // to fetch from their respective URLs.
+ overlapped.ScheduleInterval = 15 * time.Second
+
+ // In the background we're going to emit slow wakeup signals at fixed intervals
+ // after an initial waiting interval, such that goroutines unblock in order
+
+ go func() {
+ time.Sleep(250 * time.Millisecond)
+ for idx := 0; idx < 4; idx++ {
+ slowwakeup <- true
+ time.Sleep(250 * time.Millisecond)
+ }
+ close(slowwakeup)
+ }()
+
+ // Now we issue the requests and check we're getting the correct response.
+
+ apiResp, err := overlapped.Run(
+ context.Background(),
+ zeroTh.URL,
+ oneTh.URL,
+ twoTh.URL,
+ threeTh.URL,
+ )
+
+ // we do not expect to see a failure because threeTh is WAI
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // compare response to expectation
+ if diff := cmp.Diff(expectedResponse, apiResp); diff != "" {
+ t.Fatal(diff)
+ }
+}
+
+func TestNewOverlappedPostJSONWithNoURLs(t *testing.T) {
+
+ // Create client configuration. We don't care much about the
+ // JSON requests and reponses being aligned to reality.
+
+ apiReq := &apiRequest{
+ UserID: 117,
+ }
+
+ overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{
+ Authorization: "", // not relevant for this test
+ Client: http.DefaultClient,
+ Logger: log.Log,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // Now we issue the requests without any URLs and make sure
+ // the result we get is the generic overlapped error
+
+ apiResp, err := overlapped.Run(context.Background())
+
+ // we do not expect to see a failure because threeTh is WAI
+ if !errors.Is(err, ErrGenericOverlappedFailure) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // we expect a nil response
+ if apiResp != nil {
+ t.Fatal("expected nil API response")
+ }
+}
diff --git a/internal/httpclientx/postjson.go b/internal/httpclientx/postjson.go
new file mode 100644
index 000000000..5c559b184
--- /dev/null
+++ b/internal/httpclientx/postjson.go
@@ -0,0 +1,71 @@
+package httpclientx
+
+//
+// postjson.go - POST a JSON request and read a JSON response.
+//
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "net/http"
+)
+
+// PostJSON sends a POST request with a JSON body and reads a JSON response.
+//
+// Arguments:
+//
+// - ctx is the cancellable context;
+//
+// - URL is the URL to use;
+//
+// - input is the input structure to JSON serialize as the request body;
+//
+// - config is the config to use.
+//
+// This function either returns an error or a valid Output.
+func PostJSON[Input, Output any](ctx context.Context, URL string, input Input, config *Config) (Output, error) {
+ return NewOverlappedPostJSON[Input, Output](input, config).Run(ctx, URL)
+}
+
+func postJSON[Input, Output any](ctx context.Context, URL string, input Input, config *Config) (Output, error) {
+ // ensure we're not sending a nil map, pointer, or slice
+ if _, err := NilSafetyErrorIfNil(input); err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // serialize the request body
+ rawreqbody, err := json.Marshal(input)
+ if err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // log the raw request body
+ config.Logger.Debugf("POST %s: raw request body: %s", URL, string(rawreqbody))
+
+ // construct the request to use
+ req, err := http.NewRequestWithContext(ctx, "POST", URL, bytes.NewReader(rawreqbody))
+ if err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // assign the content type
+ req.Header.Set("Content-Type", "application/json")
+
+ // get the raw response body
+ rawrespbody, err := do(ctx, req, config)
+
+ // handle the case of error
+ if err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // parse the response body as JSON
+ var output Output
+ if err := json.Unmarshal(rawrespbody, &output); err != nil {
+ return zeroValue[Output](), err
+ }
+
+ // avoid returning nil pointers, maps, slices
+ return NilSafetyErrorIfNil(output)
+}
diff --git a/internal/httpclientx/postjson_test.go b/internal/httpclientx/postjson_test.go
new file mode 100644
index 000000000..bed543d8a
--- /dev/null
+++ b/internal/httpclientx/postjson_test.go
@@ -0,0 +1,468 @@
+package httpclientx
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/must"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+ "github.com/ooni/probe-cli/v3/internal/runtimex"
+ "github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+type apiRequest struct {
+ UserID int
+}
+
+func TestPostJSON(t *testing.T) {
+ t.Run("when we cannot marshal the request body", func(t *testing.T) {
+ // a channel cannot be serialized
+ req := make(chan int)
+ close(req)
+
+ resp, err := PostJSON[chan int, *apiResponse](context.Background(), "", req, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ if err.Error() != `json: unsupported type: chan int` {
+ t.Fatal("unexpected error", err)
+ }
+
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when we cannot create a request", func(t *testing.T) {
+ req := &apiRequest{117}
+
+ resp, err := PostJSON[*apiRequest, *apiResponse](
+ context.Background(),
+ "\t", // <- invalid URL that we cannot parse
+ req,
+ &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ },
+ )
+
+ t.Log(resp)
+ t.Log(err)
+
+ if err.Error() != `parse "\t": net/url: invalid control character in URL` {
+ t.Fatal("unexpected error", err)
+ }
+
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("in case of HTTP failure", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset())
+ defer server.Close()
+
+ req := &apiRequest{117}
+
+ resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, req, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ if !errors.Is(err, netxlite.ECONNRESET) {
+ t.Fatal("unexpected error", err)
+ }
+
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when we cannot parse the response body", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte("[]"))
+ }))
+ defer server.Close()
+
+ req := &apiRequest{117}
+
+ resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, req, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err.Error() != "json: cannot unmarshal array into Go value of type httpclientx.apiResponse" {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is nil.
+ if resp != nil {
+ t.Fatal("expected nil response")
+ }
+ })
+
+ t.Run("on success", func(t *testing.T) {
+ req := &apiRequest{117}
+
+ expect := &apiResponse{Name: "simone", Age: 41}
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var gotreq apiRequest
+ data := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body))
+ must.UnmarshalJSON(data, &gotreq)
+ if gotreq.UserID != req.UserID {
+ w.WriteHeader(404)
+ return
+ }
+ w.Write(must.MarshalJSON(expect))
+ }))
+ defer server.Close()
+
+ resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, req, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK.
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+}
+
+// This test ensures that PostJSON sets correct HTTP headers and sends the right body.
+func TestPostJSONCommunicationOkay(t *testing.T) {
+ var (
+ gotheaders http.Header
+ gotrawbody []byte
+ gotmu sync.Mutex
+ )
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // read the raw response body
+ rawbody := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body))
+
+ // save the raw response body and headers
+ gotmu.Lock()
+ gotrawbody = rawbody
+ gotheaders = r.Header
+ gotmu.Unlock()
+
+ // send a minimal 200 Ok response
+ w.WriteHeader(200)
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // create and serialize the expected request body
+ apireq := &apiRequest{
+ UserID: 117,
+ }
+ rawapireq := must.MarshalJSON(apireq)
+
+ // send the request and receive the response
+ apiresp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, apireq, &Config{
+ Authorization: "scribai",
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ // we do not expect to see an error here
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // given the handler, we expect to see an empty structure here
+ if apiresp.Age != 0 || apiresp.Name != "" {
+ t.Fatal("expected empty response")
+ }
+
+ // make sure there are no data races
+ defer gotmu.Unlock()
+ gotmu.Lock()
+
+ // now verify what the handler has read as the raw request body
+ if diff := cmp.Diff(rawapireq, gotrawbody); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // make sure we have sent the authorization header
+ if value := gotheaders.Get("Authorization"); value != "scribai" {
+ t.Fatal("unexpected Authorization value", value)
+ }
+
+ // now make sure we have sent content-type
+ if value := gotheaders.Get("Content-Type"); value != "application/json" {
+ t.Fatal("unexpected Content-Type value", value)
+ }
+
+ // now make sure we have sent user-agent
+ if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent {
+ t.Fatal("unexpected User-Agent value", value)
+ }
+
+ // now make sure we have sent accept-encoding
+ if value := gotheaders.Get("Accept-Encoding"); value != "gzip" {
+ t.Fatal("unexpected Accept-Encoding value", value)
+ }
+}
+
+// This test ensures PostJSON logs the request and response body at Debug level.
+func TestPostJSONLoggingOkay(t *testing.T) {
+ req := &apiRequest{117}
+
+ expect := &apiResponse{Name: "simone", Age: 41}
+
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var gotreq apiRequest
+ data := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body))
+ must.UnmarshalJSON(data, &gotreq)
+ if gotreq.UserID != req.UserID {
+ w.WriteHeader(404)
+ return
+ }
+ w.Write(must.MarshalJSON(expect))
+ }))
+ defer server.Close()
+
+ // instantiate a logger that collects logs
+ logger := &testingx.Logger{}
+
+ resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, req, &Config{
+ Client: http.DefaultClient,
+ Logger: logger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if err != nil {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure the response is OK.
+ if diff := cmp.Diff(expect, resp); diff != "" {
+ t.Fatal(diff)
+ }
+
+ // collect and verify the debug lines
+ debuglines := logger.DebugLines()
+ t.Log(debuglines)
+ if len(debuglines) != 2 {
+ t.Fatal("expected to see a single debug line")
+ }
+ if !strings.Contains(debuglines[0], "raw request body:") {
+ t.Fatal("did not see raw request body log line")
+ }
+ if !strings.Contains(debuglines[1], "raw response body:") {
+ t.Fatal("did not see raw response body log line")
+ }
+}
+
+// TestPostJSONCorrectlyRejectsNilValues ensures we do not emit and correctly reject nil values.
+func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) {
+
+ t.Run("when sending a nil map", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := PostJSON[map[string]string, *apiResponse](context.Background(), server.URL, nil, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when sending a nil struct pointer", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, nil, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when sending a nil slice", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{}`))
+ }))
+ defer server.Close()
+
+ // invoke the API
+ resp, err := PostJSON[[]string, *apiResponse](context.Background(), server.URL, nil, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when unmarshaling into a map", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // create an empty request
+ apireq := &apiRequest{}
+
+ // invoke the API
+ resp, err := PostJSON[*apiRequest, map[string]string](context.Background(), server.URL, apireq, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when unmarshaling into a struct pointer", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // create an empty request
+ apireq := &apiRequest{}
+
+ // invoke the API
+ resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, apireq, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+
+ t.Run("when unmarshaling into a slice", func(t *testing.T) {
+ server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`null`))
+ }))
+ defer server.Close()
+
+ // create an empty request
+ apireq := &apiRequest{}
+
+ // invoke the API
+ resp, err := PostJSON[*apiRequest, []string](context.Background(), server.URL, apireq, &Config{
+ Client: http.DefaultClient,
+ Logger: model.DiscardLogger,
+ UserAgent: model.HTTPHeaderUserAgent,
+ })
+
+ t.Log(resp)
+ t.Log(err)
+
+ // make sure that the error is the expected one
+ if !errors.Is(err, ErrIsNil) {
+ t.Fatal("unexpected error", err)
+ }
+
+ // make sure resp is nil
+ if resp != nil {
+ t.Fatal("expected nil resp")
+ }
+ })
+}