diff --git a/internal/erroror/erroror.go b/internal/erroror/erroror.go new file mode 100644 index 000000000..833272ee8 --- /dev/null +++ b/internal/erroror/erroror.go @@ -0,0 +1,8 @@ +// Package erroror contains code to represent an error or a value. +package erroror + +// Value represents an error or a value. +type Value[Type any] struct { + Err error + Value Type +} diff --git a/internal/httpclientx/DESIGN.md b/internal/httpclientx/DESIGN.md new file mode 100644 index 000000000..af22ad1fe --- /dev/null +++ b/internal/httpclientx/DESIGN.md @@ -0,0 +1,418 @@ +# ./internal/httpclientx + +This package aims to replace previously existing packages for interact with +backend services controlled either by us or by third parties. + +As of 2024-04-22, these packages are: + +* `./internal/httpx`, which is currently deprecated; + +* `./internal/httpapi`, which aims to automatically generate swaggers from input +and output messages and implements support for falling back. + +The rest of this document explains the requirements and describes the design. + +## Table of Contents + +- [Requirements](#requirements) +- [Design](#design) + - [Overlapped Operations](#overlapped-operations) + - [Extensibility](#extensibility) + - [Functionality Comparison](#functionality-comparison) + - [GetJSON](#getjson) + - [GetRaw](#getraw) + - [GetXML](#getxml) + - [PostJSON](#postjson) +- [Nil Safety](#nil-safety) +- [Refactoring Plan](#refactoring-plan) +- [Limitations and Future Work](#limitations-and-future-work) + +## Requirements + +We want this new package to: + +1. Implement common access patterns (GET with JSON body, GET with XML body, GET with +raw body, and POST with JSON request and response bodies). + +2. Optionally log request and response bodies. + +3. Factor common code for accessing such services. + +4. Take advantage of Go generics to automatically marshal and unmarshal without +having to write specific functions for each request/response pair. + +5. Support some kind of fallback policy like `httpapi` because we used this +for test helpers and, while, as of today, we're mapping serveral TH domain names +to a single IP address, it might still be useful to fallback. + +6. Provide an easy way for the caller to know whether an HTTP request failed +and, if so, with which status code, which is needed to intercept `401` responses +to take the appropriate logging-in actions. + +7. Make the design extensible such that re-adding unused functionality (such +as cloud fronting) does not require us to refactor the code much. + +8. Functional equivalent with existing packages (modulo the existing +functionality that is not relevant anymore). + +Non goals: + +1. automatically generate swaggers from the Go representation of API invocation +for these reasons: (1) the OONI backend swagger is only for documentational +purposes and it is not always in sync with reality; (2) by doing that, we obtained +overly complex code, which hampers maintenance. + +2. implementing algorithms such as logging in to the OONI backend and requesting +tokens, which should be the responsibility of another package. + +3. implementing cloud fronting, which is not used anymore. + +## Design + +This package supports the following operations: + +```Go +type Config struct { + Client model.HTTPClient + Logger model.Logger + UserAgent string +} + +func GetJSON[Output any](ctx context.Context, URL string, config *Config) (Output, error) + +func GetRaw(ctx context.Context, URL string, config *Config) ([]byte, error) + +func GetXML[Output any](ctx context.Context, URL string, config *Config) (Output, error) + +func PostJSON[Input, Output any](ctx context.Context, URL string, input Input, config *Config) (Output, error) +``` + +(The `*Config` is the last argument because it is handy to create it inline when calling +and having it last reduces readability the least.) + +These operations implement all the actions listed in the first requirement. + +The `Config` struct allows to add new optional fields to implement new functionality without +changing the API and with minimal code refactoring efforts (seventh requirement). + +We're using generics to automate marshaling and umarshaling (requirement four). + +The internal implementation is such that we reuse code to avoid code duplication, +thus addressing also requirement three. + +Additionally, whenever a call fails with a non-200 status code, the return +value can be converted to the following type using `errors.As`: + +```Go +type ErrRequestFailed struct { + StatusCode int +} + +func (err *ErrRequestFailed) Error() string +``` + +Therefore, we have a way to know why a request failed (requirement six). + +To avoid logging bodies, one just needs to pass `model.DiscardLogger` as the +`logger` (thus fulfilling requirement two). + +### Overlapped Operations + +The code at `./internal/httpapi` performs sequential function calls. This design +does not interact well with the `enginenetx` package and its dial tactics. A better +strategy is to allow calls to be overlapped. This means that, if the `enginenetx` +is busy trying tactics for a given API endpoint, we eventually try to use the +subsequent (semantically-equivalent) endpoint after a given time, without waiting +for the first endpoint to complete. + +We allow for overlapped operations by defining these constructors: + +```Go +func NewOverlappedGetJSON[Output any](config *Config) *Overlapped[Output] + +func NewOverlappedGetRaw(config *Config) *Overlapped[[]byte] + +func NewOverlappedGetXML[Output any](config *Config) *Overlapped[Output] + +func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Overlapped[Output] +``` + +They all construct the same `*Overlapped` struct, which looks like this: + +```Go +type Overlapped[Output any] struct { + RunFunc func(ctx context.Context, URL string) (Output, error) + + ScheduleInterval time.Duration +} +``` + +The constructor configures `RunFunc` to invoke the call corresponding to the construct +name (i.e., `NewOverlappedGetXML` configures `RunFunc` to run `GetXML`). + +Then, we define the following method: + +```Go +func (ovx *Overlapped[Output]) Run(ctx context.Context, URLs ...string) (Output, error) +``` + +This method starts N goroutines to issue the API calls with each URL. (A classic example is for +the URLs to be `https://0.th.ooni.org/`, `https://1.th.ooni.org/` and so on.) + +By default, `ScheduleInterval` is 15 seconds. If the first URL does not provide a result +within 15 seconds, we try the second one. That is, every 15 seconds, we will attempt using +another URL, until there's a successful response or we run out of URLs. + +As soon as we have a successful response, we cancel all the other pending operations +that may exist. Once all operations have terminated, we return to the caller. + +### Extensibility + +We use the `Config` object to package common settings. Thus adding a new field (e.g., a +custom `Host` header to implement cloud fronting), only means the following: + +1. Adding a new OPTIONAL field to `Config`. + +2. Honoring this field inside the internal implementation. + +_Et voilà_, this should allow for minimal efforts API upgrades. + +### Functionality Comparison + +This section compares side-by-side the operations performed by each implementation +as of [probe-cli@7dab5a29812](https://github.com/ooni/probe-cli/tree/7dab5a29812) to +show that they implement ~equivalent functionality. This should be the case, since +the `httpxclientx` package is a refactoring of `httpapi`, which in turn contains code +originally derived from `httpx`. Anyways, better to double check. + +#### GetJSON + +We compare to `httpapi.Call` and `httpx.GetJSONWithQuery`. + +| Operation | GetJSON | httpapi | httpx | +| ------------------------- | ------- | ------- | ----- | +| enforce a call timeout | NO | yes | NO | +| parse base URL | NO | yes | yes | +| join path and base URL | NO | yes | yes | +| append query to URL | NO | yes | yes | +| NewRequestWithContext | yes️ | yes | yes | +| handle cloud front | NO | yes | yes | +| set Authorization | yes | yes | yes | +| set Accept | NO | yes | yes | +| set User-Agent | yes ️ | yes | yes | +| set Accept-Encoding gzip | yes️ | yes | NO | +| (HTTPClient).Do() | yes | yes | yes | +| defer resp.Body.Close() | yes | yes | yes | +| handle gzip encoding | yes | yes | NO | +| limit io.Reader | yes | yes | yes | +| netxlite.ReadAllContext() | yes | yes | yes | +| handle truncated body | NO | yes | NO | +| log response body | yes | yes | yes | +| handle non-200 response | ️ yes | yes* | yes* | +| unmarshal JSON | yes | yes | yes | + +The `yes*` means that `httpapi` rejects responses with status codes `>= 400` (like cURL) +while the new package only accepts status codes `== 200`. This difference should be of little +practical significance for all the APIs we invoke and the new behavior is stricter. + +Regarding all the other cases for which `GetJSON` is marked as "NO": + +1. Enforcing a call timeout is better done just through the context like `httpx` does. + +2. `GetJSON` lets the caller completely manage the construction of the URL, so we do not need +code to join together a base URL, possibly including a base path, a path, and a query (and we're +introducing the new `./internal/urlx` package to handle this situation). + +3. `GetJSON` does not handle cloud fronting because we don't use it. The design where the +`Config` contains mandatory and optional fields would still allow doing that easily. + +4. Setting the `Accept` header does not seem to matter in out context because we mostly +call API for which there's no need for content negotiation. + +5. It's difficult to say whether a body size was exactly the amount specified for truncation +or the body has been truncated. While this is a corner case, it seems perhaps wiser to let +the caller try parsing the body and failing if it is indeed truncated. + +#### GetRaw + +Here we're comparing to `httpapi.Call` and `httpx.FetchResource`. + +| Operation | GetRaw | httpapi | httpx | +| ------------------------- | ------- | ------- | ----- | +| enforce a call timeout | NO | yes | NO | +| parse base URL | NO | yes | yes | +| join path and base URL | NO | yes | yes | +| append query to URL | NO | yes | yes | +| NewRequestWithContext | yes️ | yes | yes | +| handle cloud front | NO | yes | yes | +| set Authorization | yes | yes | yes | +| set Accept | NO | yes | yes | +| set User-Agent | yes ️ | yes | yes | +| set Accept-Encoding gzip | yes️ | yes | NO | +| (HTTPClient).Do() | yes | yes | yes | +| defer resp.Body.Close() | yes | yes | yes | +| handle gzip encoding | yes | yes | NO | +| limit io.Reader | yes | yes | yes | +| netxlite.ReadAllContext() | yes | yes | yes | +| handle truncated body | NO | yes | NO | +| log response body | yes | yes | yes | +| handle non-200 response | ️ yes | yes* | yes | + +Here we can basically make equivalent remarks as those of the previous section. + +#### GetXML + +There's no direct equivalent of `GetXML` in `httpapi` and `httpx`. Therefore, when using these +two APIs, the caller would need to fetch a raw body and then manually parse XML. + +| Operation | GetXML | httpapi | httpx | +| ------------------------- | ------- | ------- | ----- | +| enforce a call timeout | NO | N/A | N/A | +| parse base URL | NO | N/A | N/A | +| join path and base URL | NO | N/A | N/A | +| append query to URL | NO | N/A | N/A | +| NewRequestWithContext | yes️ | N/A | N/A | +| handle cloud front | NO | N/A | N/A | +| set Authorization | yes | N/A | N/A | +| set Accept | NO | N/A | N/A | +| set User-Agent | yes ️ | N/A | N/A | +| set Accept-Encoding gzip | yes️ | N/A | N/A | +| (HTTPClient).Do() | yes | N/A | N/A | +| defer resp.Body.Close() | yes | N/A | N/A | +| handle gzip encoding | yes | N/A | N/A | +| limit io.Reader | yes | N/A | N/A | +| netxlite.ReadAllContext() | yes | N/A | N/A | +| handle truncated body | NO | N/A | N/A | +| log response body | yes | N/A | N/A | +| handle non-200 response | ️ yes | N/A | N/A | +| unmarshal XML | yes | N/A | N/A | + +Because comparison is not possible, there is not much else to say. + +#### PostJSON + +Here we're comparing to `httpapi.Call` and `httpx.PostJSON`. + +| Operation | PostJSON | httpapi | httpx | +| ------------------------- | -------- | ------- | ----- | +| marshal JSON | yes | yes~ | yes | +| log request body | yes | yes | yes | +| enforce a call timeout | NO | yes | NO | +| parse base URL | NO | yes | yes | +| join path and base URL | NO | yes | yes | +| append query to URL | NO | yes | yes | +| NewRequestWithContext | yes️ | yes | yes | +| handle cloud front | NO | yes | yes | +| set Authorization | yes | yes | yes | +| set Accept | NO | yes | yes | +| set Content-Type | yes | yes | yes | +| set User-Agent | yes ️ | yes | yes | +| set Accept-Encoding gzip | yes️ | yes | NO | +| (HTTPClient).Do() | yes | yes | yes | +| defer resp.Body.Close() | yes | yes | yes | +| handle gzip encoding | yes | yes | NO | +| limit io.Reader | yes | yes | yes | +| netxlite.ReadAllContext() | yes | yes | yes | +| handle truncated body | NO | yes | NO | +| log response body | yes | yes | yes | +| handle non-200 response | ️ yes | yes* | yes* | +| unmarshal JSON | yes | yes | yes | + +The `yes*` means that `httpapi` rejects responses with status codes `>= 400` (like cURL) +while the new package only accepts status codes `== 200`. This difference should be of little +practical significance for all the APIs we invoke and the new behavior is stricter. + +The `yes~` means that `httpapi` already receives a marshaled body from a higher-level API +that is part of the same package, while in this package we marshal in `PostJSON`. + +## Nil Safety + +Consider the following code snippet: + +```Go +resp, err := httpclientx.GetJSON[*APIResponse](ctx, URL, config) +runtimex.Assert((resp == nil && err != nil) || (resp != nil && err == nil), "ouch") +``` + +Now, consider the case where `URL` refers to a server that returns `null` as the JSON +answer, rather than returning a JSON object. The `encoding/json` package will accept the +`null` value and unmarshal it into a `nil` pointer. So, `GetJSON` will return `nil` and +`nil`, and the `runtimex.Assert` will fail. + +The `httpx` package did not have this issue because the usage pattern was: + +```Go +var resp APIResponse +err := apiClient.GetJSON(ctx, "/foobar", &resp) // where apiClient implements httpx.APIClient +``` + +In such a case, the `null` would have no effect and `resp` would be an empty response. + +However, it is still handy to return a value and an error, and it is the most commonly used +pattern in Go and, as a result, in OONI Probe. So, what do we do? + +Well, here's the strategy: + +1. When sending pointers, slices, or maps in `PostJSON`, we return `ErrIsNil` if the pointer, +slice, or map is `nil`, to avoid sending literal `null` to servers. + +2. `GetJSON`, `GetXML`, and `PostJSON` include checks after unmarshaling so that, if the API response +type is a slice, pointer, or map, and it is `nil`, we also return `ErrIsNil`. + +Strictly speaking, it is still unclear to us whether this could happen with `GetXML` but we have +decided to implements these checks for `GetXML` as well, just in case. + +## Refactoring Plan + +The overall goal is to replace usages of `httpapi` and `httpx` with usages of `httpclient`. + +The following packages use `httpapi`: + +1. `internal/experiment/webconnectivity`: uses `httpapi.SeqCaller` to chain calls to all +the available test helpers, which we can replace with using `*Overlapped`; + +2. `internal/experiment/webconnectivitylte`: same as above; + +3. `internal/ooapi`: uses `httpapi` to define the OONI backend APIs for the purpose of +generating and cross-validating swaggers, which is something we defined as a non-goal given +that we never really managed to do it reliably, and it has only led to code complexity; + +4. `internal/probeservices`: uses `httpapi` to implement check-in and the main reason why +this is the case is because it supports `"gzip"` encoding; + +The following packages use `httpx`: + +1. `internal/cmd/apitool`: this is just a byproduct of `probeservices.Client` embedding +the `httpx.APIClientTemplate`, so this should really be easy to get rid of; + +2. `internal/enginelocate`: we're using `httpx` convenience functions to figure out +the probe IP and we can easily replace these calls with `httpclientx`; + +3. `internal/oonirun`: uses `httpx` to fetch descriptors and can be easily replaced; + +4. `internal/probeservices`: uses `httpx` for most other calls. + +Based on the above information, it seems the easiest way to proceed is this: + +1. `internal/enginelocate`: replace `httpx` with `httpclientx`; + +2. `internal/oonirun`: replace `httpx` with `httpclientx`; + +3. `internal/probeservices`: replace the check-in implementation to use `httpclientx` +instead of using the `httpapi` package; + +4. `internal/experiment/webconnectivity{,lte}`: replace the `httpapi.SeqCaller` usage +with invocations of `*Overlapped`; + +5. remove the `httpapi` and `ooapi` packages, now unused; + +6. finish replacing `httpx` with `httpclientx` in `internal/probeservices` + +7. remove the `httpx` package. + +## Limitations and Future Work + +The current implementation of `*Overlapped` may cause us to do more work than needed in +case the network is really slow and an attempt is slowly fetching the body. In such a case, +starting a new attempt duplicates work. Handling this case does not seem straightforward +currently, therefore, we will focus on this as part of future work. diff --git a/internal/httpclientx/config.go b/internal/httpclientx/config.go new file mode 100644 index 000000000..71bdeb616 --- /dev/null +++ b/internal/httpclientx/config.go @@ -0,0 +1,20 @@ +package httpclientx + +import "github.com/ooni/probe-cli/v3/internal/model" + +// Config contains configuration shared by [GetJSON], [GetXML], [GetRaw], and [PostJSON]. +// +// The zero value is invalid; initialize the MANDATORY fields. +type Config struct { + // Authorization contains the OPTIONAL Authorization header value to use. + Authorization string + + // Client is the MANDATORY [model.HTTPClient] to use. + Client model.HTTPClient + + // Logger is the MANDATORY [model.Logger] to use. + Logger model.Logger + + // UserAgent is the MANDATORY User-Agent header value to use. + UserAgent string +} diff --git a/internal/httpclientx/getjson.go b/internal/httpclientx/getjson.go new file mode 100644 index 000000000..fb0896656 --- /dev/null +++ b/internal/httpclientx/getjson.go @@ -0,0 +1,44 @@ +package httpclientx + +// +// getjson.go - GET a JSON response. +// + +import ( + "context" + "encoding/json" +) + +// GetJSON sends a GET request and reads a JSON response. +// +// Arguments: +// +// - ctx is the cancellable context; +// +// - URL is the URL to use; +// +// - config contains the config. +// +// This function either returns an error or a valid Output. +func GetJSON[Output any](ctx context.Context, URL string, config *Config) (Output, error) { + return NewOverlappedGetJSON[Output](config).Run(ctx, URL) +} + +func getJSON[Output any](ctx context.Context, URL string, config *Config) (Output, error) { + // read the raw body + rawrespbody, err := GetRaw(ctx, URL, config) + + // handle the case of error + if err != nil { + return zeroValue[Output](), err + } + + // parse the response body as JSON + var output Output + if err := json.Unmarshal(rawrespbody, &output); err != nil { + return zeroValue[Output](), err + } + + // avoid returning nil pointers, maps, slices + return NilSafetyErrorIfNil(output) +} diff --git a/internal/httpclientx/getjson_test.go b/internal/httpclientx/getjson_test.go new file mode 100644 index 000000000..bbc453c75 --- /dev/null +++ b/internal/httpclientx/getjson_test.go @@ -0,0 +1,290 @@ +package httpclientx + +import ( + "context" + "errors" + "net/http" + "strings" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/testingx" +) + +type apiResponse struct { + Age int + Name string +} + +func TestGetJSON(t *testing.T) { + t.Run("when GetRaw fails", func(t *testing.T) { + // create a server that RST connections + server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset()) + defer server.Close() + + // invoke the API + resp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, netxlite.ECONNRESET) { + t.Fatal("unexpected error", err) + } + + // make sure the response is nil. + if resp != nil { + t.Fatal("expected nil response") + } + }) + + t.Run("when JSON parsing fails", func(t *testing.T) { + // create a server that returns an invalid JSON type + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("[]")) + })) + defer server.Close() + + // invoke the API + resp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if err.Error() != "json: cannot unmarshal array into Go value of type httpclientx.apiResponse" { + t.Fatal("unexpected error", err) + } + + // make sure the response is nil. + if resp != nil { + t.Fatal("expected nil response") + } + }) + + t.Run("on success", func(t *testing.T) { + // create a server that returns a legit response + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"Name": "simone", "Age": 41}`)) + })) + defer server.Close() + + // invoke the API + resp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if err != nil { + t.Fatal("unexpected error", err) + } + + // make sure the response is OK + expect := &apiResponse{Name: "simone", Age: 41} + if diff := cmp.Diff(expect, resp); diff != "" { + t.Fatal(diff) + } + }) +} + +// This test ensures that GetJSON sets correct HTTP headers +func TestGetJSONHeadersOkay(t *testing.T) { + var ( + gotheaders http.Header + gotmu sync.Mutex + ) + + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // save the headers + gotmu.Lock() + gotheaders = r.Header + gotmu.Unlock() + + // send a minimal 200 Ok response + w.WriteHeader(200) + w.Write([]byte(`{}`)) + })) + defer server.Close() + + // send the request and receive the response + apiresp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{ + Authorization: "scribai", + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + // we do not expect to see an error here + if err != nil { + t.Fatal(err) + } + + // given the handler, we expect to see an empty structure here + if apiresp.Age != 0 || apiresp.Name != "" { + t.Fatal("expected empty response") + } + + // make sure there are no data races + defer gotmu.Unlock() + gotmu.Lock() + + // make sure we have sent the authorization header + if value := gotheaders.Get("Authorization"); value != "scribai" { + t.Fatal("unexpected Authorization value", value) + } + + // now make sure we have sent user-agent + if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent { + t.Fatal("unexpected User-Agent value", value) + } + + // now make sure we have sent accept-encoding + if value := gotheaders.Get("Accept-Encoding"); value != "gzip" { + t.Fatal("unexpected Accept-Encoding value", value) + } +} + +// This test ensures GetJSON logs the response body at Debug level. +func TestGetJSONLoggingOkay(t *testing.T) { + // create a server that returns a legit response + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"Name": "simone", "Age": 41}`)) + })) + defer server.Close() + + // instantiate a logger that collects logs + logger := &testingx.Logger{} + + // invoke the API + resp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: logger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if err != nil { + t.Fatal("unexpected error", err) + } + + // make sure the response is OK + expect := &apiResponse{Name: "simone", Age: 41} + if diff := cmp.Diff(expect, resp); diff != "" { + t.Fatal(diff) + } + + // collect and verify the debug lines + debuglines := logger.DebugLines() + t.Log(debuglines) + if len(debuglines) != 1 { + t.Fatal("expected to see a single debug line") + } + if !strings.Contains(debuglines[0], "raw response body:") { + t.Fatal("did not see raw response body log line") + } +} + +// TestGetJSONCorrectlyRejectsNilValues ensures we correctly reject nil values. +func TestGetJSONCorrectlyRejectsNilValues(t *testing.T) { + + t.Run("when unmarshaling into a map", func(t *testing.T) { + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`null`)) + })) + defer server.Close() + + // invoke the API + resp, err := GetJSON[map[string]string](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, ErrIsNil) { + t.Fatal("unexpected error", err) + } + + // make sure resp is nil + if resp != nil { + t.Fatal("expected nil resp") + } + }) + + t.Run("when unmarshaling into a struct pointer", func(t *testing.T) { + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`null`)) + })) + defer server.Close() + + // invoke the API + resp, err := GetJSON[*apiResponse](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, ErrIsNil) { + t.Fatal("unexpected error", err) + } + + // make sure resp is nil + if resp != nil { + t.Fatal("expected nil resp") + } + }) + + t.Run("when unmarshaling into a slice", func(t *testing.T) { + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`null`)) + })) + defer server.Close() + + // invoke the API + resp, err := GetJSON[[]string](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, ErrIsNil) { + t.Fatal("unexpected error", err) + } + + // make sure resp is nil + if resp != nil { + t.Fatal("expected nil resp") + } + }) +} diff --git a/internal/httpclientx/getraw.go b/internal/httpclientx/getraw.go new file mode 100644 index 000000000..193df282e --- /dev/null +++ b/internal/httpclientx/getraw.go @@ -0,0 +1,36 @@ +package httpclientx + +// +// getraw.go - GET a raw response. +// + +import ( + "context" + "net/http" +) + +// GetRaw sends a GET request and reads a raw response. +// +// Arguments: +// +// - ctx is the cancellable context; +// +// - config is the config to use; +// +// - URL is the URL to use. +// +// This function either returns an error or a valid Output. +func GetRaw(ctx context.Context, URL string, config *Config) ([]byte, error) { + return NewOverlappedGetRaw(config).Run(ctx, URL) +} + +func getRaw(ctx context.Context, URL string, config *Config) ([]byte, error) { + // construct the request to use + req, err := http.NewRequestWithContext(ctx, "GET", URL, nil) + if err != nil { + return nil, err + } + + // get raw response body + return do(ctx, req, config) +} diff --git a/internal/httpclientx/getraw_test.go b/internal/httpclientx/getraw_test.go new file mode 100644 index 000000000..28c03d410 --- /dev/null +++ b/internal/httpclientx/getraw_test.go @@ -0,0 +1,165 @@ +package httpclientx + +import ( + "context" + "net/http" + "strings" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/testingx" +) + +func TestGetRaw(t *testing.T) { + t.Run("when we cannot create a request", func(t *testing.T) { + // create API call config + + rawrespbody, err := GetRaw( + context.Background(), + "\t", // <- invalid URL that we cannot parse + &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }, + ) + + t.Log(rawrespbody) + t.Log(err) + + if err.Error() != `parse "\t": net/url: invalid control character in URL` { + t.Fatal("unexpected error", err) + } + + if len(rawrespbody) != 0 { + t.Fatal("expected zero-length body") + } + }) + + t.Run("on success", func(t *testing.T) { + expected := []byte(`Bonsoir, Elliot`) + + // create a server that returns a legit response + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(expected) + })) + defer server.Close() + + rawrespbody, err := GetRaw(context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(rawrespbody) + t.Log(err) + + if err != nil { + t.Fatal("unexpected error", err) + } + + if diff := cmp.Diff(expected, rawrespbody); diff != "" { + t.Fatal(diff) + } + }) +} + +// This test ensures that GetRaw sets correct HTTP headers +func TestGetRawHeadersOkay(t *testing.T) { + var ( + gotheaders http.Header + gotmu sync.Mutex + ) + + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // save the headers + gotmu.Lock() + gotheaders = r.Header + gotmu.Unlock() + + // send a minimal 200 Ok response + w.WriteHeader(200) + w.Write([]byte(``)) + })) + defer server.Close() + + // send the request and receive the response + rawresp, err := GetRaw(context.Background(), server.URL, &Config{ + Authorization: "scribai", + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + // we do not expect to see an error here + if err != nil { + t.Fatal(err) + } + + // make sure the raw response is exactly what we expect to receive + if diff := cmp.Diff([]byte(``), rawresp); diff != "" { + t.Fatal("unexpected raw response") + } + + // make sure there are no data races + defer gotmu.Unlock() + gotmu.Lock() + + // make sure we have sent the authorization header + if value := gotheaders.Get("Authorization"); value != "scribai" { + t.Fatal("unexpected Authorization value", value) + } + + // now make sure we have sent user-agent + if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent { + t.Fatal("unexpected User-Agent value", value) + } + + // now make sure we have sent accept-encoding + if value := gotheaders.Get("Accept-Encoding"); value != "gzip" { + t.Fatal("unexpected Accept-Encoding value", value) + } +} + +// This test ensures GetRaw logs the response body at Debug level. +func TestGetRawLoggingOkay(t *testing.T) { + expected := []byte(`Bonsoir, Elliot`) + + // create a server that returns a legit response + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(expected) + })) + defer server.Close() + + // instantiate a logger that collects logs + logger := &testingx.Logger{} + + rawrespbody, err := GetRaw(context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: logger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(rawrespbody) + t.Log(err) + + if err != nil { + t.Fatal("unexpected error", err) + } + + if diff := cmp.Diff(expected, rawrespbody); diff != "" { + t.Fatal(diff) + } + + // collect and verify the debug lines + debuglines := logger.DebugLines() + t.Log(debuglines) + if len(debuglines) != 1 { + t.Fatal("expected to see a single debug line") + } + if !strings.Contains(debuglines[0], "raw response body:") { + t.Fatal("did not see raw response body log line") + } +} diff --git a/internal/httpclientx/getxml.go b/internal/httpclientx/getxml.go new file mode 100644 index 000000000..34ddac2ed --- /dev/null +++ b/internal/httpclientx/getxml.go @@ -0,0 +1,48 @@ +package httpclientx + +// +// getxml.go - GET an XML response. +// + +import ( + "context" + "encoding/xml" +) + +// GetXML sends a GET request and reads an XML response. +// +// Arguments: +// +// - ctx is the cancellable context; +// +// - URL is the URL to use; +// +// - config is the config to use. +// +// This function either returns an error or a valid Output. +func GetXML[Output any](ctx context.Context, URL string, config *Config) (Output, error) { + return NewOverlappedGetXML[Output](config).Run(ctx, URL) +} + +func getXML[Output any](ctx context.Context, URL string, config *Config) (Output, error) { + // read the raw body + rawrespbody, err := GetRaw(ctx, URL, config) + + // handle the case of error + if err != nil { + return zeroValue[Output](), err + } + + // parse the response body as JSON + var output Output + if err := xml.Unmarshal(rawrespbody, &output); err != nil { + return zeroValue[Output](), err + } + + // TODO(bassosimone): it's unclear to me whether output can be nil when unmarshaling + // XML input, since there is no "null" in XML. In any case, the code below checks for + // and avoids emitting nil, so I guess we should be fine here. + + // avoid returning nil pointers, maps, slices + return NilSafetyErrorIfNil(output) +} diff --git a/internal/httpclientx/getxml_test.go b/internal/httpclientx/getxml_test.go new file mode 100644 index 000000000..e7a85337d --- /dev/null +++ b/internal/httpclientx/getxml_test.go @@ -0,0 +1,201 @@ +package httpclientx + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/testingx" +) + +func TestGetXML(t *testing.T) { + t.Run("when GetRaw fails", func(t *testing.T) { + // create a server that RST connections + server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset()) + defer server.Close() + + // invoke the API + resp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, netxlite.ECONNRESET) { + t.Fatal("unexpected error", err) + } + + // make sure the response is nil. + if resp != nil { + t.Fatal("expected nil response") + } + }) + + t.Run("when XML parsing fails", func(t *testing.T) { + // create a server that returns an invalid XML file + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("[]")) + })) + defer server.Close() + + // invoke the API + resp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, io.EOF) { + t.Fatal("unexpected error", err) + } + + // make sure the response is nil. + if resp != nil { + t.Fatal("expected nil response") + } + }) + + t.Run("on success", func(t *testing.T) { + // create a server that returns a legit response + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`simone41`)) + })) + defer server.Close() + + // invoke the API + resp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if err != nil { + t.Fatal("unexpected error", err) + } + + // make sure the response is OK + expect := &apiResponse{Name: "simone", Age: 41} + if diff := cmp.Diff(expect, resp); diff != "" { + t.Fatal(diff) + } + }) +} + +// This test ensures that GetXML sets correct HTTP headers +func TestGetXMLHeadersOkay(t *testing.T) { + var ( + gotheaders http.Header + gotmu sync.Mutex + ) + + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // save the headers + gotmu.Lock() + gotheaders = r.Header + gotmu.Unlock() + + // send a minimal 200 Ok response + w.WriteHeader(200) + w.Write([]byte(``)) + })) + defer server.Close() + + // send the request and receive the response + apiresp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{ + Authorization: "scribai", + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + // we do not expect to see an error here + if err != nil { + t.Fatal(err) + } + + // given the handler, we expect to see an empty structure here + if apiresp.Age != 0 || apiresp.Name != "" { + t.Fatal("expected empty response") + } + + // make sure there are no data races + defer gotmu.Unlock() + gotmu.Lock() + + // make sure we have sent the authorization header + if value := gotheaders.Get("Authorization"); value != "scribai" { + t.Fatal("unexpected Authorization value", value) + } + + // now make sure we have sent user-agent + if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent { + t.Fatal("unexpected User-Agent value", value) + } + + // now make sure we have sent accept-encoding + if value := gotheaders.Get("Accept-Encoding"); value != "gzip" { + t.Fatal("unexpected Accept-Encoding value", value) + } +} + +// This test ensures GetXML logs the response body at Debug level. +func TestGetXMLLoggingOkay(t *testing.T) { + // create a server that returns a legit response + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`simone41`)) + })) + defer server.Close() + + // instantiate a logger that collects logs + logger := &testingx.Logger{} + + // invoke the API + resp, err := GetXML[*apiResponse](context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: logger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if err != nil { + t.Fatal("unexpected error", err) + } + + // make sure the response is OK + expect := &apiResponse{Name: "simone", Age: 41} + if diff := cmp.Diff(expect, resp); diff != "" { + t.Fatal(diff) + } + + // collect and verify the debug lines + debuglines := logger.DebugLines() + t.Log(debuglines) + if len(debuglines) != 1 { + t.Fatal("expected to see a single debug line") + } + if !strings.Contains(debuglines[0], "raw response body:") { + t.Fatal("did not see raw response body log line") + } +} diff --git a/internal/httpclientx/httpclientx.go b/internal/httpclientx/httpclientx.go new file mode 100644 index 000000000..a5e826e7d --- /dev/null +++ b/internal/httpclientx/httpclientx.go @@ -0,0 +1,103 @@ +// Package httpclientx contains extensions to more easily invoke HTTP APIs. +package httpclientx + +// +// httpclientx.go - common code +// + +import ( + "compress/gzip" + "context" + "io" + "net/http" + + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// ErrRequestFailed indicates that an HTTP request status indicates failure. +type ErrRequestFailed struct { + StatusCode int +} + +var _ error = &ErrRequestFailed{} + +// Error returns the error as a string. +// +// The string returned by this error starts with the httpx prefix for backwards +// compatibility with the legacy httpx package. +func (err *ErrRequestFailed) Error() string { + return "httpx: request failed" +} + +// zeroValue is a convenience function to return the zero value. +func zeroValue[T any]() T { + return *new(T) +} + +// newLimitReader is a wrapper for [io.LimitReader] that automatically +// sets the maximum readable amount of bytes. +func newLimitReader(r io.Reader) io.Reader { + return io.LimitReader(r, 1<<24) +} + +// do is the internal function to finish preparing the request and getting a raw response. +func do(ctx context.Context, req *http.Request, config *Config) ([]byte, error) { + // optionally assign authorization + if value := config.Authorization; value != "" { + req.Header.Set("Authorization", value) + } + + // assign the user agent + req.Header.Set("User-Agent", config.UserAgent) + + // say that we're accepting gzip encoded bodies + req.Header.Set("Accept-Encoding", "gzip") + + // get the response + resp, err := config.Client.Do(req) + + // handle the case of error + if err != nil { + return nil, err + } + + // eventually close the response body + defer resp.Body.Close() + + // Implementation note: here we choose to always read the response + // body before checking the status code because it helps a lot to log + // the response body received on failure when testing a backend + + var baseReader io.Reader = resp.Body + + // handle the case of gzip encoded body + if resp.Header.Get("Content-Encoding") == "gzip" { + gzreader, err := gzip.NewReader(baseReader) + if err != nil { + return nil, err + } + baseReader = gzreader + } + + // protect against unreasonably large response bodies + limitReader := newLimitReader(baseReader) + + // read the raw body + rawrespbody, err := netxlite.ReadAllContext(ctx, limitReader) + + // handle the case of failure + if err != nil { + return nil, err + } + + // log the response body for debugging purposes + config.Logger.Debugf("%s %s: raw response body: %s", req.Method, req.URL.String(), string(rawrespbody)) + + // handle the case of HTTP error + if resp.StatusCode != 200 { + return nil, &ErrRequestFailed{resp.StatusCode} + } + + // make sure we replace a nil slice with an empty slice + return NilSafetyAvoidNilBytesSlice(rawrespbody), nil +} diff --git a/internal/httpclientx/httpclientx_test.go b/internal/httpclientx/httpclientx_test.go new file mode 100644 index 000000000..26a7fbd46 --- /dev/null +++ b/internal/httpclientx/httpclientx_test.go @@ -0,0 +1,132 @@ +package httpclientx + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "net/http" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/testingx" +) + +func TestGzipDecompression(t *testing.T) { + t.Run("we correctly handle gzip encoding", func(t *testing.T) { + expected := []byte(`Bonsoir, Elliot!!!`) + + // create a server returning compressed content + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var buffer bytes.Buffer + writer := gzip.NewWriter(&buffer) + _ = runtimex.Try1(writer.Write(expected)) + runtimex.Try0(writer.Close()) + w.Header().Add("Content-Encoding", "gzip") + w.Write(buffer.Bytes()) + })) + defer server.Close() + + // make sure we can read it + respbody, err := GetRaw(context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(respbody) + t.Log(err) + + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(expected, respbody); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("we correctly handle the case where we cannot decode gzip", func(t *testing.T) { + expected := []byte(`Bonsoir, Elliot!!!`) + + // create a server pretending to return compressed content + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Encoding", "gzip") + w.Write(expected) + })) + defer server.Close() + + // attempt to get a response body + respbody, err := GetRaw(context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(respbody) + t.Log(err) + + if err.Error() != "gzip: invalid header" { + t.Fatal(err) + } + + if respbody != nil { + t.Fatal("expected nil response body") + } + }) +} + +func TestHTTPStatusCodeHandling(t *testing.T) { + server := testingx.MustNewHTTPServer(testingx.HTTPHandlerBlockpage451()) + defer server.Close() + + respbody, err := GetRaw(context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(respbody) + t.Log(err) + + if err.Error() != "httpx: request failed" { + t.Fatal(err) + } + + if respbody != nil { + t.Fatal("expected nil response body") + } + + var orig *ErrRequestFailed + if !errors.As(err, &orig) { + t.Fatal("not an *ErrRequestFailed instance") + } + if orig.StatusCode != 451 { + t.Fatal("unexpected status code", orig.StatusCode) + } +} + +func TestHTTPReadBodyErrorsHandling(t *testing.T) { + server := testingx.MustNewHTTPServer(testingx.HTTPHandlerResetWhileReadingBody()) + defer server.Close() + + respbody, err := GetRaw(context.Background(), server.URL, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(respbody) + t.Log(err) + + if !errors.Is(err, netxlite.ECONNRESET) { + t.Fatal("expected ECONNRESET, got", err) + } + + if respbody != nil { + t.Fatal("expected nil response body") + } +} diff --git a/internal/httpclientx/nilsafety.go b/internal/httpclientx/nilsafety.go new file mode 100644 index 000000000..c73323cb1 --- /dev/null +++ b/internal/httpclientx/nilsafety.go @@ -0,0 +1,31 @@ +package httpclientx + +import ( + "errors" + "reflect" +) + +// ErrIsNil indicates that [NilSafetyErrorIfNil] was passed a nil value. +var ErrIsNil = errors.New("nil map, pointer, or slice") + +// NilSafetyErrorIfNil returns [ErrIsNil] iff input is a nil map, struct, or slice. +// +// This mechanism prevents us from mistakenly sending to a server a literal JSON "null" and +// protects us from attempting to process a literal JSON "null" from a server. +func NilSafetyErrorIfNil[Type any](value Type) (Type, error) { + switch rv := reflect.ValueOf(value); rv.Kind() { + case reflect.Map, reflect.Pointer, reflect.Slice: + if rv.IsNil() { + return zeroValue[Type](), ErrIsNil + } + } + return value, nil +} + +// NilSafetyAvoidNilBytesSlice replaces a nil bytes slice with an empty slice. +func NilSafetyAvoidNilBytesSlice(input []byte) []byte { + if input == nil { + input = []byte{} + } + return input +} diff --git a/internal/httpclientx/nilsafety_test.go b/internal/httpclientx/nilsafety_test.go new file mode 100644 index 000000000..85551266e --- /dev/null +++ b/internal/httpclientx/nilsafety_test.go @@ -0,0 +1,122 @@ +package httpclientx + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestNilSafetyErrorIfNil(t *testing.T) { + + // testcase is a test case implemented by this function. + type testcase struct { + name string + input any + err error + output any + } + + cases := []testcase{{ + name: "with a nil map", + input: func() any { + var v map[string]string + return v + }(), + err: ErrIsNil, + output: nil, + }, { + name: "with a non-nil but empty map", + input: make(map[string]string), + err: nil, + output: make(map[string]string), + }, { + name: "with a non-nil non-empty map", + input: map[string]string{"a": "b"}, + err: nil, + output: map[string]string{"a": "b"}, + }, { + name: "with a nil pointer", + input: func() any { + var v *apiRequest + return v + }(), + err: ErrIsNil, + output: nil, + }, { + name: "with a non-nil empty pointer", + input: &apiRequest{}, + err: nil, + output: &apiRequest{}, + }, { + name: "with a non-nil non-empty pointer", + input: &apiRequest{UserID: 11}, + err: nil, + output: &apiRequest{UserID: 11}, + }, { + name: "with a nil slice", + input: func() any { + var v []int + return v + }(), + err: ErrIsNil, + output: nil, + }, { + name: "with a non-nil empty slice", + input: []int{}, + err: nil, + output: []int{}, + }, { + name: "with a non-nil non-empty slice", + input: []int{44}, + err: nil, + output: []int{44}, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + output, err := NilSafetyErrorIfNil(tc.input) + + switch { + case err == nil && tc.err == nil: + if diff := cmp.Diff(tc.output, output); diff != "" { + t.Fatal(diff) + } + return + + case err != nil && tc.err == nil: + t.Fatal("expected", tc.err.Error(), "got", err.Error()) + return + + case err == nil && tc.err != nil: + t.Fatal("expected", tc.err.Error(), "got", err.Error()) + return + + case err != nil && tc.err != nil: + if err.Error() != tc.err.Error() { + t.Fatal("expected", tc.err.Error(), "got", err.Error()) + } + return + } + }) + } +} + +func TestNilSafetyAvoidNilByteSlice(t *testing.T) { + t.Run("for nil byte slice", func(t *testing.T) { + output := NilSafetyAvoidNilBytesSlice(nil) + if output == nil { + t.Fatal("expected non-nil") + } + if len(output) != 0 { + t.Fatal("expected zero length") + } + }) + + t.Run("for non-nil byte slice", func(t *testing.T) { + expected := []byte{44} + output := NilSafetyAvoidNilBytesSlice(expected) + if diff := cmp.Diff(expected, output); diff != "" { + t.Fatal("not the same pointer") + } + }) +} diff --git a/internal/httpclientx/overlapped.go b/internal/httpclientx/overlapped.go new file mode 100644 index 000000000..a60e776fd --- /dev/null +++ b/internal/httpclientx/overlapped.go @@ -0,0 +1,163 @@ +package httpclientx + +// +// overlapped.go - overlapped operations. +// + +import ( + "context" + "errors" + "time" + + "github.com/ooni/probe-cli/v3/internal/erroror" +) + +// OverlappedDefaultScheduleInterval is the default schedule interval. After this interval +// has elapsed for a URL without seeing a success, we will schedule the next URL. +const OverlappedDefaultScheduleInterval = 15 * time.Second + +// Overlapped represents the possibility of overlapping HTTP calls for a set of +// functionally equivalent URLs, such that we start a new call if the previous one +// has failed to produce a result within the configured ScheduleInterval. +// +// # Limitations +// +// Under very bad networking conditions, [*Overlapped] would cause a new network +// call to start while the previous one is still in progress and very slowly downloading +// a response. A future implementation SHOULD probably account for this possibility. +type Overlapped[Output any] struct { + // RunFunc is the MANDATORY function that fetches the given URL. + // + // This field is typically initialized by [NewOverlappedGetJSON], [NewOverlappedGetRaw], + // [NewOverlappedGetXML], or [NewOverlappedPostJSON] to be the proper function that + // makes sense for the operation that you requested with the constructor. + // + // If you set it manually, you MUST modify it before calling [*Overlapped.Run]. + RunFunc func(ctx context.Context, URL string) (Output, error) + + // ScheduleInterval is the MANDATORY scheduling interval. + // + // This field is typically initialized by [NewOverlappedGetJSON], [NewOverlappedGetRaw], + // [NewOverlappedGetXML], or [NewOverlappedPostJSON] to be [OverlappedDefaultScheduleInterval]. + // + // If you set it manually, you MUST modify it before calling [*Overlapped.Run]. + ScheduleInterval time.Duration +} + +func newOverlappedWithFunc[Output any](fx func(context.Context, string) (Output, error)) *Overlapped[Output] { + return &Overlapped[Output]{ + RunFunc: fx, + ScheduleInterval: OverlappedDefaultScheduleInterval, + } +} + +// NewOverlappedGetJSON constructs a [*Overlapped] for calling [GetJSON] with multiple URLs. +func NewOverlappedGetJSON[Output any](config *Config) *Overlapped[Output] { + return newOverlappedWithFunc(func(ctx context.Context, URL string) (Output, error) { + return getJSON[Output](ctx, URL, config) + }) +} + +// NewOverlappedGetRaw constructs a [*Overlapped] for calling [GetRaw] with multiple URLs. +func NewOverlappedGetRaw(config *Config) *Overlapped[[]byte] { + return newOverlappedWithFunc(func(ctx context.Context, URL string) ([]byte, error) { + return getRaw(ctx, URL, config) + }) +} + +// NewOverlappedGetXML constructs a [*Overlapped] for calling [GetXML] with multiple URLs. +func NewOverlappedGetXML[Output any](config *Config) *Overlapped[Output] { + return newOverlappedWithFunc(func(ctx context.Context, URL string) (Output, error) { + return getXML[Output](ctx, URL, config) + }) +} + +// NewOverlappedPostJSON constructs a [*Overlapped] for calling [PostJSON] with multiple URLs. +func NewOverlappedPostJSON[Input, Output any](input Input, config *Config) *Overlapped[Output] { + return newOverlappedWithFunc(func(ctx context.Context, URL string) (Output, error) { + return postJSON[Input, Output](ctx, URL, input, config) + }) +} + +// ErrGenericOverlappedFailure indicates that a generic [*Overlapped] failure occurred. +var ErrGenericOverlappedFailure = errors.New("overlapped: generic failure") + +// Run runs the overlapped operations, returning the result of the first operation +// that succeeds and otherwise returning an error describing what happened. +// +// # Limitations +// +// This implementation creates a new goroutine for each provided URL under the assumption that +// the overall number of URLs is small. A future revision would address this issue. +func (ovx *Overlapped[Output]) Run(ctx context.Context, URLs ...string) (Output, error) { + // create cancellable context for early cancellation + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // construct channel for collecting the results + output := make(chan *erroror.Value[Output]) + + // schedule a measuring goroutine per URL. + for idx := 0; idx < len(URLs); idx++ { + go ovx.transact(ctx, idx, URLs[idx], output) + } + + // we expect to see exactly a response for each goroutine + var ( + firstOutput *Output + errorv []error + ) + for idx := 0; idx < len(URLs); idx++ { + // get a result from one of the goroutines + result := <-output + + // handle the error case + if result.Err != nil { + errorv = append(errorv, result.Err) + continue + } + + // possibly record the first success + if firstOutput == nil { + firstOutput = &result.Value + } + + // make sure we interrupt all the other goroutines + cancel() + } + + // handle the case of success + if firstOutput != nil { + return *firstOutput, nil + } + + // handle the case where there's no error + if len(errorv) <= 0 { + errorv = append(errorv, ErrGenericOverlappedFailure) + } + + // return zero value and errors list + return *new(Output), errors.Join(errorv...) +} + +// transact performs an HTTP transaction with the given URL and writes results to the output channel. +func (ovx *Overlapped[Output]) transact(ctx context.Context, idx int, URL string, output chan<- *erroror.Value[Output]) { + // wait for our time to start + // + // add one nanosecond to make sure the delay is always positive + timer := time.NewTimer(time.Duration(idx)*ovx.ScheduleInterval + time.Nanosecond) + defer timer.Stop() + select { + case <-ctx.Done(): + output <- &erroror.Value[Output]{Err: ctx.Err()} + return + case <-timer.C: + // fallthrough + } + + // obtain the results + value, err := ovx.RunFunc(ctx, URL) + + // emit the results + output <- &erroror.Value[Output]{Err: err, Value: value} +} diff --git a/internal/httpclientx/overlapped_test.go b/internal/httpclientx/overlapped_test.go new file mode 100644 index 000000000..350823e9f --- /dev/null +++ b/internal/httpclientx/overlapped_test.go @@ -0,0 +1,209 @@ +package httpclientx + +import ( + "context" + "errors" + "net/http" + "testing" + "time" + + "github.com/apex/log" + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/testingx" +) + +// Implementation note: because top-level functions such as GetRaw always use +// an [*Overlapped], we do not necessarily need to test that each top-level constructor +// are WAI; rather, we should focus on the mechanics of multiple URLs. + +func TestNewOverlappedPostJSONIsPerformingOverlappedCalls(t *testing.T) { + + // Scenario: + // + // - 0.th.ooni.org is SNI blocked + // - 1.th.ooni.org is SNI blocked + // - 2.th.ooni.org is SNI blocked + // - 3.th.ooni.org WAIs + + zeroTh := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset()) + defer zeroTh.Close() + + oneTh := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset()) + defer oneTh.Close() + + twoTh := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset()) + defer twoTh.Close() + + expectedResponse := &apiResponse{ + Age: 41, + Name: "sbs", + } + + threeTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(must.MarshalJSON(expectedResponse)) + })) + defer threeTh.Close() + + // Create client configuration. We don't care much about the + // JSON requests and reponses being aligned to reality. + + apiReq := &apiRequest{ + UserID: 117, + } + + overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{ + Authorization: "", // not relevant for this test + Client: http.DefaultClient, + Logger: log.Log, + UserAgent: model.HTTPHeaderUserAgent, + }) + + // make sure we set a low scheduling interval to make test faster + overlapped.ScheduleInterval = time.Second + + // Now we issue the requests and check we're getting the correct response. + + apiResp, err := overlapped.Run( + context.Background(), + zeroTh.URL, + oneTh.URL, + twoTh.URL, + threeTh.URL, + ) + + // we do not expect to see a failure because threeTh is WAI + if err != nil { + t.Fatal(err) + } + + // compare response to expectation + if diff := cmp.Diff(expectedResponse, apiResp); diff != "" { + t.Fatal(diff) + } +} + +func TestNewOverlappedPostJSONCancelsPendingCalls(t *testing.T) { + + // Scenario: + // + // - 0.th.ooni.org is WAI but slow + // - 1.th.ooni.org is WAI + // - 2.th.ooni.org is WAI + // - 3.th.ooni.org is WAI + + expectedResponse := &apiResponse{ + Age: 41, + Name: "sbs", + } + + slowwakeup := make(chan any) + + zeroTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-slowwakeup + w.Write(must.MarshalJSON(expectedResponse)) + })) + defer zeroTh.Close() + + oneTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-slowwakeup + w.Write(must.MarshalJSON(expectedResponse)) + })) + defer oneTh.Close() + + twoTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-slowwakeup + w.Write(must.MarshalJSON(expectedResponse)) + })) + defer twoTh.Close() + + threeTh := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-slowwakeup + w.Write(must.MarshalJSON(expectedResponse)) + })) + defer threeTh.Close() + + // Create client configuration. We don't care much about the + // JSON requests and reponses being aligned to reality. + + apiReq := &apiRequest{ + UserID: 117, + } + + overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{ + Authorization: "", // not relevant for this test + Client: http.DefaultClient, + Logger: log.Log, + UserAgent: model.HTTPHeaderUserAgent, + }) + + // make sure the schedule interval is high because we want + // all the goroutines but the first to be waiting for permission + // to fetch from their respective URLs. + overlapped.ScheduleInterval = 15 * time.Second + + // In the background we're going to emit slow wakeup signals at fixed intervals + // after an initial waiting interval, such that goroutines unblock in order + + go func() { + time.Sleep(250 * time.Millisecond) + for idx := 0; idx < 4; idx++ { + slowwakeup <- true + time.Sleep(250 * time.Millisecond) + } + close(slowwakeup) + }() + + // Now we issue the requests and check we're getting the correct response. + + apiResp, err := overlapped.Run( + context.Background(), + zeroTh.URL, + oneTh.URL, + twoTh.URL, + threeTh.URL, + ) + + // we do not expect to see a failure because threeTh is WAI + if err != nil { + t.Fatal(err) + } + + // compare response to expectation + if diff := cmp.Diff(expectedResponse, apiResp); diff != "" { + t.Fatal(diff) + } +} + +func TestNewOverlappedPostJSONWithNoURLs(t *testing.T) { + + // Create client configuration. We don't care much about the + // JSON requests and reponses being aligned to reality. + + apiReq := &apiRequest{ + UserID: 117, + } + + overlapped := NewOverlappedPostJSON[*apiRequest, *apiResponse](apiReq, &Config{ + Authorization: "", // not relevant for this test + Client: http.DefaultClient, + Logger: log.Log, + UserAgent: model.HTTPHeaderUserAgent, + }) + + // Now we issue the requests without any URLs and make sure + // the result we get is the generic overlapped error + + apiResp, err := overlapped.Run(context.Background()) + + // we do not expect to see a failure because threeTh is WAI + if !errors.Is(err, ErrGenericOverlappedFailure) { + t.Fatal("unexpected error", err) + } + + // we expect a nil response + if apiResp != nil { + t.Fatal("expected nil API response") + } +} diff --git a/internal/httpclientx/postjson.go b/internal/httpclientx/postjson.go new file mode 100644 index 000000000..5c559b184 --- /dev/null +++ b/internal/httpclientx/postjson.go @@ -0,0 +1,71 @@ +package httpclientx + +// +// postjson.go - POST a JSON request and read a JSON response. +// + +import ( + "bytes" + "context" + "encoding/json" + "net/http" +) + +// PostJSON sends a POST request with a JSON body and reads a JSON response. +// +// Arguments: +// +// - ctx is the cancellable context; +// +// - URL is the URL to use; +// +// - input is the input structure to JSON serialize as the request body; +// +// - config is the config to use. +// +// This function either returns an error or a valid Output. +func PostJSON[Input, Output any](ctx context.Context, URL string, input Input, config *Config) (Output, error) { + return NewOverlappedPostJSON[Input, Output](input, config).Run(ctx, URL) +} + +func postJSON[Input, Output any](ctx context.Context, URL string, input Input, config *Config) (Output, error) { + // ensure we're not sending a nil map, pointer, or slice + if _, err := NilSafetyErrorIfNil(input); err != nil { + return zeroValue[Output](), err + } + + // serialize the request body + rawreqbody, err := json.Marshal(input) + if err != nil { + return zeroValue[Output](), err + } + + // log the raw request body + config.Logger.Debugf("POST %s: raw request body: %s", URL, string(rawreqbody)) + + // construct the request to use + req, err := http.NewRequestWithContext(ctx, "POST", URL, bytes.NewReader(rawreqbody)) + if err != nil { + return zeroValue[Output](), err + } + + // assign the content type + req.Header.Set("Content-Type", "application/json") + + // get the raw response body + rawrespbody, err := do(ctx, req, config) + + // handle the case of error + if err != nil { + return zeroValue[Output](), err + } + + // parse the response body as JSON + var output Output + if err := json.Unmarshal(rawrespbody, &output); err != nil { + return zeroValue[Output](), err + } + + // avoid returning nil pointers, maps, slices + return NilSafetyErrorIfNil(output) +} diff --git a/internal/httpclientx/postjson_test.go b/internal/httpclientx/postjson_test.go new file mode 100644 index 000000000..bed543d8a --- /dev/null +++ b/internal/httpclientx/postjson_test.go @@ -0,0 +1,468 @@ +package httpclientx + +import ( + "context" + "errors" + "net/http" + "strings" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/testingx" +) + +type apiRequest struct { + UserID int +} + +func TestPostJSON(t *testing.T) { + t.Run("when we cannot marshal the request body", func(t *testing.T) { + // a channel cannot be serialized + req := make(chan int) + close(req) + + resp, err := PostJSON[chan int, *apiResponse](context.Background(), "", req, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + if err.Error() != `json: unsupported type: chan int` { + t.Fatal("unexpected error", err) + } + + if resp != nil { + t.Fatal("expected nil resp") + } + }) + + t.Run("when we cannot create a request", func(t *testing.T) { + req := &apiRequest{117} + + resp, err := PostJSON[*apiRequest, *apiResponse]( + context.Background(), + "\t", // <- invalid URL that we cannot parse + req, + &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }, + ) + + t.Log(resp) + t.Log(err) + + if err.Error() != `parse "\t": net/url: invalid control character in URL` { + t.Fatal("unexpected error", err) + } + + if resp != nil { + t.Fatal("expected nil resp") + } + }) + + t.Run("in case of HTTP failure", func(t *testing.T) { + server := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset()) + defer server.Close() + + req := &apiRequest{117} + + resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, req, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + if !errors.Is(err, netxlite.ECONNRESET) { + t.Fatal("unexpected error", err) + } + + if resp != nil { + t.Fatal("expected nil resp") + } + }) + + t.Run("when we cannot parse the response body", func(t *testing.T) { + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("[]")) + })) + defer server.Close() + + req := &apiRequest{117} + + resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, req, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if err.Error() != "json: cannot unmarshal array into Go value of type httpclientx.apiResponse" { + t.Fatal("unexpected error", err) + } + + // make sure the response is nil. + if resp != nil { + t.Fatal("expected nil response") + } + }) + + t.Run("on success", func(t *testing.T) { + req := &apiRequest{117} + + expect := &apiResponse{Name: "simone", Age: 41} + + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var gotreq apiRequest + data := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body)) + must.UnmarshalJSON(data, &gotreq) + if gotreq.UserID != req.UserID { + w.WriteHeader(404) + return + } + w.Write(must.MarshalJSON(expect)) + })) + defer server.Close() + + resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, req, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if err != nil { + t.Fatal("unexpected error", err) + } + + // make sure the response is OK. + if diff := cmp.Diff(expect, resp); diff != "" { + t.Fatal(diff) + } + }) +} + +// This test ensures that PostJSON sets correct HTTP headers and sends the right body. +func TestPostJSONCommunicationOkay(t *testing.T) { + var ( + gotheaders http.Header + gotrawbody []byte + gotmu sync.Mutex + ) + + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // read the raw response body + rawbody := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body)) + + // save the raw response body and headers + gotmu.Lock() + gotrawbody = rawbody + gotheaders = r.Header + gotmu.Unlock() + + // send a minimal 200 Ok response + w.WriteHeader(200) + w.Write([]byte(`{}`)) + })) + defer server.Close() + + // create and serialize the expected request body + apireq := &apiRequest{ + UserID: 117, + } + rawapireq := must.MarshalJSON(apireq) + + // send the request and receive the response + apiresp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, apireq, &Config{ + Authorization: "scribai", + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + // we do not expect to see an error here + if err != nil { + t.Fatal(err) + } + + // given the handler, we expect to see an empty structure here + if apiresp.Age != 0 || apiresp.Name != "" { + t.Fatal("expected empty response") + } + + // make sure there are no data races + defer gotmu.Unlock() + gotmu.Lock() + + // now verify what the handler has read as the raw request body + if diff := cmp.Diff(rawapireq, gotrawbody); diff != "" { + t.Fatal(diff) + } + + // make sure we have sent the authorization header + if value := gotheaders.Get("Authorization"); value != "scribai" { + t.Fatal("unexpected Authorization value", value) + } + + // now make sure we have sent content-type + if value := gotheaders.Get("Content-Type"); value != "application/json" { + t.Fatal("unexpected Content-Type value", value) + } + + // now make sure we have sent user-agent + if value := gotheaders.Get("User-Agent"); value != model.HTTPHeaderUserAgent { + t.Fatal("unexpected User-Agent value", value) + } + + // now make sure we have sent accept-encoding + if value := gotheaders.Get("Accept-Encoding"); value != "gzip" { + t.Fatal("unexpected Accept-Encoding value", value) + } +} + +// This test ensures PostJSON logs the request and response body at Debug level. +func TestPostJSONLoggingOkay(t *testing.T) { + req := &apiRequest{117} + + expect := &apiResponse{Name: "simone", Age: 41} + + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var gotreq apiRequest + data := runtimex.Try1(netxlite.ReadAllContext(r.Context(), r.Body)) + must.UnmarshalJSON(data, &gotreq) + if gotreq.UserID != req.UserID { + w.WriteHeader(404) + return + } + w.Write(must.MarshalJSON(expect)) + })) + defer server.Close() + + // instantiate a logger that collects logs + logger := &testingx.Logger{} + + resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, req, &Config{ + Client: http.DefaultClient, + Logger: logger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if err != nil { + t.Fatal("unexpected error", err) + } + + // make sure the response is OK. + if diff := cmp.Diff(expect, resp); diff != "" { + t.Fatal(diff) + } + + // collect and verify the debug lines + debuglines := logger.DebugLines() + t.Log(debuglines) + if len(debuglines) != 2 { + t.Fatal("expected to see a single debug line") + } + if !strings.Contains(debuglines[0], "raw request body:") { + t.Fatal("did not see raw request body log line") + } + if !strings.Contains(debuglines[1], "raw response body:") { + t.Fatal("did not see raw response body log line") + } +} + +// TestPostJSONCorrectlyRejectsNilValues ensures we do not emit and correctly reject nil values. +func TestPostJSONCorrectlyRejectsNilValues(t *testing.T) { + + t.Run("when sending a nil map", func(t *testing.T) { + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{}`)) + })) + defer server.Close() + + // invoke the API + resp, err := PostJSON[map[string]string, *apiResponse](context.Background(), server.URL, nil, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, ErrIsNil) { + t.Fatal("unexpected error", err) + } + + // make sure resp is nil + if resp != nil { + t.Fatal("expected nil resp") + } + }) + + t.Run("when sending a nil struct pointer", func(t *testing.T) { + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{}`)) + })) + defer server.Close() + + // invoke the API + resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, nil, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, ErrIsNil) { + t.Fatal("unexpected error", err) + } + + // make sure resp is nil + if resp != nil { + t.Fatal("expected nil resp") + } + }) + + t.Run("when sending a nil slice", func(t *testing.T) { + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{}`)) + })) + defer server.Close() + + // invoke the API + resp, err := PostJSON[[]string, *apiResponse](context.Background(), server.URL, nil, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, ErrIsNil) { + t.Fatal("unexpected error", err) + } + + // make sure resp is nil + if resp != nil { + t.Fatal("expected nil resp") + } + }) + + t.Run("when unmarshaling into a map", func(t *testing.T) { + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`null`)) + })) + defer server.Close() + + // create an empty request + apireq := &apiRequest{} + + // invoke the API + resp, err := PostJSON[*apiRequest, map[string]string](context.Background(), server.URL, apireq, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, ErrIsNil) { + t.Fatal("unexpected error", err) + } + + // make sure resp is nil + if resp != nil { + t.Fatal("expected nil resp") + } + }) + + t.Run("when unmarshaling into a struct pointer", func(t *testing.T) { + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`null`)) + })) + defer server.Close() + + // create an empty request + apireq := &apiRequest{} + + // invoke the API + resp, err := PostJSON[*apiRequest, *apiResponse](context.Background(), server.URL, apireq, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, ErrIsNil) { + t.Fatal("unexpected error", err) + } + + // make sure resp is nil + if resp != nil { + t.Fatal("expected nil resp") + } + }) + + t.Run("when unmarshaling into a slice", func(t *testing.T) { + server := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`null`)) + })) + defer server.Close() + + // create an empty request + apireq := &apiRequest{} + + // invoke the API + resp, err := PostJSON[*apiRequest, []string](context.Background(), server.URL, apireq, &Config{ + Client: http.DefaultClient, + Logger: model.DiscardLogger, + UserAgent: model.HTTPHeaderUserAgent, + }) + + t.Log(resp) + t.Log(err) + + // make sure that the error is the expected one + if !errors.Is(err, ErrIsNil) { + t.Fatal("unexpected error", err) + } + + // make sure resp is nil + if resp != nil { + t.Fatal("expected nil resp") + } + }) +}