From af68086ec52695ac76f1e2e4cb1e9d148c4ba85a Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 28 Sep 2023 21:09:37 +0200 Subject: [PATCH] cleanup: headers map now uses ArchivalMaybeBinaryString (#1333) This diff modifies the headers-as-a-map representation such that we use ArchivalMaybeBinaryString instead of ArchivalMaybeBinaryData. The overall idea is to migrate all users of the latter to the former, such that we can remove the latter and implement extra scrubbing for all ArchivalMaybeBinaryString users. Part of https://github.com/ooni/probe/issues/2531 --- internal/experiment/hhfm/hhfm_test.go | 16 +-- .../webconnectivity/httpanalysis_test.go | 118 +++++++++--------- internal/legacy/tracex/archival_test.go | 24 ++-- internal/measurexlite/http.go | 4 +- internal/measurexlite/http_test.go | 38 +++--- internal/model/archival.go | 34 +++-- internal/model/archival_test.go | 98 +++++++-------- 7 files changed, 165 insertions(+), 167 deletions(-) diff --git a/internal/experiment/hhfm/hhfm_test.go b/internal/experiment/hhfm/hhfm_test.go index 59d0b3007c..1e272ed9fc 100644 --- a/internal/experiment/hhfm/hhfm_test.go +++ b/internal/experiment/hhfm/hhfm_test.go @@ -750,9 +750,9 @@ func TestNewRequestEntryList(t *testing.T) { Key: "User-aGENT", Value: tracex.MaybeBinaryValue{Value: "foo/1.0"}, }}, - Headers: map[string]tracex.MaybeBinaryValue{ - "ContENt-tYPE": {Value: "text/plain"}, - "User-aGENT": {Value: "foo/1.0"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "ContENt-tYPE": "text/plain", + "User-aGENT": "foo/1.0", }, Method: "GeT", URL: "http://10.0.0.1/", @@ -773,7 +773,7 @@ func TestNewRequestEntryList(t *testing.T) { wantOut: []tracex.RequestEntry{{ Request: tracex.HTTPRequest{ Method: "GeT", - Headers: make(map[string]tracex.MaybeBinaryValue), + Headers: make(map[string]model.ArchivalMaybeBinaryString), HeadersList: []tracex.HTTPHeader{}, URL: "http://10.0.0.1/", }, @@ -820,9 +820,9 @@ func TestNewHTTPResponse(t *testing.T) { Key: "User-Agent", Value: tracex.MaybeBinaryValue{Value: "foo/1.0"}, }}, - Headers: map[string]tracex.MaybeBinaryValue{ - "Content-Type": {Value: "text/plain"}, - "User-Agent": {Value: "foo/1.0"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Content-Type": "text/plain", + "User-Agent": "foo/1.0", }, }, }, { @@ -834,7 +834,7 @@ func TestNewHTTPResponse(t *testing.T) { Body: model.ArchivalMaybeBinaryString(""), Code: 200, HeadersList: []tracex.HTTPHeader{}, - Headers: map[string]tracex.MaybeBinaryValue{}, + Headers: map[string]model.ArchivalMaybeBinaryString{}, }, }} for _, tt := range tests { diff --git a/internal/experiment/webconnectivity/httpanalysis_test.go b/internal/experiment/webconnectivity/httpanalysis_test.go index 737a969fff..77b54cba8a 100644 --- a/internal/experiment/webconnectivity/httpanalysis_test.go +++ b/internal/experiment/webconnectivity/httpanalysis_test.go @@ -366,8 +366,8 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, }, @@ -382,8 +382,8 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, }, @@ -402,8 +402,8 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, }, @@ -425,9 +425,9 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, - "Antani": {Value: "MASCETTI"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Date": "Mon Jul 13 21:10:08 CEST 2020", + "Antani": "MASCETTI", }, Code: 200, }, @@ -450,9 +450,9 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, - "Antani": {Value: "MASCETTI"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Date": "Mon Jul 13 21:10:08 CEST 2020", + "Antani": "MASCETTI", }, Code: 200, }, @@ -475,19 +475,19 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Accept-Ranges": {Value: "bytes"}, - "Age": {Value: "404727"}, - "Cache-Control": {Value: "max-age=604800"}, - "Content-Length": {Value: "1256"}, - "Content-Type": {Value: "text/html; charset=UTF-8"}, - "Date": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"}, - "Etag": {Value: "\"3147526947\""}, - "Expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"}, - "Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"}, - "Server": {Value: "ECS (dcb/7F3C)"}, - "Vary": {Value: "Accept-Encoding"}, - "X-Cache": {Value: "HIT"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept-Ranges": "bytes", + "Age": "404727", + "Cache-Control": "max-age=604800", + "Content-Length": "1256", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 14 Jul 2020 22:26:09 GMT", + "Etag": "\"3147526947\"", + "Expires": "Tue, 21 Jul 2020 22:26:09 GMT", + "Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT", + "Server": "ECS (dcb/7F3C)", + "Vary": "Accept-Encoding", + "X-Cache": "HIT", }, Code: 200, }, @@ -522,18 +522,18 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Accept-Ranges": {Value: "bytes"}, - "Age": {Value: "404727"}, - "Cache-Control": {Value: "max-age=604800"}, - "Content-Length": {Value: "1256"}, - "Content-Type": {Value: "text/html; charset=UTF-8"}, - "Date": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"}, - "Etag": {Value: "\"3147526947\""}, - "Expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"}, - "Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"}, - "Server": {Value: "ECS (dcb/7F3C)"}, - "Vary": {Value: "Accept-Encoding"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept-Ranges": "bytes", + "Age": "404727", + "Cache-Control": "max-age=604800", + "Content-Length": "1256", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 14 Jul 2020 22:26:09 GMT", + "Etag": "\"3147526947\"", + "Expires": "Tue, 21 Jul 2020 22:26:09 GMT", + "Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT", + "Server": "ECS (dcb/7F3C)", + "Vary": "Accept-Encoding", }, Code: 200, }, @@ -567,17 +567,17 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Accept-Ranges": {Value: "bytes"}, - "Age": {Value: "404727"}, - "Cache-Control": {Value: "max-age=604800"}, - "Content-Type": {Value: "text/html; charset=UTF-8"}, - "Date": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"}, - "Etag": {Value: "\"3147526947\""}, - "Expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"}, - "Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"}, - "Server": {Value: "ECS (dcb/7F3C)"}, - "Vary": {Value: "Accept-Encoding"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept-Ranges": "bytes", + "Age": "404727", + "Cache-Control": "max-age=604800", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 14 Jul 2020 22:26:09 GMT", + "Etag": "\"3147526947\"", + "Expires": "Tue, 21 Jul 2020 22:26:09 GMT", + "Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT", + "Server": "ECS (dcb/7F3C)", + "Vary": "Accept-Encoding", }, Code: 200, }, @@ -608,17 +608,17 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "accept-ranges": {Value: "bytes"}, - "AGE": {Value: "404727"}, - "cache-Control": {Value: "max-age=604800"}, - "Content-TyPe": {Value: "text/html; charset=UTF-8"}, - "DatE": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"}, - "etag": {Value: "\"3147526947\""}, - "expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"}, - "Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"}, - "SerVer": {Value: "ECS (dcb/7F3C)"}, - "Vary": {Value: "Accept-Encoding"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "accept-ranges": "bytes", + "AGE": "404727", + "cache-Control": "max-age=604800", + "Content-TyPe": "text/html; charset=UTF-8", + "DatE": "Tue, 14 Jul 2020 22:26:09 GMT", + "etag": "\"3147526947\"", + "expires": "Tue, 21 Jul 2020 22:26:09 GMT", + "Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT", + "SerVer": "ECS (dcb/7F3C)", + "Vary": "Accept-Encoding", }, Code: 200, }, diff --git a/internal/legacy/tracex/archival_test.go b/internal/legacy/tracex/archival_test.go index 28274f08bc..2a8ad7a6e4 100644 --- a/internal/legacy/tracex/archival_test.go +++ b/internal/legacy/tracex/archival_test.go @@ -178,15 +178,15 @@ func TestNewRequestList(t *testing.T) { Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]MaybeBinaryValue{ - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", URL: "https://www.example.com/result", }, Response: HTTPResponse{ HeadersList: []HTTPHeader{}, - Headers: make(map[string]MaybeBinaryValue), + Headers: make(map[string]model.ArchivalMaybeBinaryString), }, T: 0.02, }, { @@ -198,8 +198,8 @@ func TestNewRequestList(t *testing.T) { Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]MaybeBinaryValue{ - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "User-Agent": "miniooni/0.1.0-dev", }, Method: "POST", URL: "https://www.example.com/submit", @@ -213,8 +213,8 @@ func TestNewRequestList(t *testing.T) { Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]MaybeBinaryValue{ - "Server": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Server": "miniooni/0.1.0-dev", }, Locations: nil, }, @@ -248,8 +248,8 @@ func TestNewRequestList(t *testing.T) { Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]MaybeBinaryValue{ - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", URL: "https://www.example.com/", @@ -272,9 +272,9 @@ func TestNewRequestList(t *testing.T) { Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]MaybeBinaryValue{ - "Server": {Value: "miniooni/0.1.0-dev"}, - "Location": {Value: "https://x.example.com"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Server": "miniooni/0.1.0-dev", + "Location": "https://x.example.com", }, Locations: []string{ "https://x.example.com", "https://y.example.com", diff --git a/internal/measurexlite/http.go b/internal/measurexlite/http.go index 448d627a28..8c196926ce 100644 --- a/internal/measurexlite/http.go +++ b/internal/measurexlite/http.go @@ -95,7 +95,7 @@ func newHTTPRequestHeaderList(req *http.Request) []model.ArchivalHTTPHeader { // newHTTPRequestHeaderMap calls newHTTPHeaderMap with the request headers or // return an empty map in case the request is nil. -func newHTTPRequestHeaderMap(req *http.Request) map[string]model.ArchivalMaybeBinaryData { +func newHTTPRequestHeaderMap(req *http.Request) map[string]model.ArchivalMaybeBinaryString { m := http.Header{} if req != nil { m = req.Header @@ -139,7 +139,7 @@ func newHTTPResponseHeaderList(resp *http.Response) (out []model.ArchivalHTTPHea // newHTTPResponseHeaderMap calls newHTTPHeaderMap with the request headers or // return an empty map in case the request is nil. -func newHTTPResponseHeaderMap(resp *http.Response) (out map[string]model.ArchivalMaybeBinaryData) { +func newHTTPResponseHeaderMap(resp *http.Response) (out map[string]model.ArchivalMaybeBinaryString) { m := http.Header{} if resp != nil { m = resp.Header diff --git a/internal/measurexlite/http_test.go b/internal/measurexlite/http_test.go index 6c41ff1a21..c6cd69f679 100644 --- a/internal/measurexlite/http_test.go +++ b/internal/measurexlite/http_test.go @@ -61,7 +61,7 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { Body: model.ArchivalMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryData{}, + Headers: map[string]model.ArchivalMaybeBinaryString{}, Method: "", Tor: model.ArchivalHTTPTor{}, Transport: "", @@ -72,7 +72,7 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryData{}, + Headers: map[string]model.ArchivalMaybeBinaryString{}, Locations: []string{}, }, T0: 0, @@ -130,9 +130,9 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {Value: "*/*"}, - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept": "*/*", + "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", Tor: model.ArchivalHTTPTor{}, @@ -144,7 +144,7 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryData{}, + Headers: map[string]model.ArchivalMaybeBinaryString{}, Locations: []string{}, }, T0: 0.25, @@ -205,9 +205,9 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {Value: "*/*"}, - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept": "*/*", + "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", Tor: model.ArchivalHTTPTor{}, @@ -231,9 +231,9 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { Value: "Apache", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Content-Type": {Value: "text/html; charset=iso-8859-1"}, - "Server": {Value: "Apache"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Content-Type": "text/html; charset=iso-8859-1", + "Server": "Apache", }, Locations: []string{}, }, @@ -303,9 +303,9 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {Value: "*/*"}, - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept": "*/*", + "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", Tor: model.ArchivalHTTPTor{}, @@ -332,10 +332,10 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { Value: "Apache", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Content-Type": {Value: "text/html; charset=iso-8859-1"}, - "Location": {Value: "/v2/index.html"}, - "Server": {Value: "Apache"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Content-Type": "text/html; charset=iso-8859-1", + "Location": "/v2/index.html", + "Server": "Apache", }, Locations: []string{ "https://dns.google/v2/index.html", diff --git a/internal/model/archival.go b/internal/model/archival.go index 2057bb8a17..891feb85ac 100644 --- a/internal/model/archival.go +++ b/internal/model/archival.go @@ -325,14 +325,14 @@ type ArchivalHTTPRequestResult struct { // Headers are a map in Web Connectivity data format but // we have added support for a list since January 2020. type ArchivalHTTPRequest struct { - Body ArchivalMaybeBinaryString `json:"body"` - BodyIsTruncated bool `json:"body_is_truncated"` - HeadersList []ArchivalHTTPHeader `json:"headers_list"` - Headers map[string]ArchivalMaybeBinaryData `json:"headers"` - Method string `json:"method"` - Tor ArchivalHTTPTor `json:"tor"` - Transport string `json:"x_transport"` - URL string `json:"url"` + Body ArchivalMaybeBinaryString `json:"body"` + BodyIsTruncated bool `json:"body_is_truncated"` + HeadersList []ArchivalHTTPHeader `json:"headers_list"` + Headers map[string]ArchivalMaybeBinaryString `json:"headers"` + Method string `json:"method"` + Tor ArchivalHTTPTor `json:"tor"` + Transport string `json:"x_transport"` + URL string `json:"url"` } // ArchivalHTTPResponse contains an HTTP response. @@ -340,11 +340,11 @@ type ArchivalHTTPRequest struct { // Headers are a map in Web Connectivity data format but // we have added support for a list since January 2020. type ArchivalHTTPResponse struct { - Body ArchivalMaybeBinaryString `json:"body"` - BodyIsTruncated bool `json:"body_is_truncated"` - Code int64 `json:"code"` - HeadersList []ArchivalHTTPHeader `json:"headers_list"` - Headers map[string]ArchivalMaybeBinaryData `json:"headers"` + Body ArchivalMaybeBinaryString `json:"body"` + BodyIsTruncated bool `json:"body_is_truncated"` + Code int64 `json:"code"` + HeadersList []ArchivalHTTPHeader `json:"headers_list"` + Headers map[string]ArchivalMaybeBinaryString `json:"headers"` // The following fields are not serialised but are useful to simplify // analysing the measurements in telegram, whatsapp, etc. @@ -381,13 +381,11 @@ func ArchivalNewHTTPHeadersList(source http.Header) (out []ArchivalHTTPHeader) { } // ArchivalNewHTTPHeadersMap creates a map representation of HTTP headers -func ArchivalNewHTTPHeadersMap(header http.Header) (out map[string]ArchivalMaybeBinaryData) { - out = make(map[string]ArchivalMaybeBinaryData) +func ArchivalNewHTTPHeadersMap(header http.Header) (out map[string]ArchivalMaybeBinaryString) { + out = make(map[string]ArchivalMaybeBinaryString) for key, values := range header { for _, value := range values { - out[key] = ArchivalMaybeBinaryData{ - Value: value, - } + out[key] = ArchivalMaybeBinaryString(value) break // just the first header } } diff --git a/internal/model/archival_test.go b/internal/model/archival_test.go index f88a2afddc..814461ee48 100644 --- a/internal/model/archival_test.go +++ b/internal/model/archival_test.go @@ -1462,9 +1462,9 @@ func TestArchivalHTTPRequestResult(t *testing.T) { Value: "miniooni/0.1.0", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, - "User-Agent": {"miniooni/0.1.0"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", }, Method: "GET", Tor: model.ArchivalHTTPTor{ @@ -1488,9 +1488,9 @@ func TestArchivalHTTPRequestResult(t *testing.T) { Key: "Server", Value: model.ArchivalMaybeBinaryData{"Apache"}, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Age": {"131833"}, - "Server": {"Apache"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Age": "131833", + "Server": "Apache", }, Locations: nil, }, @@ -1528,9 +1528,9 @@ func TestArchivalHTTPRequestResult(t *testing.T) { Value: "miniooni/0.1.0", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, - "User-Agent": {"miniooni/0.1.0"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", }, Method: "GET", Tor: model.ArchivalHTTPTor{ @@ -1546,7 +1546,7 @@ func TestArchivalHTTPRequestResult(t *testing.T) { BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryData{}, + Headers: map[string]model.ArchivalMaybeBinaryString{}, Locations: nil, }, T0: 0.4, @@ -1594,10 +1594,10 @@ func TestArchivalHTTPRequestResult(t *testing.T) { Value: string(archivalBinaryInput[7:14]), }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, - "User-Agent": {"miniooni/0.1.0"}, - "Antani": {string(archivalBinaryInput[:7])}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", + "Antani": model.ArchivalMaybeBinaryString(archivalBinaryInput[:7]), }, Method: "GET", Tor: model.ArchivalHTTPTor{ @@ -1631,10 +1631,10 @@ func TestArchivalHTTPRequestResult(t *testing.T) { Value: string(archivalBinaryInput[21:28]), }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Age": {"131833"}, - "Server": {"Apache"}, - "Mascetti": {string(archivalEncodedBinaryInput[14:21])}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Age": "131833", + "Server": "Apache", + "Mascetti": model.ArchivalMaybeBinaryString(archivalEncodedBinaryInput[14:21]), }, Locations: nil, }, @@ -1698,13 +1698,13 @@ func TestArchivalHTTPRequestResult(t *testing.T) { Value: "[2606:2800:220:1:248:1893:25c8:1946]:5222", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, - "User-Agent": {"miniooni/0.1.0"}, - "AntaniV4": {"130.192.91.211"}, - "AntaniV6": {"2606:2800:220:1:248:1893:25c8:1946"}, - "AntaniV4Epnt": {"130.192.91.211:443"}, - "AntaniV6Epnt": {"[2606:2800:220:1:248:1893:25c8:1946]:5222"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", + "AntaniV4": "130.192.91.211", + "AntaniV6": "2606:2800:220:1:248:1893:25c8:1946", + "AntaniV4Epnt": "130.192.91.211:443", + "AntaniV6Epnt": "[2606:2800:220:1:248:1893:25c8:1946]:5222", }, Method: "GET", Tor: model.ArchivalHTTPTor{ @@ -1748,13 +1748,13 @@ func TestArchivalHTTPRequestResult(t *testing.T) { Value: "[2606:2800:220:1:248:1893:25c8:1946]:5222", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Age": {"131833"}, - "Server": {"Apache"}, - "MascettiV4": {"130.192.91.211"}, - "MascettiV6": {"2606:2800:220:1:248:1893:25c8:1946"}, - "MascettiV4Epnt": {"130.192.91.211:443"}, - "MascettiV6Epnt": {"[2606:2800:220:1:248:1893:25c8:1946]:5222"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Age": "131833", + "Server": "Apache", + "MascettiV4": "130.192.91.211", + "MascettiV6": "2606:2800:220:1:248:1893:25c8:1946", + "MascettiV4Epnt": "130.192.91.211:443", + "MascettiV6Epnt": "[2606:2800:220:1:248:1893:25c8:1946]:5222", }, Locations: nil, }, @@ -1842,9 +1842,9 @@ func TestArchivalHTTPRequestResult(t *testing.T) { Value: "miniooni/0.1.0", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, - "User-Agent": {"miniooni/0.1.0"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", }, Method: "GET", Tor: model.ArchivalHTTPTor{ @@ -1868,9 +1868,9 @@ func TestArchivalHTTPRequestResult(t *testing.T) { Key: "Server", Value: model.ArchivalMaybeBinaryData{"Apache"}, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Age": {"131833"}, - "Server": {"Apache"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Age": "131833", + "Server": "Apache", }, Locations: nil, }, @@ -1905,9 +1905,9 @@ func TestArchivalHTTPRequestResult(t *testing.T) { Value: "miniooni/0.1.0", }, }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, - "User-Agent": {"miniooni/0.1.0"}, + Headers: map[string]model.ArchivalMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", }, Method: "GET", Tor: model.ArchivalHTTPTor{ @@ -1923,7 +1923,7 @@ func TestArchivalHTTPRequestResult(t *testing.T) { BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryData{}, + Headers: map[string]model.ArchivalMaybeBinaryString{}, Locations: nil, }, T0: 0.4, @@ -2212,17 +2212,17 @@ func TestArchivalNewHTTPHeadersMap(t *testing.T) { type testcase struct { name string input http.Header - expect map[string]model.ArchivalMaybeBinaryData + expect map[string]model.ArchivalMaybeBinaryString } cases := []testcase{{ name: "with nil input", input: nil, - expect: map[string]model.ArchivalMaybeBinaryData{}, + expect: map[string]model.ArchivalMaybeBinaryString{}, }, { name: "with empty input", input: map[string][]string{}, - expect: map[string]model.ArchivalMaybeBinaryData{}, + expect: map[string]model.ArchivalMaybeBinaryString{}, }, { name: "common case", input: map[string][]string{ @@ -2230,10 +2230,10 @@ func TestArchivalNewHTTPHeadersMap(t *testing.T) { "Via": {"a", "b", "c"}, "User-Agent": {"miniooni/0.1.0"}, }, - expect: map[string]model.ArchivalMaybeBinaryData{ - "Content-Type": {"text/html; charset=utf-8"}, - "Via": {"a"}, - "User-Agent": {"miniooni/0.1.0"}, + expect: map[string]model.ArchivalMaybeBinaryString{ + "Content-Type": "text/html; charset=utf-8", + "Via": "a", + "User-Agent": "miniooni/0.1.0", }, }}