feat: implement proper request sanitization

.github/workflows/ci.yaml

@@ -39,4 +39,4 @@ jobs:
       - name: Test
         run: |
           go get -d -t ./...
-          make test
+          go test ./...

client.go

@@ -75,7 +75,7 @@ func NewDefaultTestClient(t *testing.T, opts ...Option) *http.Client {
 		t.Log("test record request mode is on - all requests will be recorded")
 		transport = newRecordTransport(cfg.parentHTTPClient.Transport, cfg.namingScheme, cfg.requestSanitizer)
 	} else {
-		t.Log("test record request mode is off - requests will be read from disk")
+		t.Log("test record request mode is off - requests will be read from a directory if available, otherwise they will fail")
 		transport = newReplayTransport(cfg.namingScheme)
 	}
 	cfg.parentHTTPClient.Transport = transport

examples/nasa/testdata/TestImagesLister_ListNASAImages/0.req.http

@@ -1,4 +1,4 @@
-GET https://api.nasa.gov/planetary/apod?api_key=DEMO_KEY&end_date=2021-01-04&start_date=2021-01-01 HTTP/1.1
+GET https://api.nasa.gov/planetary/apod?api_key=SANITIZED&end_date=2021-01-04&start_date=2021-01-01 HTTP/1.1
 Host: api.nasa.gov
 User-Agent: Go-http-client/1.1
 

examples/nasa/testdata/TestImagesLister_ListNASAImages/0.resp.http

@@ -1,19 +1,19 @@
 HTTP/2.0 200 OK
 Access-Control-Allow-Origin: *
 Access-Control-Expose-Headers: X-RateLimit-Limit, X-RateLimit-Remaining
-Age: 1
+Age: 0
 Content-Type: application/json
-Date: Tue, 12 Mar 2024 20:22:47 GMT
+Date: Tue, 12 Mar 2024 21:53:56 GMT
 Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
 Vary: Accept-Encoding
 Via: https/1.1 api-umbrella (ApacheTrafficServer [cMsSf ])
-X-Api-Umbrella-Request-Id: ce6hafbahbq5e9ol59e0
+X-Api-Umbrella-Request-Id: ce6ik6kkr58bsfa9no2g
 X-Cache: MISS
 X-Content-Type-Options: nosniff
 X-Frame-Options: DENY
 X-Ratelimit-Limit: 40
-X-Ratelimit-Remaining: 37
-X-Vcap-Request-Id: 5b84c60d-d28c-4c95-70c0-6e2a4f8e245f
+X-Ratelimit-Remaining: 39
+X-Vcap-Request-Id: 1c66c096-6f05-4e53-6ec5-51af8b6ab64e
 X-Xss-Protection: 1; mode=block
 
 [{"copyright":"Petr Horalek","date":"2021-01-01","explanation":"The South Celestial Pole is easy to spot in star trail images of the southern sky. The extension of Earth's axis of rotation to the south, it's at the center of all the southern star trail arcs. In this starry panorama streching about 60 degrees across deep southern skies the South Celestial Pole is somewhere near the middle though, flanked by bright galaxies and southern celestial gems. Across the top of the frame are the stars and nebulae along the plane of our own Milky Way Galaxy. Gamma Crucis, a yellowish giant star heads the Southern Cross near top center, with the dark expanse of the Coalsack nebula tucked under the cross arm on the left. Eta Carinae and the reddish glow of the Great Carina Nebula shine along the galactic plane near the right edge. At the bottom are the Large and Small Magellanic clouds, external galaxies in their own right and satellites of the mighty Milky Way. A line from Gamma Crucis through the blue star at the bottom of the southern cross, Alpha Crucis, points toward the South Celestial Pole, but where exactly is it? Just look for south pole star Sigma Octantis. Analog to Polaris the north pole star, Sigma Octantis is little over one degree fom the the South Celestial pole.","hdurl":"https://apod.nasa.gov/apod/image/2101/2020_12_16_Kujal_Jizni_Pol_1500px-3.png","media_type":"image","service_version":"v1","title":"Galaxies and the South Celestial Pole","url":"https://apod.nasa.gov/apod/image/2101/2020_12_16_Kujal_Jizni_Pol_1500px-3.jpg"},{"copyright":"Mike Smolinsky","date":"2021-01-02","explanation":"In the mid 19th century, one of the first photographic technologies used to record the lunar surface was the wet-plate collodion process, notably employed by British astronomer Warren De la Rue. To capture an image, a thick, transparent mixture was used to coat a glass plate, sensitized with silver nitrate, exposed at the telescope, and then developed to create a negative image on the plate. To maintain photographic sensitivity, the entire process, from coating to exposure to developing, had to be completed before the plate dried, in a span of about 10 to 15 minutes. This modern version of a wet-plate collodion image celebrates lunar photography's early days, reproducing the process using modern chemicals to coat a glass plate from a 21st century hardware store. Captured last November 28 with an 8x10 view camera and backyard telescope, it faithfully records large craters, bright rays, and dark, smooth mare of the waxing gibbous Moon. Subsequently digitized, the image on the plate was 8.5 centimeters in diameter and exposed while tracking for 2 minutes. The wet plate's effective photographic sensitivity was about ISO 1.  In your smart phone, the camera sensor probably has a photographic sensitivity range of ISO 100 to 6400 (and needs to be kept dry ...).","hdurl":"https://apod.nasa.gov/apod/image/2101/WetCollodionLunar112820SMO.jpg","media_type":"image","service_version":"v1","title":"21st Century Wet Collodion Moon","url":"https://apod.nasa.gov/apod/image/2101/WetCollodionLunar112820SMO_1024.jpg"},{"copyright":"\nHallgrimur P. Helgason;\n Rollover Annotation:  Judy Schmidt\n","date":"2021-01-03","explanation":"All of the other aurora watchers had gone home. By 3:30 am in Iceland, on a quiet September night, much of that night's auroras had died down. Suddenly, unexpectedly, a new burst of particles streamed down from space, lighting up the Earth's atmosphere once again. This time, surprisingly, pareidoliacally, the night lit up with an  amazing shape reminiscent of a giant phoenix. With camera equipment at the ready, two quick sky images were taken, followed immediately by a third of the land.  The mountain in the background is Helgafell, while the small foreground river is called Kald\u00e1, both located about 30 kilometers north of Iceland's capital Reykjav\u00edk. Seasoned skywatchers will note that just above the mountain, toward the left, is the constellation of Orion, while the Pleiades star cluster is also visible just above the frame center.  The 2016 aurora, which lasted only a minute and was soon gone forever --  would possibly be dismissed as an fanciful fable -- were it not captured in the featured, digitally-composed, image mosaic.    Almost Hyperspace: Random APOD Generator","hdurl":"https://apod.nasa.gov/apod/image/2101/PhoenixAurora_Helgason_3130.jpg","media_type":"image","service_version":"v1","title":"A Phoenix Aurora over Iceland","url":"https://apod.nasa.gov/apod/image/2101/PhoenixAurora_Helgason_960.jpg"},{"copyright":"\nMatthew G McHarg, Jacob L Harley, \nThomas Ashcraft, Hans Nielsen\n","date":"2021-01-04","explanation":"What causes sprite lightning? Mysterious bursts of light in the sky that momentarily resemble gigantic jellyfish have been recorded for over 30 years, but apart from a general association with positive cloud-to-ground lightning, their root cause remains unknown. Some thunderstorms have them -- most don't.  Recently, however, high speed videos are better detailing how sprites actually develop.  The featured video, captured in mid-2019, is fast enough -- at about 100,000 frames per second -- to time-resolve several sprite \"bombs\" dropping and developing into the multi-pronged streamers that appear on still images. Unfortunately, the visual clues provided by videos like these do not fully resolve the sprite origins mystery. High speed vidoes do indicate to some researchers, though, that sprites are more likely to occur when plasma irregularities exist in the upper atmosphere.    Astrophysicists: Browse 2,300+ codes in the Astrophysics Source Code Library","media_type":"video","service_version":"v1","title":"Sprite Lightning at 100,000 Frames Per Second","url":"https://www.youtube.com/embed/zS_XgF9i8tc?rel=0"}]

record_transport.go

@@ -12,10 +12,15 @@ import (
 type recordTransport struct {
 	httpTransport http.RoundTripper
 	namingScheme  NamingScheme
+	sanitizer     RequestSanitizer
 }
 
 func newRecordTransport(httpTransport http.RoundTripper, namingScheme NamingScheme, sanitizer RequestSanitizer) *recordTransport {
-	return &recordTransport{httpTransport: httpTransport, namingScheme: namingScheme}
+	return &recordTransport{
+		httpTransport: httpTransport,
+		namingScheme:  namingScheme,
+		sanitizer:     sanitizer,
+	}
 }
 
 func (d *recordTransport) RoundTrip(req *http.Request) (*http.Response, error) {
@@ -42,28 +47,28 @@ func (d *recordTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 }
 
 func (d *recordTransport) dumpReqToFile(name string, req *http.Request) (*http.Request, error) {
-	var buf bytes.Buffer
-	err := req.WriteProxy(&buf)
-	if err != nil {
-		return nil, err
+	if req.Body == nil {
+		req.Body = http.NoBody
 	}
+	reqClone := req.Clone(req.Context())
+	var originalReqBody bytes.Buffer
+	teeReader := io.TeeReader(req.Body, &originalReqBody)
+	reqClone.Body = io.NopCloser(teeReader)
+	sanitizedReq := d.sanitizer.SanitizeRequest(reqClone)
 
 	f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
 	if err != nil {
 		return nil, err
 	}
-	defer f.Close()
-	reqBytes := buf.Bytes()
-	_, err = io.Copy(f, bytes.NewReader(reqBytes))
+	err = sanitizedReq.WriteProxy(f)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error writing request to file %s: %w", name, err)
 	}
-
-	req, err = http.ReadRequest(bufio.NewReader(bytes.NewReader(reqBytes)))
-	if err != nil {
-		return nil, err
+	if err := f.Close(); err != nil {
+		return nil, fmt.Errorf("error closing file %s: %w", name, err)
 	}
 
+	req.Body = io.NopCloser(&originalReqBody)
 	return req, nil
 }