Add HTTP traffic logger (#471)

build/common.go

@@ -273,15 +273,10 @@ func e2eProxy(mode e2e.ProxyMode) error {
 	}
 	fixtures := make([]*fixture.Fixture, 0)
 	for _, s := range cfg.Storage {
-		var store storage.Storage
-		if cfg.Storage == nil || s.Type == config.StorageTypeHAR {
-			har := storage.NewHARStorage(s.Path)
-			if err := har.Load(); err != nil {
-				har.Init()
-			}
-			store = har
+		if s.Type == config.StorageTypeHAR {
+			store := storage.NewHARStorage(s.Path)
+			fixtures = append(fixtures, fixture.NewFixture(store))
 		}
-		fixtures = append(fixtures, fixture.NewFixture(store))
 	}
 	proxy := e2e.NewProxy(mode, fixtures, cfg)
 	return proxy.Start()

experimental/e2e/README.md

@@ -244,37 +244,39 @@ func CustomE2E() error {
 		return err
 	}
 
-	var store storage.Storage
-	if cfg.Storage[0] == nil || cfg.Storage[0].Type == config.StorageTypeHAR {
-		har := storage.NewHARStorage(cfg.Storage[0].Path)
-		if err := har.Load(); err != nil {
-			har.Init()
+	fixtures := make([]*fixture.Fixture, 0)
+	for _, s := range cfg.Storage {
+		if s.Type == config.StorageTypeHAR {
+			continue
 		}
-		store = har
+
+		store := storage.NewHARStorage(s.Path)
+		f := fixture.NewFixture(store)
+
+		// modify incoming requests
+		f.WithRequestProcessor(func(req *http.Request) *http.Request {
+			req.URL.Host = "example.com"
+			req.URL.Path = "/hello/world"
+			return req
+		})
+
+		// modify incoming responses
+		f.WithResponseProcessor(func(res *http.Response) *http.Response {
+			res.StatusCode = http.StatusNotFound
+			res.Header = http.Header{}
+			res.Body = ioutil.NopCloser(bytes.NewBufferString("Not found"))
+			return res
+		})
+
+		// modify matching behavior
+		f.WithMatcher(func(a, b *http.Request) bool {
+			return true
+		})
+
+		fixtures = append(fixtures, fixture.NewFixture(store))
 	}
-	f := fixture.NewFixture(store)
-
-	// modify incoming requests
-	f.WithRequestProcessor(func(req *http.Request) *http.Request {
-		req.URL.Host = "example.com"
-		req.URL.Path = "/hello/world"
-		return req
-	})
-
-	// modify incoming responses
-	f.WithResponseProcessor(func(res *http.Response) *http.Response {
-		res.StatusCode = http.StatusNotFound
-		res.Header = http.Header{}
-		res.Body = ioutil.NopCloser(bytes.NewBufferString("Not found"))
-		return res
-	})
-
-	// modify matching behavior
-	f.WithMatcher(func(a, b *http.Request) bool {
-		return true
-	})
-
-	proxy := e2e.NewProxy(e2e.ProxyModeAppend, []*fixture.Fixture{f}, cfg)
+
+	proxy := e2e.NewProxy(e2e.ProxyModeAppend, fixtures, cfg)
 	return proxy.Start()
 }
 ```

experimental/e2e/storage/har.go

@@ -35,6 +35,7 @@ func NewHARStorage(path string) *HARStorage {
 		currentTime: time.Now,
 		newUUID:     newUUID,
 	}
+	storage.Init()
 	return storage
 }
 
@@ -51,6 +52,9 @@ func (s *HARStorage) WithUUIDOverride(fn func() string) {
 }
 
 func (s *HARStorage) Init() {
+	if err := s.Load(); err == nil {
+		return
+	}
 	s.lock.Lock()
 	defer s.lock.Unlock()
 	s.har.Log = &har.Log{

experimental/e2e/utils/http.go

@@ -3,7 +3,6 @@ package utils
 import (
 	"bytes"
 	"crypto/tls"
-	"errors"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -15,7 +14,7 @@ import (
 // ReadRequestBody reads the request body without closing the req.Body.
 func ReadRequestBody(r *http.Request) ([]byte, error) {
 	if r.Body == nil {
-		return nil, errors.New("response body is nil")
+		return []byte{}, nil
 	}
 	body, err := io.ReadAll(r.Body)
 	if err != nil {

experimental/http_logger/README.md

@@ -0,0 +1,120 @@
+# HTTP Traffic Logger
+
+This is intended to be used to record HTTP traffic between a backend data source plugin and
+the target API for debugging purposes. For example, let's say a user is attempting to demonstrate a bug
+with the GitHub data source plugin that is not reproducible with the developer's personal account. Currently,
+it would be very difficult to determine the cause of the bug without having access to the user's GitHub	account. With the
+HTTP logger, the workflow would look like this:
+
+1. The user enables the	HTTP logger for the GitHub data source plugin in their `grafana.ini` configuration file.
+1. The user reproduces the bug in their environment.
+1. The user reviews the [HAR file](https://en.wikipedia.org/wiki/HAR_(file_format)) generated by the HTTP logger in their browser's developer tools, and removes any sensitive information from the HAR file.
+1. The user shares the HAR file with the data source developer, along with a dashboard JSON that contains the queries used to reproduce the bug.
+1. The data source developer uses the E2E proxy in replay mode to replay the HTTP traffic recorded by the HTTP logger.
+
+| ![User's envrionment](user.png)  |  ![Developer debugging](local.png) |
+|---|---|
+
+## Enabling the HTTP Logger
+
+To enable the HTTP logger for a plugin, add the following to your `grafana.ini` configuration file:
+
+```
+[plugin.grafana-github-datasource]
+har_log_enabled = true
+har_log_path = /home/example/github.har
+```
+
+In the example above, `grafana-github-datasource` is the plugin ID, which can be found in the plugin's `src/plugin.json` file.
+
+## Adding Support for HTTP Logging in a Data Source Plugin
+
+`HTTPLogger` implements the `http.RoundTripper` interface, and wraps the existing `http.RoundTripper` implementation in
+the data source plugin's HTTP client. For example, if the plugin's current HTTP client looks like this:
+
+```go
+client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				Renegotiation:      tls.RenegotiateFreelyAsClient,
+				InsecureSkipVerify: settings.TLSSkipVerify,
+			},
+			Proxy: http.ProxyFromEnvironment,
+			Dial: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+				DualStack: true,
+			}).Dial,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+		},
+		Timeout: time.Second * 30,
+		Transport: hl,
+		Timeout:   time.Second * 30,
+	},
+}
+```
+
+Then, the `HTTPLogger` can be added by wrapping the existing `http.Transport` like this:
+
+```go
+transport := &http.Transport{
+	TLSClientConfig: &tls.Config{
+		Renegotiation:      tls.RenegotiateFreelyAsClient,
+		InsecureSkipVerify: settings.TLSSkipVerify,
+	},
+	Proxy: http.ProxyFromEnvironment,
+	Dial: (&net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+		DualStack: true,
+	}).Dial,
+	TLSHandshakeTimeout:   10 * time.Second,
+	ExpectContinueTimeout: 1 * time.Second,
+	MaxIdleConns:          100,
+	IdleConnTimeout:       90 * time.Second,
+}
+h := http_logger.NewHTTPLogger("grafana-github-datasource", transport)
+client := &http.Client{
+	Transport: h,
+	Timeout:   time.Second * 30,
+}
+```
+
+In the example above, `grafana-github-datasource` is the plugin ID, which should match the `id` property in the `src/plugin.json` file.
+
+## Redacting Sensitive Information
+
+By default, the HTTP logger will remove cookies and authorization headers from the HAR file, but the user should carefully
+review the HAR file in their browser's dev tools to ensure that any sensitive information is removed from responses before sharing it with the data source developer.
+
+![Har review](review.png)
+
+Since HAR files are actually JSON files, the user can edit the responses in a text editor. The user should verify that their browser is still able to load the HAR file after editing is complete.
+
+## Local Debugging
+
+Follow the E2E proxy's [Quick Setup](../e2e/README.md#quick-setup) instructions to configure the proxy to replay the recorded traffic.
+
+Make sure specify the path to the HAR file that the user shared in the `proxy.json` file:
+
+```json
+{
+	"storage": [{
+		"type": "har",
+		"path": "github.har"
+	}],
+	"address": "127.0.0.1:9999",
+	"hosts": ["github.com"]
+}
+```
+
+Start the proxy in replay mode to avoid overwriting the original HAR file shared by the user:
+
+```
+mage e2e:replay
+```
+
+It will be simpler to reproduce the issue if you request that the user saves a dashboard JSON with the query that caused the bug. The dashboard should be using an **absolute time range**. This will ensure that the query will match the traffic recorded in the HAR file when the dashboard is loaded by the developer.

experimental/http_logger/http_logger.go

@@ -0,0 +1,98 @@
+package httplogger
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/experimental/e2e/fixture"
+	"github.com/grafana/grafana-plugin-sdk-go/experimental/e2e/storage"
+	"github.com/grafana/grafana-plugin-sdk-go/experimental/e2e/utils"
+)
+
+const (
+	// PluginHARLogEnabledEnv is a constant for the GF_PLUGIN_HAR_LOG_ENABLED environment variable used to enable HTTP request and responses in HAR format for debugging purposes.
+	PluginHARLogEnabledEnv = "GF_PLUGIN_HAR_LOG_ENABLED"
+	// PluginHARLogPathEnv is a constant for the GF_PLUGIN_HAR_LOG_PATH environment variable used to specify a path to store HTTP request and responses in HAR format for debugging purposes.
+	PluginHARLogPathEnv = "GF_PLUGIN_HAR_LOG_PATH"
+)
+
+// HTTPLogger is a http.RoundTripper that logs requests and responses in HAR format.
+type HTTPLogger struct {
+	pluginID string
+	enabled  func() bool
+	proxied  http.RoundTripper
+	fixture  *fixture.Fixture
+}
+
+// NewHTTPLogger creates a new HTTPLogger.
+func NewHTTPLogger(pluginID string, proxied http.RoundTripper) *HTTPLogger {
+	path := defaultPath(pluginID)
+	s := storage.NewHARStorage(path)
+	f := fixture.NewFixture(s)
+
+	return &HTTPLogger{
+		pluginID: pluginID,
+		proxied:  proxied,
+		fixture:  f,
+		enabled:  defaultEnabledCheck,
+	}
+}
+
+// RoundTrip implements the http.RoundTripper interface.
+func (hl *HTTPLogger) RoundTrip(req *http.Request) (*http.Response, error) {
+	if !hl.enabled() {
+		return hl.proxied.RoundTrip(req)
+	}
+
+	buf := []byte{}
+	if req.Body != nil {
+		if b, err := utils.ReadRequestBody(req); err == nil {
+			req.Body = ioutil.NopCloser(bytes.NewReader(b))
+			buf = b
+		}
+	}
+
+	res, err := hl.proxied.RoundTrip(req)
+	if err != nil {
+		return res, err
+	}
+
+	// reset the request body before saving
+	if req.Body != nil {
+		req.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
+	}
+
+	// skip saving if there's an existing entry for this request
+	if _, exists := hl.fixture.Match(req); exists != nil {
+		return res, err
+	}
+
+	hl.fixture.Add(req, res)
+	err = hl.fixture.Save()
+
+	return res, err
+}
+
+func defaultPath(pluginID string) string {
+	if path, ok := os.LookupEnv(PluginHARLogPathEnv); ok {
+		return path
+	}
+	return getTempFilePath(pluginID)
+}
+
+func defaultEnabledCheck() bool {
+	if v, ok := os.LookupEnv(PluginHARLogEnabledEnv); ok && v == "true" {
+		return true
+	}
+	return false
+}
+
+func getTempFilePath(pluginID string) string {
+	filename := fmt.Sprintf("%s_%d.har", pluginID, time.Now().UnixMilli())
+	return path.Join(os.TempDir(), filename)
+}