rpc: measure the amount of time spend sleeping in retry backoff

This information is useful to understand if the HBase clusters is
overloaded (CallQueueTooBigException) and track the impact of it on
gohbase latency.

prometheus.go

@@ -41,4 +41,16 @@ var (
 		Name:      "regions_total",
 		Help:      "Total number of regions in the cache",
 	})
+
+	retryBackoffDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: "gohbase",
+		Subsystem: "retry",
+		Name:      "backoff_duration_seconds",
+		Help:      "Time spend sleeping in retry backoff",
+		// Buckets match the exact backoff time generated by the sleepAndIncreaseBackoff function
+		Buckets: []float64{
+			0.016, 0.032, 0.064, 0.128, 0.256, 0.512, 1.024, 2.048,
+			4.096, 8.192, 13.192, 18.192, 23.192, 28.192, 33.192,
+		},
+	})
 )

rpc.go

@@ -1004,6 +1004,11 @@ func sleepAndIncreaseBackoff(ctx context.Context, backoff time.Duration) (time.D
 		return 0, ctx.Err()
 	}
 
+	// Keep track of the amount of time spend sleeping in retry backoff. Ignore if context was
+	// canceled.
+	retryBackoffDuration.Observe(backoff.Seconds())
+
+	// When changing this formula, update the buckets of the retryBackoffDuration metric too.
 	if backoff < 5*time.Second {
 		return backoff * 2, nil
 	} else if backoff < 30*time.Second {