Metrics added (#95)

.codecov.yml

@@ -0,0 +1,29 @@
+# Docs: <https://docs.codecov.io/docs/commit-status>
+
+ignore:
+  - "pkg/hostsfile/hostname_validator.go" # generated file
+
+coverage:
+  # coverage lower than 50 is red, higher than 90 green
+  range: 30..80
+
+  status:
+    project:
+      default:
+        # Choose a minimum coverage ratio that the commit must meet to be considered a success.
+        #
+        # `auto` will use the coverage from the base commit (pull request base or parent commit) coverage to compare
+        # against.
+        target: auto
+
+        # Allow the coverage to drop by X%, and posting a success status.
+        threshold: 5%
+
+        # Resulting status will pass no matter what the coverage is or what other settings are specified.
+        informational: true
+
+    patch:
+      default:
+        target: auto
+        threshold: 5%
+        informational: true

CHANGELOG.md

@@ -10,6 +10,11 @@ The format is based on [Keep a Changelog][keepachangelog] and this project adher
 
 - Go version updated from `1.16.2` up to `1.16.3`
 
+### Added
+
+- HTTP route `/metrics` with metrics in [prometheus](https://github.com/prometheus) format
+- Comment in generated script with "script generation" time
+
 ## v4.2.0
 
 ### Changed

go.mod

@@ -10,6 +10,8 @@ require (
 	github.com/go-redis/redis/v8 v8.8.0
 	github.com/gorilla/mux v1.8.0
 	github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d
+	github.com/prometheus/client_golang v1.10.0
+	github.com/prometheus/client_model v0.2.0
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0

internal/pkg/http/handlers/generate/handler.go

@@ -25,11 +25,18 @@ import (
 	"go.uber.org/zap"
 )
 
+type metrics interface {
+	IncrementCacheHits()
+	IncrementCacheMisses()
+	ObserveGenerationDuration(time.Duration)
+}
+
 type handler struct {
 	ctx    context.Context
 	log    *zap.Logger
 	cacher cache.Cacher
 	cfg    *config.Config
+	m      metrics
 
 	defaultRedirectIP net.IP
 
@@ -45,7 +52,13 @@ const (
 )
 
 // NewHandler creates RouterOS script generation handler.
-func NewHandler(ctx context.Context, log *zap.Logger, cacher cache.Cacher, cfg *config.Config) (http.Handler, error) {
+func NewHandler(
+	ctx context.Context,
+	log *zap.Logger,
+	cacher cache.Cacher,
+	cfg *config.Config,
+	m metrics,
+) (http.Handler, error) {
 	if containsIllegalSymbols(cfg.RouterScript.Comment) {
 		return nil, errors.New("wrong config: script comment contains illegal symbols")
 	}
@@ -67,6 +80,7 @@ func NewHandler(ctx context.Context, log *zap.Logger, cacher cache.Cacher, cfg *
 		log:    log,
 		cacher: cacher,
 		cfg:    cfg,
+		m:      m,
 
 		httpClient: &http.Client{Timeout: httpClientTimeout, CheckRedirect: checkRedirectFn},
 	}
@@ -90,6 +104,7 @@ type hostsFileData struct {
 
 func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { //nolint:funlen,gocognit,gocyclo
 	params := newReqParams(h.defaultRedirectIP)
+	startedAt := time.Now()
 
 	if r == nil || r.URL == nil {
 		w.WriteHeader(http.StatusBadRequest)
@@ -230,8 +245,12 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { //nolint:f
 		}
 
 		if data.cacheHit {
+			h.m.IncrementCacheHits()
+
 			h.writeComment(w, fmt.Sprintf("Cache HIT for <%s> (expires after %s)", data.url, data.cacheTTL.Round(time.Second)))
 		} else {
+			h.m.IncrementCacheMisses()
+
 			h.writeComment(w, fmt.Sprintf("Cache miss for <%s>", data.url))
 		}
 
@@ -304,6 +323,10 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { //nolint:f
 		len(result),
 		int(atomic.LoadUint32(&hostsRecordsCount))-len(result),
 	))
+
+	generationDuration := time.Since(startedAt)
+	h.writeComment(w, fmt.Sprintf("Generated in %s", generationDuration))
+	h.m.ObserveGenerationDuration(generationDuration)
 }
 
 func containsIllegalSymbols(s string) bool {

internal/pkg/http/handlers/generate/handler_test.go

@@ -24,6 +24,15 @@ type fakeHTTPClientFunc func(*http.Request) (*http.Response, error)
 
 func (f fakeHTTPClientFunc) Do(req *http.Request) (*http.Response, error) { return f(req) }
 
+type fakeMetrics struct {
+	h, m int
+	d    time.Duration
+}
+
+func (f *fakeMetrics) IncrementCacheHits()                       { f.h++ }
+func (f *fakeMetrics) IncrementCacheMisses()                     { f.m++ }
+func (f *fakeMetrics) ObserveGenerationDuration(d time.Duration) { f.d = d }
+
 var httpMock fakeHTTPClientFunc = func(req *http.Request) (*http.Response, error) { //nolint:gochecknoglobals
 	path, absErr := filepath.Abs(testDataPath + req.URL.RequestURI())
 	if absErr != nil {
@@ -70,7 +79,7 @@ func BenchmarkHandler_ServeHTTP(b *testing.B) {
 	cacher := cache.NewInMemoryCache(time.Minute, time.Second)
 	defer cacher.Close()
 
-	h, _ := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig())
+	h, _ := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig(), &fakeMetrics{})
 
 	h.(*handler).httpClient = httpMock
 
@@ -103,7 +112,9 @@ func TestHandler_ServeHTTP(t *testing.T) {
 	cacher := cache.NewInMemoryCache(time.Minute, time.Second)
 	defer cacher.Close()
 
-	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig())
+	m := fakeMetrics{}
+
+	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig(), &m)
 	assert.NoError(t, err)
 
 	h.(*handler).httpClient = httpMock
@@ -153,13 +164,18 @@ func TestHandler_ServeHTTP(t *testing.T) {
 	assert.Regexp(t, `Source.+non-existing-file\.txt.+404`, body)
 
 	assert.Equal(t, 1234+1, len(lineWithoutCommentsRegex.FindAllStringIndex(body, -1)))
+
+	assert.Equal(t, 2, m.m)
+	assert.Equal(t, 2, m.h)
 }
 
 func TestHandler_ServeHTTPHostnamesExcluding(t *testing.T) {
 	cacher := cache.NewInMemoryCache(time.Minute, time.Second)
 	defer cacher.Close()
 
-	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig())
+	m := fakeMetrics{}
+
+	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig(), &m)
 	assert.NoError(t, err)
 
 	var customHTTPMock fakeHTTPClientFunc = func(req *http.Request) (*http.Response, error) {
@@ -206,13 +222,18 @@ broken line format
 	assert.Contains(t, body, "name=\"example.com\"")
 	assert.Contains(t, body, "/ip dns static")
 	assert.Equal(t, strings.Count(body, "add address=127.0.0.1 comment=\"foo\" disabled=no"), 5)
+
+	assert.Equal(t, 0, m.h)
+	assert.Equal(t, 1, m.m)
 }
 
 func TestHandler_ServeHTTPWithoutRequest(t *testing.T) { //nolint:dupl
 	cacher := cache.NewInMemoryCache(time.Minute, time.Second)
 	defer cacher.Close()
 
-	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig())
+	m := fakeMetrics{}
+
+	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig(), &m)
 	assert.NoError(t, err)
 
 	var rr = httptest.NewRecorder()
@@ -221,13 +242,18 @@ func TestHandler_ServeHTTPWithoutRequest(t *testing.T) { //nolint:dupl
 
 	assert.Equal(t, http.StatusBadRequest, rr.Code)
 	assert.Equal(t, "## Empty request or query parameters\n", rr.Body.String())
+
+	assert.Equal(t, 0, m.h)
+	assert.Equal(t, 0, m.m)
 }
 
 func TestHandler_ServeHTTPRequestWithoutSourcesURLs(t *testing.T) { //nolint:dupl
 	cacher := cache.NewInMemoryCache(time.Minute, time.Second)
 	defer cacher.Close()
 
-	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig())
+	m := fakeMetrics{}
+
+	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig(), &m)
 	assert.NoError(t, err)
 
 	var (
@@ -239,13 +265,18 @@ func TestHandler_ServeHTTPRequestWithoutSourcesURLs(t *testing.T) { //nolint:dup
 
 	assert.Equal(t, http.StatusBadRequest, rr.Code)
 	assert.Regexp(t, `(?mU)## Query parameters error.*sources_urls`, rr.Body.String())
+
+	assert.Equal(t, 0, m.h)
+	assert.Equal(t, 0, m.m)
 }
 
 func TestHandler_ServeHTTPRequestEmptySourcesURLs(t *testing.T) { //nolint:dupl
 	cacher := cache.NewInMemoryCache(time.Minute, time.Second)
 	defer cacher.Close()
 
-	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig())
+	m := fakeMetrics{}
+
+	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig(), &m)
 	assert.NoError(t, err)
 
 	var (
@@ -257,13 +288,18 @@ func TestHandler_ServeHTTPRequestEmptySourcesURLs(t *testing.T) { //nolint:dupl
 
 	assert.Equal(t, http.StatusBadRequest, rr.Code)
 	assert.Regexp(t, `(?mU)## Query parameters.*fail.*empty.*sources`, rr.Body.String())
+
+	assert.Equal(t, 0, m.h)
+	assert.Equal(t, 0, m.m)
 }
 
 func TestHandler_ServeHTTPRequestWrongFormat(t *testing.T) { //nolint:dupl
 	cacher := cache.NewInMemoryCache(time.Minute, time.Second)
 	defer cacher.Close()
 
-	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig())
+	m := fakeMetrics{}
+
+	h, err := NewHandler(context.Background(), zap.NewNop(), cacher, createConfig(), &m)
 	assert.NoError(t, err)
 
 	var (
@@ -275,4 +311,7 @@ func TestHandler_ServeHTTPRequestWrongFormat(t *testing.T) { //nolint:dupl
 
 	assert.Equal(t, http.StatusBadRequest, rr.Code)
 	assert.Regexp(t, `(?mU)## Unsupported format.*foobar`, rr.Body.String())
+
+	assert.Equal(t, 0, m.h)
+	assert.Equal(t, 0, m.m)
 }

internal/pkg/http/handlers/metrics/handler.go

@@ -0,0 +1,16 @@
+// Package metrics contains HTTP handler for application metrics (prometheus format) generation.
+package metrics
+
+import (
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// NewHandler creates metrics handler.
+func NewHandler(registry prometheus.Gatherer) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}).ServeHTTP(w, r)
+	}
+}

internal/pkg/http/handlers/metrics/handler_test.go

@@ -0,0 +1,41 @@
+package metrics_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/assert"
+	"github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/http/handlers/metrics"
+)
+
+func TestNewHandlerError(t *testing.T) {
+	var (
+		req, _     = http.NewRequest(http.MethodGet, "http://testing?foo=bar", http.NoBody)
+		rr         = httptest.NewRecorder()
+		registry   = prometheus.NewRegistry()
+		testMetric = prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "foo",
+				Subsystem: "bar",
+				Name:      "test",
+				Help:      "Test metric.",
+			},
+			[]string{"foo"},
+		)
+	)
+
+	registry.MustRegister(testMetric)
+	testMetric.WithLabelValues("bar").Set(1)
+
+	metrics.NewHandler(registry)(rr, req)
+
+	assert.Equal(t, rr.Code, http.StatusOK)
+	assert.Equal(t, 200, rr.Code)
+	assert.Equal(t, `# HELP foo_bar_test Test metric.
+# TYPE foo_bar_test gauge
+foo_bar_test{foo="bar"} 1
+`, rr.Body.String())
+	assert.Regexp(t, "^text/plain.*$", rr.Header().Get("Content-Type"))
+}

internal/pkg/http/routes.go

@@ -3,18 +3,26 @@ package http
 import (
 	"net/http"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/checkers"
 	"github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/http/fileserver"
 	apiSettings "github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/http/handlers/api/settings"
 	apiVersion "github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/http/handlers/api/version"
 	"github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/http/handlers/generate"
 	"github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/http/handlers/healthz"
+	metricsHandler "github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/http/handlers/metrics"
 	"github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/http/middlewares/nocache"
+	"github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/metrics"
 	"github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/version"
 )
 
-func (s *Server) registerScriptGeneratorHandlers() error {
-	h, err := generate.NewHandler(s.ctx, s.log, s.cacher, s.cfg)
+func (s *Server) registerScriptGeneratorHandlers(registerer prometheus.Registerer) error {
+	m := metrics.NewGenerator()
+	if err := m.Register(registerer); err != nil {
+		return err
+	}
+
+	h, err := generate.NewHandler(s.ctx, s.log, s.cacher, s.cfg, &m)
 	if err != nil {
 		return err
 	}
@@ -45,7 +53,12 @@ func (s *Server) registerAPIHandlers() {
 		Name("api_get_version")
 }
 
-func (s *Server) registerServiceHandlers() {
+func (s *Server) registerServiceHandlers(registry prometheus.Gatherer) {
+	s.router.
+		HandleFunc("/metrics", metricsHandler.NewHandler(registry)).
+		Methods(http.MethodGet).
+		Name("metrics")
+
 	s.router.
 		HandleFunc("/ready", healthz.NewHandler(checkers.NewReadyChecker(s.ctx, s.rdb))).
 		Methods(http.MethodGet, http.MethodHead).