From ac1ec91b9b1fd21adabc740a0bad19944bc1c245 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 15 Sep 2021 12:58:29 +0530 Subject: [PATCH 1/5] restructured files,use real time temp data --- async.go | 53 ++++++++++++++++++----------------------------------- go.mod | 1 + go.sum | 2 ++ sync.go | 13 +++++++++---- temp.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 39 deletions(-) create mode 100644 temp.go diff --git a/async.go b/async.go index 0ce184e..f37b53d 100644 --- a/async.go +++ b/async.go @@ -1,12 +1,10 @@ package main import ( - "encoding/json" - "fmt" - "io/ioutil" "net/http" "time" + "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -36,49 +34,34 @@ var jobsInQueue = promauto.NewGauge( }, ) -type Temperature struct { - Temp float64 `json:"temperature"` -} +var httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "http_response_time_seconds", + Help: "Duration of HTTP requests.", +}, []string{"path"}) -type Final struct { - StartTime string `json:"startTime"` - Values Temperature `json:"values"` -} +func prometheusMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + route := mux.CurrentRoute(r) + path, _ := route.GetPathTemplate() -type Interval struct { - Timestep string `json:"timestep"` - StartTime string `json:"startTime"` - EndTime string `json:"endTime"` - TempVal []Final `json:"intervals"` -} - -type Timelines struct { - Timestep []Interval `json:"timelines"` -} + timer := prometheus.NewTimer(httpDuration.WithLabelValues(path)) -type Response struct { - Data Timelines `json:"data"` + timer.ObserveDuration() + }) } func main() { - url := fmt.Sprintf("https://api.tomorrow.io/v4/timelines?location=%f,%f&fields=temperature×teps=%s&units=%s", 73.98529171943665, 40.75872069597532, "1h", "metric") - req, _ := http.NewRequest("GET", url, nil) - req.Header.Add("apikey", "APIKEY") - res, _ := http.DefaultClient.Do(req) - defer res.Body.Close() - - body, _ := ioutil.ReadAll(res.Body) - - var dat Response - if err := json.Unmarshal(body, &dat); err != nil { - panic(err) - } + dat := getTempData() for _, interval := range dat.Data.Timestep[0].TempVal { recordMetrics(interval.Values.Temp) } - http.Handle("/metrics", promhttp.Handler()) + router := mux.NewRouter() + router.Use(prometheusMiddleware) + + //http.Handle("/metrics", promhttp.Handler()) + router.Path("/metrics").Handler(promhttp.Handler()) http.ListenAndServe(":2112", nil) } diff --git a/go.mod b/go.mod index e970c26..0f45560 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( github.com/google/go-cmp v0.5.6 // indirect + github.com/gorilla/mux v1.8.0 github.com/prometheus/client_golang v1.11.0 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect ) diff --git a/go.sum b/go.sum index 8cc5161..26cb9d2 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/sync.go b/sync.go index 2b268be..266c5e3 100644 --- a/sync.go +++ b/sync.go @@ -15,12 +15,17 @@ type CityStats struct { //temperature by country func (c *CityStats) TemperatureAndHumidity() ( - tempByCity map[string]int, humidityByCity map[string]float64, + tempByCity map[string]float64, humidityByCity map[string]float64, ) { - tempByCity = map[string]int{ - "bangalore": rand.Intn(100), - "london": rand.Intn(1000), + // get real time API temp data here + tempByCity = make(map[string]float64) + dat := getTempData() + cities := []string{"bangalore", "london"} + + for ind, interval := range dat.Data.Timestep[0].TempVal { + tempByCity[cities[ind%2]] = interval.Values.Temp } + humidityByCity = map[string]float64{ "bangalore": rand.Float64(), "london": rand.Float64(), diff --git a/temp.go b/temp.go new file mode 100644 index 0000000..e08c2d5 --- /dev/null +++ b/temp.go @@ -0,0 +1,50 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +type Temperature struct { + Temp float64 `json:"temperature"` +} + +type Final struct { + StartTime string `json:"startTime"` + Values Temperature `json:"values"` +} + +type Interval struct { + Timestep string `json:"timestep"` + StartTime string `json:"startTime"` + EndTime string `json:"endTime"` + TempVal []Final `json:"intervals"` +} + +type Timelines struct { + Timestep []Interval `json:"timelines"` +} + +type Response struct { + Data Timelines `json:"data"` +} + +func getTempData() Response { + url := fmt.Sprintf("https://api.tomorrow.io/v4/timelines?location=%f,%f&fields=temperature×teps=%s&units=%s", 73.98529171943665, 40.75872069597532, "1h", "metric") + + req, _ := http.NewRequest("GET", url, nil) + req.Header.Add("apikey", "APIKEY") + res, _ := http.DefaultClient.Do(req) + defer res.Body.Close() + + body, _ := ioutil.ReadAll(res.Body) + + var dat Response + if err := json.Unmarshal(body, &dat); err != nil { + panic(err) + } + + return dat +} From 8ab98925417dbe1b39f3b772ceeb8e0161dbc8e3 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Wed, 15 Sep 2021 13:02:22 +0530 Subject: [PATCH 2/5] updated README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 973ee6a..b5e7392 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,7 @@ # prom-exporter Repo for learning how to implement Prometheus exporter + +### Instructions to run: +* Have Prometheus running on port 9090 and make sure port 2112 is not in use. +* Async exporter: `go run async.go temp.go`. +* Sync exporter: `go run sync.go temp.go`. From 4540a91aeac1ea47a76366abe2208620893caf43 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Thu, 16 Sep 2021 10:38:12 +0530 Subject: [PATCH 3/5] moved API call in async --- async.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/async.go b/async.go index f37b53d..49d5d39 100644 --- a/async.go +++ b/async.go @@ -10,14 +10,17 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) -func recordMetrics(temp float64) { - go func() { - for { - opsProcessed.Inc() - jobsInQueue.Set(temp) - time.Sleep(2 * time.Second) +func recordMetrics() { + for { + dat := getTempData() + + for _, interval := range dat.Data.Timestep[0].TempVal { + jobsInQueue.Set(interval.Values.Temp) } - }() + + opsProcessed.Inc() + time.Sleep(2 * time.Second) + } } var opsProcessed = promauto.NewGauge( @@ -52,11 +55,7 @@ func prometheusMiddleware(next http.Handler) http.Handler { func main() { - dat := getTempData() - - for _, interval := range dat.Data.Timestep[0].TempVal { - recordMetrics(interval.Values.Temp) - } + go recordMetrics() router := mux.NewRouter() router.Use(prometheusMiddleware) From 8308972fa0dd4c03390e3e97cc760081e5c0090f Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Thu, 16 Sep 2021 11:18:53 +0530 Subject: [PATCH 4/5] added basic error handling Signed-off-by: metonymic-smokey --- async.go | 12 ++++++++++-- sync.go | 11 ++++++++++- temp.go | 22 ++++++++++++++++------ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/async.go b/async.go index 49d5d39..6a9747b 100644 --- a/async.go +++ b/async.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "net/http" "time" @@ -12,7 +13,15 @@ import ( func recordMetrics() { for { - dat := getTempData() + dat, err := getTempData() + if err != nil { + fmt.Println(err) + time.Sleep(2 * time.Second) + continue + } + if len(dat.Data.Timestep) == 0 { + continue + } for _, interval := range dat.Data.Timestep[0].TempVal { jobsInQueue.Set(interval.Values.Temp) @@ -54,7 +63,6 @@ func prometheusMiddleware(next http.Handler) http.Handler { } func main() { - go recordMetrics() router := mux.NewRouter() diff --git a/sync.go b/sync.go index 266c5e3..d3793ad 100644 --- a/sync.go +++ b/sync.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "log" "math/rand" "net/http" @@ -19,7 +20,15 @@ func (c *CityStats) TemperatureAndHumidity() ( ) { // get real time API temp data here tempByCity = make(map[string]float64) - dat := getTempData() + dat, err := getTempData() + if err != nil { + fmt.Println(err) + return + } + if len(dat.Data.Timestep) == 0 { + return + } + cities := []string{"bangalore", "london"} for ind, interval := range dat.Data.Timestep[0].TempVal { diff --git a/temp.go b/temp.go index e08c2d5..e101d67 100644 --- a/temp.go +++ b/temp.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "fmt" "io/ioutil" "net/http" @@ -31,20 +32,29 @@ type Response struct { Data Timelines `json:"data"` } -func getTempData() Response { +func getTempData() (Response, error) { url := fmt.Sprintf("https://api.tomorrow.io/v4/timelines?location=%f,%f&fields=temperature×teps=%s&units=%s", 73.98529171943665, 40.75872069597532, "1h", "metric") - req, _ := http.NewRequest("GET", url, nil) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return Response{}, errors.New("error in GET request") + } req.Header.Add("apikey", "APIKEY") - res, _ := http.DefaultClient.Do(req) + res, err := http.DefaultClient.Do(req) + if err != nil { + return Response{}, err + } defer res.Body.Close() - body, _ := ioutil.ReadAll(res.Body) + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return Response{}, errors.New("error reading response") + } var dat Response if err := json.Unmarshal(body, &dat); err != nil { - panic(err) + return Response{}, errors.New("error unmarshalling JSON") } - return dat + return dat, nil } From fe332adfe1c05062c1b0e8df18d21951bf44e270 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Thu, 16 Sep 2021 20:43:50 +0530 Subject: [PATCH 5/5] histogram: part 1 need to convert to a histogram graph --- sync.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sync.go b/sync.go index d3793ad..bd60aed 100644 --- a/sync.go +++ b/sync.go @@ -5,6 +5,7 @@ import ( "log" "math/rand" "net/http" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -26,6 +27,7 @@ func (c *CityStats) TemperatureAndHumidity() ( return } if len(dat.Data.Timestep) == 0 { + fmt.Println("empty result!") return } @@ -57,6 +59,11 @@ var ( "humidity of a city as a fraction", []string{"city"}, nil, ) + httpDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "http_response_time_seconds", + Help: "Duration of HTTP requests.", + Buckets: prometheus.LinearBuckets(20, 5, 5), + }) ) func (cc CityStatsCollector) Describe(ch chan<- *prometheus.Desc) { @@ -64,7 +71,11 @@ func (cc CityStatsCollector) Describe(ch chan<- *prometheus.Desc) { } func (cc CityStatsCollector) Collect(ch chan<- prometheus.Metric) { + begin := time.Now() tempByCity, humidityByCity := cc.CityStats.TemperatureAndHumidity() + duration := time.Since(begin) + httpDuration.Observe(float64(duration)) + for city, temp := range tempByCity { ch <- prometheus.MustNewConstMetric( tempDesc, @@ -102,6 +113,7 @@ func main() { reg.MustRegister( prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}), prometheus.NewGoCollector(), + httpDuration, ) http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))