Feature 'multiple MTR jobs'

This commit introduces a feature to trigger multiple
mtr-jobs. One goal is to keep the existing user interface
intact while providing the new feature.

Added flag: -label <job-label>

In order to distinguish multiple running jobs in the resulting
Prometheus /metrics response, each job needs a distinct label.

Added flag: -jobs <path-to-jobs-file>

To define multiple MTR jobs, the concep of a "jobs file" is introduced.
It contains, per line, the label for the job (equivalent for -label),
the schedule upon which the job is launched (equivalent to -schedule)
and the flags for `mtr`.

Added flag: -watch-jobs <schedule>

The -watch-jobs flag makes `mtr-exporter` to reparse the file given
via the -jobs flag at the defined schedule. When change is detected,
all previous jobs are stopped and the new jobs are put into effect.
If no changes are detected, nothing happens.

Note: No INOTIFY or other similar mechanic are used as of now to
keep cross-platform nature of `mtr-exporter` in place without
introducing new dependencies, build-changes etc.

Makefile

@@ -8,8 +8,9 @@ BINARIES=$(addprefix bin/mtr-exporter-$(VERSION)., $(BUILDS))
 
 LDFLAGS=-ldflags "-X main.Version=$(VERSION) -X main.BuildDate=$(BUILD_DATE) -X main.GitHash=$(GIT_HASH)"
 
-mtr-exporter: cmd/mtr-exporter
-	go build -v -o $@ ./$^
+mtr-exporter: bin/mtr-exporter
+bin/mtr-exporter: cmd/mtr-exporter bin
+	go build -v -o $@ ./$<
 
 ######################################################
 ## release related

README.md

@@ -43,34 +43,40 @@ When [prometheus] scrapes the data, you can visualise the observed values:
 
 ## Usage
 
-    $> mtr-exporter [OPTS] -- [MTR-OPTS]
-
-    mtr-exporter [FLAGS] -- [MTR-FLAGS]
+    $> mtr-exporter [FLAGS] -- [MTR-FLAGS]
 
     FLAGS:
-    -bind <bind-address>
-              bind address (default ":8080")
-    -h        show help
-    -mtr <path-to-binary>
-              path to mtr binary (default "mtr")
-    -schedule <schedule>
-              schedule at which often mtr is launched (default "@every 60s")
-              examples:
-                @every <dur>  - example "@every 60s"
-                @hourly       - run once per hour
-                10 * * * *    - execute 10 minutes after the full hour
-              see https://en.wikipedia.org/wiki/Cron
-    -tslogs
-              use timestamps in logs
+    -bind       <bind-address>
+                bind address (default ":8080")
+    -h          
+                show help
+    -jobs       <path-to-jobsfile>
+                file describing multiple mtr-jobs. syntax is given below.
+    -label      <job-label>
+                use <job-label> in prometheus-metrics (default: "mtr-exporter-cli")
+    -mtr        <path-to-binary>
+                path to mtr binary (default: "mtr")
+    -schedule   <schedule>
+                schedule at which often mtr is launched (default: "@every 60s")
+                examples:
+                   @every <dur>  - example "@every 60s"
+                   @hourly       - run once per hour
+                   10 * * * *    - execute 10 minutes after the full hour
+                see https://en.wikipedia.org/wiki/Cron
+    -tslogs     
+                use timestamps in logs
+    -watch-jobs <schedule>
+                periodically watch the file defined via -jobs (default: "")
+                if it has changed stop previously running mtr-jobs and apply
+                all jobs defined in -jobs.
     -version
-              show version
-
+                show version
     MTR-FLAGS:
             see "man mtr" for valid flags to mtr.
 
 At `/metrics` the measured values of the last run are exposed.
 
-Examples:
+### Examples
 
     $> mtr-exporter -- example.com
     # probe every minute "example.com"
@@ -82,6 +88,18 @@ Examples:
     # probe every 30s "example.com", wait 1s for response, try a max of 3 hops,
     # use interface "ven3", do not resolve DNS.
 
+### Jobs-File Syntax
+
+    # comment lines start with '#' are ignored
+    # empty lines are ignored as well
+    label -- <schedule> -- mtr-flags
+
+Example:
+
+    quad9       -- @every 120s -- -I ven1 -n 9.9.9.9
+    example.com -- @every 45s  -- -I ven2 -n example.com
+
+
 ## Requirements
 
 Runtime:
@@ -100,7 +118,7 @@ Build:
 
 One-off building and "installation":
 
-    $> go get github.com/mgumz/mtr-exporter/cmd/mtr-exporter
+    $> go install github.com/mgumz/mtr-exporter/cmd/mtr-exporter@latest
 
 ## License
 

cmd/mtr-exporter/main.go

@@ -10,15 +10,20 @@ import (
 	"net/http"
 	"os"
 
+	"github.com/mgumz/mtr-exporter/pkg/job"
+
 	"github.com/robfig/cron/v3"
 )
 
 func main() {
 	log.SetFlags(0)
 
 	mtrBin := flag.String("mtr", "mtr", "path to `mtr` binary")
+	jobLabel := flag.String("label", "mtr-exporter-cli", "job label")
 	bind := flag.String("bind", ":8080", "bind address")
-	schedule := flag.String("schedule", "@every 60s", "Schedule at which often `mtr` is launched")
+	jobFile := flag.String("jobs", "", "file containing job definitions")
+	schedule := flag.String("schedule", "@every 60s", "schedule at which often `mtr` is launched")
+	doWatchJobsFile := flag.String("watch-jobs", "", "re-parse -jobs file to schedule")
 	doPrintVersion := flag.Bool("version", false, "show version")
 	doPrintUsage := flag.Bool("h", false, "show help")
 	doTimeStampLogs := flag.Bool("tslogs", false, "use timestamps in logs")
@@ -30,44 +35,61 @@ func main() {
 		printVersion()
 		return
 	}
-
 	if *doPrintUsage {
 		flag.Usage()
 		return
 	}
-
 	if *doTimeStampLogs {
 		log.SetFlags(log.LstdFlags | log.LUTC)
 	}
 
-	if len(flag.Args()) == 0 {
-		log.Println("error: no mtr arguments given - at least the target host must be defined.")
-		os.Exit(1)
+	collector := job.NewCollector()
+	scheduler := cron.New()
 
-		return
+	if len(flag.Args()) > 0 {
+		j := job.NewJob(*mtrBin, flag.Args(), *schedule)
+		j.Label = *jobLabel
+		if _, err := scheduler.AddJob(j.Schedule, j); err != nil {
+			log.Printf("error: unable to add %q to scheduler: %v", j.Label, err)
+			os.Exit(1)
+		}
+		if !collector.AddJob(j.JobMeta) {
+			log.Printf("error: unable to add %q to collector", j.Label)
+			os.Exit(1)
+		}
+		j.UpdateFn = func(meta job.JobMeta) bool { return collector.UpdateJob(meta) }
 	}
 
-	job := newMtrJob(*mtrBin, flag.Args())
-
-	c := cron.New()
-
-	_, err := c.AddFunc(*schedule, func() {
-		log.Println("launching", job.cmdLine)
-		if err := job.Launch(); err != nil {
-			log.Println("failed:", err)
-			return
+	if *jobFile != "" {
+		if *doWatchJobsFile != "" {
+			log.Printf("info: watching %q at %q", *jobFile, *doWatchJobsFile)
+			job.WatchJobsFile(*jobFile, *mtrBin, *doWatchJobsFile, collector)
+		} else {
+			jobs, _, err := job.ParseJobFile(*jobFile, *mtrBin)
+			if err != nil {
+				log.Printf("error: parsing jobs file %q: %s", *jobFile, err)
+				os.Exit(1)
+			}
+			if jobs.Empty() {
+				log.Println("error: no mtr jobs defined - provide at least one via -file or via arguments")
+				os.Exit(1)
+			}
+			for _, j := range jobs {
+				if collector.AddJob(j.JobMeta) {
+					if _, err := scheduler.AddJob(j.Schedule, j); err != nil {
+						log.Printf("error: unable to add %q to collector: %v", j.Label, err)
+						os.Exit(1)
+					}
+					j.UpdateFn = func(meta job.JobMeta) bool { return collector.UpdateJob(meta) }
+				} // FIXME: log failed addition to collector, most likely
+				// due to duplicate label
+			}
 		}
-		log.Println("done: ",
-			len(job.Report.Hubs), "hops in", job.Duration, ".")
-	})
-	if err != nil {
-		log.Fatalf(err.Error())
-		os.Exit(1)
 	}
 
-	c.Start()
+	scheduler.Start()
 
-	http.Handle("/metrics", job)
+	http.Handle("/metrics", collector)
 	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintf(w, "OK")
 	})

cmd/mtr-exporter/usage.go

@@ -3,25 +3,35 @@ package main
 import "fmt"
 
 func usage() {
-	usage := `Usage: mtr-exporter [FLAGS] -- [MTR-FLAGS]
+
+	const usage string = `Usage: mtr-exporter [FLAGS] -- [MTR-FLAGS]
 
 FLAGS:
--bind <bind-address>
-	  bind address (default ":8080")
--h	  show help
--mtr <path-to-binary>
-	  path to mtr binary (default "mtr")
--schedule <schedule>
-	  schedule at which often mtr is launched (default "@every 60s")
-	  examples:
-	    @every <dur>  - example "@every 60s"
-	    @hourly       - run once per hour
-	    10 * * * *    - execute 10 minutes after the full hour
-	  see https://en.wikipedia.org/wiki/Cron
--tslogs
-	  use timestamps in logs
+-bind       <bind-address>
+            bind address (default ":8080")
+-h          
+            show help
+-jobs       <path-to-jobsfile>
+            file describing multiple mtr-jobs. syntax is given below.
+-label      <job-label>
+            use <job-label> in prometheus-metrics (default: "mtr-exporter-cli")
+-mtr        <path-to-binary>
+            path to mtr binary (default: "mtr")
+-schedule   <schedule>
+            schedule at which often mtr is launched (default: "@every 60s")
+            examples:
+               @every <dur>  - example "@every 60s"
+               @hourly       - run once per hour
+               10 * * * *    - execute 10 minutes after the full hour
+            see https://en.wikipedia.org/wiki/Cron
+-tslogs     
+            use timestamps in logs
+-watch-jobs <schedule>
+            periodically watch the file defined via -jobs (default: "")
+            if it has changed stop previously running mtr-jobs and apply
+            all jobs defined in -jobs.
 -version
-	  show version
+            show version
 
 MTR-FLAGS:
 see "man mtr" for valid flags to mtr.
@@ -36,7 +46,13 @@ $> mtr-exporter -- -n example.com
 
 $> mtr-exporter -schedule "@every 30s" -- -G 1 -m 3 -I ven3 -n example.com
 # probe every 30s "example.com", wait 1s for response, try a max of 3 hops,
-# use interface "ven3", do not resolve DNS.`
+# use interface "ven3", do not resolve DNS.
+
+Example Job File:
+
+    # comments are ignored
+    job1 -- @every 30s -- -I ven1 -n example.com
+    job2 -- @every 30s -- -I ven2 -n example.com`
 
 	fmt.Println(usage)
 }

go.mod

@@ -3,5 +3,6 @@ module github.com/mgumz/mtr-exporter
 go 1.14
 
 require (
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/robfig/cron/v3 v3.0.1
 )

go.sum

@@ -1,2 +1,4 @@
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=