From bb3008a2408125ab0a6de6909ca46a9304f43707 Mon Sep 17 00:00:00 2001 From: UweOhse Date: Sun, 28 Apr 2019 10:56:49 +0200 Subject: [PATCH 1/3] Add files via upload 1. add a few more metrics from nodes/.../rrddata 2. add --one-shot option to aid development. --- pve_exporter.go | 144 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 135 insertions(+), 9 deletions(-) diff --git a/pve_exporter.go b/pve_exporter.go index c582341..1ab70cc 100644 --- a/pve_exporter.go +++ b/pve_exporter.go @@ -40,11 +40,25 @@ type node struct { DiskTotal json.Number `json:"maxdisk"` DiskFree json.Number `json:"disk"` } - type nodeResponse struct { Data []node `json:"data"` } +type nodeRRDData struct { + CPU json.Number `json:"cpu"` + IOWait json.Number `json:"iowait"` + LoadAvg json.Number `json:"loadavg"` + NetIn json.Number `json:"netin"` + NetOut json.Number `json:"netout"` + SwapUsed json.Number `json:"swapused"` + SwapTotal json.Number `json:"swaptotal"` + Time json.Number `json:"time"` +} +type nodeRRDResponse struct { + Data []nodeRRDData `json:"data"` +} + + type lxc struct { Name string `json:"name"` Status string `json:"status"` @@ -193,6 +207,21 @@ func (c *Client) GetNodes() (data []node, err error) { return nodeData.Data, nil } +func (c *Client) GetNodeRRD(name string) (data []nodeRRDData, err error) { + + var respData nodeRRDResponse + + resp, err := c.Do("nodes/"+name+"/rrddata?timeframe=hour") + if err != nil { + return nil, err + } + + if err := json.Unmarshal(resp, &respData); err != nil { + return nil, err + } + + return respData.Data, nil +} func (c *Client) GetLxc(nodeID string) (data []lxc, err error) { @@ -233,6 +262,7 @@ var ( versionUrl = "https://github.com/wakeful/pve_exporter" showVersion = flag.Bool("version", false, "show version and exit") + oneShot = flag.Bool("one-shot", false, "collect data, output to stdout, exit.") listenAddress = flag.String("listen-address", ":9090", "Address on which to expose metrics.") metricsPath = flag.String("telemetry-path", "/metrics", "Path under which to expose metrics.") pveUrl = flag.String("pve-url", "https://127.0.0.1:8006", "URL to your PVE control panel") @@ -271,6 +301,41 @@ var ( "Free disk space on each node", []string{"node"}, nil, ) + clusterNodeIOWait = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "nodes", "io_wait"), + "IOWait on each node", + []string{"node"}, nil, + ) + clusterNodeCPU = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "nodes", "cpu"), + "CPU on each node", + []string{"node"}, nil, + ) + clusterNodeLoadAvg = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "nodes", "load_avg"), + "LoadAvg on each node", + []string{"node"}, nil, + ) + clusterNodeNetIn = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "nodes", "net_in"), + "NetIn on each node", + []string{"node"}, nil, + ) + clusterNodeNetOut = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "nodes", "net_out"), + "NetOut on each node", + []string{"node"}, nil, + ) + clusterNodeSwapUsed = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "nodes", "swap_used"), + "SwapUsed on each node", + []string{"node"}, nil, + ) + clusterNodeSwapTotal = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "nodes", "swap_total"), + "SwapTotal on each node", + []string{"node"}, nil, + ) clusterLxcUp = prometheus.NewDesc( prometheus.BuildFQName(nameSpace, "lxc", "up"), "is the LXC running", @@ -441,6 +506,47 @@ func (e Exporter) Collect(ch chan<- prometheus.Metric) { clusterNodeDiskFree, prometheus.GaugeValue, jNumberToFloat(node.DiskFree), node.Name, ) + rList, err := e.pve.GetNodeRRD(node.Name); + if (err != nil) { + log.Errorln(err) + } else { + // this sorting might be superfluous, i've always seen ordered json. + // but pvesh doesn't print sorted results, and nothing in the API + // descriptions guarantees sorting. + newestIdx := 0 + var newestVal json.Number + for idx, r := range rList { + if (r.Time>newestVal) { + newestVal=r.Time + newestIdx=idx + } + } + if (len(rList)>0) { + r:=rList[newestIdx] + ch <- prometheus.MustNewConstMetric( + clusterNodeIOWait, prometheus.GaugeValue, jNumberToFloat(r.IOWait), node.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterNodeCPU, prometheus.GaugeValue, jNumberToFloat(r.CPU), node.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterNodeLoadAvg, prometheus.GaugeValue, jNumberToFloat(r.LoadAvg), node.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterNodeNetIn, prometheus.GaugeValue, jNumberToFloat(r.NetIn), node.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterNodeNetOut, prometheus.GaugeValue, jNumberToFloat(r.NetOut), node.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterNodeSwapUsed, prometheus.GaugeValue, jNumberToFloat(r.SwapUsed), node.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterNodeSwapTotal, prometheus.GaugeValue, jNumberToFloat(r.SwapTotal), node.Name, + ) + } + } + qemuList, err := e.pve.GetQemu(node.Name) if err != nil { log.Errorln(err) @@ -571,16 +677,36 @@ func main() { } log.Infoln("Starting pve_exporter") + if *oneShot { + // this is very crude, but good enough. + e:=NewExporter() + ch := make(chan prometheus.Metric); + go func() { + e.Collect(ch); + close(ch) + }() + for { + x, ok:= <-ch + if !ok { + break + } + d:=x.Desc(); + fmt.Printf("# HELP %s %s\n",d.String(),""); + fmt.Printf("%+v\n",x); + + } - prometheus.Unregister(prometheus.NewGoCollector()) - prometheus.Unregister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) - prometheus.MustRegister(NewExporter()) + } else { + prometheus.Unregister(prometheus.NewGoCollector()) + prometheus.Unregister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) + prometheus.MustRegister(NewExporter()) - http.Handle(*metricsPath, promhttp.Handler()) - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently) - }) + http.Handle(*metricsPath, promhttp.Handler()) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently) + }) - log.Fatal(http.ListenAndServe(*listenAddress, nil)) + log.Fatal(http.ListenAndServe(*listenAddress, nil)) + } } From 251c203e909ec4f0ae2660bf27a22a76fc38ec94 Mon Sep 17 00:00:00 2001 From: "Uwe Ohse (x8 go)" Date: Thu, 16 May 2019 06:58:17 +0000 Subject: [PATCH 2/3] add _used metrics, leave _free alone --- pve_exporter.go | 98 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 21 deletions(-) diff --git a/pve_exporter.go b/pve_exporter.go index 1ab70cc..2a713f7 100644 --- a/pve_exporter.go +++ b/pve_exporter.go @@ -36,9 +36,9 @@ type node struct { UpTime json.Number `json:"uptime"` CpuTotal json.Number `json:"maxcpu"` RamTotal json.Number `json:"maxmem"` - RamFree json.Number `json:"mem"` + RamUsed json.Number `json:"mem"` DiskTotal json.Number `json:"maxdisk"` - DiskFree json.Number `json:"disk"` + DiskUsed json.Number `json:"disk"` } type nodeResponse struct { Data []node `json:"data"` @@ -65,13 +65,13 @@ type lxc struct { UpTime json.Number `json:"uptime"` CpuCount json.Number `json:"cpus"` DiskTotal json.Number `json:"maxdisk"` - DiskFree json.Number `json:"disk"` + DiskUsed json.Number `json:"disk"` DiskRead json.Number `json:"diskread"` DiskWrite json.Number `json:"diskwrite"` RamTotal json.Number `json:"maxmem"` - RamFree json.Number `json:"mem"` + RamUsed json.Number `json:"mem"` SwapTotal json.Number `json:"maxswap"` - SwapFree json.Number `json:"swap"` + SwapUsed json.Number `json:"swap"` NetIn json.Number `json:"netin"` NetOut json.Number `json:"netout"` } @@ -86,11 +86,11 @@ type qemu struct { UpTime json.Number `json:"uptime"` CpuCount json.Number `json:"cpus"` DiskTotal json.Number `json:"maxdisk"` - DiskFree json.Number `json:"disk"` + DiskUsed json.Number `json:"disk"` DiskRead json.Number `json:"diskread"` DiskWrite json.Number `json:"diskwrite"` RamTotal json.Number `json:"maxmem"` - RamFree json.Number `json:"mem"` + RamUsed json.Number `json:"mem"` NetIn json.Number `json:"netin"` NetOut json.Number `json:"netout"` } @@ -288,7 +288,12 @@ var ( ) clusterNodeRamFree = prometheus.NewDesc( prometheus.BuildFQName(nameSpace, "nodes", "ram_free"), - "Free RAM on each node", + "*Used* RAM on each node (legacy, badly named)", + []string{"node"}, nil, + ) + clusterNodeRamUsed = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "nodes", "ram_used"), + "Used RAM on each node", []string{"node"}, nil, ) clusterNodeDiskTotal = prometheus.NewDesc( @@ -298,7 +303,12 @@ var ( ) clusterNodeDiskFree = prometheus.NewDesc( prometheus.BuildFQName(nameSpace, "nodes", "disk_free"), - "Free disk space on each node", + "*Used* disk space on each node (legacy, badly named)", + []string{"node"}, nil, + ) + clusterNodeDiskUsed = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "nodes", "disk_used"), + "Used disk space on each node", []string{"node"}, nil, ) clusterNodeIOWait = prometheus.NewDesc( @@ -358,7 +368,12 @@ var ( ) clusterLxcDiskFree = prometheus.NewDesc( prometheus.BuildFQName(nameSpace, "lxc", "disk_free"), - "Free disk space for each LXC", + "*Used* disk space for each LXC (legacy, badly named)", + []string{"node", "lxc"}, nil, + ) + clusterLxcDiskUsed = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "lxc", "disk_used"), + "Used disk space for each LXC", []string{"node", "lxc"}, nil, ) clusterLxcDiskRead = prometheus.NewDesc( @@ -378,7 +393,12 @@ var ( ) clusterLxcRamFree = prometheus.NewDesc( prometheus.BuildFQName(nameSpace, "lxc", "ram_free"), - "LXC Free RAM", + "LXC *Used* RAM (legacy, badly named)", + []string{"node", "lxc"}, nil, + ) + clusterLxcRamUsed = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "lxc", "ram_used"), + "LXC Used RAM", []string{"node", "lxc"}, nil, ) clusterLxcSwapTotal = prometheus.NewDesc( @@ -388,7 +408,12 @@ var ( ) clusterLxcSwapFree = prometheus.NewDesc( prometheus.BuildFQName(nameSpace, "lxc", "swap_free"), - "LXC Free SWAP", + "LXC *Used* SWAP (legacy, badly named)", + []string{"node", "lxc"}, nil, + ) + clusterLxcSwapUsed = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "lxc", "swap_used"), + "LXC Used SWAP", []string{"node", "lxc"}, nil, ) clusterLxcNetIn = prometheus.NewDesc( @@ -423,7 +448,12 @@ var ( ) clusterQemuDiskFree = prometheus.NewDesc( prometheus.BuildFQName(nameSpace, "qemu", "disk_free"), - "Free disk space for each QEMU VM", + "*Used* disk space for each QEMU VM (legacy, badly named)", + []string{"node", "qemu"}, nil, + ) + clusterQemuDiskUsed = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "qemu", "disk_used"), + "Used disk space for each QEMU VM", []string{"node", "qemu"}, nil, ) clusterQemuDiskRead = prometheus.NewDesc( @@ -443,7 +473,12 @@ var ( ) clusterQemuRamFree = prometheus.NewDesc( prometheus.BuildFQName(nameSpace, "qemu", "ram_free"), - "QEMU VM Free RAM", + "QEMU VM *Used* RAM (legacy, badly named)", + []string{"node", "qemu"}, nil, + ) + clusterQemuRamUsed = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "qemu", "ram_used"), + "QEMU VM Used RAM", []string{"node", "qemu"}, nil, ) clusterQemuNetIn = prometheus.NewDesc( @@ -497,13 +532,19 @@ func (e Exporter) Collect(ch chan<- prometheus.Metric) { clusterNodeRamTotal, prometheus.GaugeValue, jNumberToFloat(node.RamTotal), node.Name, ) ch <- prometheus.MustNewConstMetric( - clusterNodeRamFree, prometheus.GaugeValue, jNumberToFloat(node.RamFree), node.Name, + clusterNodeRamFree, prometheus.GaugeValue, jNumberToFloat(node.RamUsed), node.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterNodeRamUsed, prometheus.GaugeValue, jNumberToFloat(node.RamUsed), node.Name, ) ch <- prometheus.MustNewConstMetric( clusterNodeDiskTotal, prometheus.GaugeValue, jNumberToFloat(node.DiskTotal), node.Name, ) ch <- prometheus.MustNewConstMetric( - clusterNodeDiskFree, prometheus.GaugeValue, jNumberToFloat(node.DiskFree), node.Name, + clusterNodeDiskFree, prometheus.GaugeValue, jNumberToFloat(node.DiskUsed), node.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterNodeDiskUsed, prometheus.GaugeValue, jNumberToFloat(node.DiskUsed), node.Name, ) rList, err := e.pve.GetNodeRRD(node.Name); @@ -572,7 +613,10 @@ func (e Exporter) Collect(ch chan<- prometheus.Metric) { clusterQemuDiskTotal, prometheus.GaugeValue, jNumberToFloat(qVM.DiskTotal), node.Name, qVM.Name, ) ch <- prometheus.MustNewConstMetric( - clusterQemuDiskFree, prometheus.GaugeValue, jNumberToFloat(qVM.DiskFree), node.Name, qVM.Name, + clusterQemuDiskFree, prometheus.GaugeValue, jNumberToFloat(qVM.DiskUsed), node.Name, qVM.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterQemuDiskUsed, prometheus.GaugeValue, jNumberToFloat(qVM.DiskUsed), node.Name, qVM.Name, ) ch <- prometheus.MustNewConstMetric( clusterQemuDiskRead, prometheus.GaugeValue, jNumberToFloat(qVM.DiskRead), node.Name, qVM.Name, @@ -584,7 +628,10 @@ func (e Exporter) Collect(ch chan<- prometheus.Metric) { clusterQemuRamTotal, prometheus.GaugeValue, jNumberToFloat(qVM.RamTotal), node.Name, qVM.Name, ) ch <- prometheus.MustNewConstMetric( - clusterQemuRamFree, prometheus.GaugeValue, jNumberToFloat(qVM.RamFree), node.Name, qVM.Name, + clusterQemuRamFree, prometheus.GaugeValue, jNumberToFloat(qVM.RamUsed), node.Name, qVM.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterQemuRamUsed, prometheus.GaugeValue, jNumberToFloat(qVM.RamUsed), node.Name, qVM.Name, ) ch <- prometheus.MustNewConstMetric( clusterQemuNetIn, prometheus.GaugeValue, jNumberToFloat(qVM.NetIn), node.Name, qVM.Name, @@ -620,7 +667,10 @@ func (e Exporter) Collect(ch chan<- prometheus.Metric) { clusterLxcDiskTotal, prometheus.GaugeValue, jNumberToFloat(lxc.DiskTotal), node.Name, lxc.Name, ) ch <- prometheus.MustNewConstMetric( - clusterLxcDiskFree, prometheus.GaugeValue, jNumberToFloat(lxc.DiskFree), node.Name, lxc.Name, + clusterLxcDiskFree, prometheus.GaugeValue, jNumberToFloat(lxc.DiskUsed), node.Name, lxc.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterLxcDiskUsed, prometheus.GaugeValue, jNumberToFloat(lxc.DiskUsed), node.Name, lxc.Name, ) ch <- prometheus.MustNewConstMetric( clusterLxcDiskRead, prometheus.GaugeValue, jNumberToFloat(lxc.DiskRead), node.Name, lxc.Name, @@ -632,13 +682,19 @@ func (e Exporter) Collect(ch chan<- prometheus.Metric) { clusterLxcRamTotal, prometheus.GaugeValue, jNumberToFloat(lxc.RamTotal), node.Name, lxc.Name, ) ch <- prometheus.MustNewConstMetric( - clusterLxcRamFree, prometheus.GaugeValue, jNumberToFloat(lxc.RamFree), node.Name, lxc.Name, + clusterLxcRamFree, prometheus.GaugeValue, jNumberToFloat(lxc.RamUsed), node.Name, lxc.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterLxcRamUsed, prometheus.GaugeValue, jNumberToFloat(lxc.RamUsed), node.Name, lxc.Name, ) ch <- prometheus.MustNewConstMetric( clusterLxcSwapTotal, prometheus.GaugeValue, jNumberToFloat(lxc.SwapTotal), node.Name, lxc.Name, ) ch <- prometheus.MustNewConstMetric( - clusterLxcSwapFree, prometheus.GaugeValue, jNumberToFloat(lxc.SwapFree), node.Name, lxc.Name, + clusterLxcSwapFree, prometheus.GaugeValue, jNumberToFloat(lxc.SwapUsed), node.Name, lxc.Name, + ) + ch <- prometheus.MustNewConstMetric( + clusterLxcSwapUsed, prometheus.GaugeValue, jNumberToFloat(lxc.SwapUsed), node.Name, lxc.Name, ) ch <- prometheus.MustNewConstMetric( clusterLxcNetIn, prometheus.GaugeValue, jNumberToFloat(lxc.NetIn), node.Name, lxc.Name, From 1a1d8013fc2fb38c3266d3c22ac70268daa63f1b Mon Sep 17 00:00:00 2001 From: "Uwe Ohse (x8 go)" Date: Fri, 17 May 2019 06:52:48 +0000 Subject: [PATCH 3/3] add container/vm cpu usage --- pve_exporter.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pve_exporter.go b/pve_exporter.go index 2a713f7..e1ee373 100644 --- a/pve_exporter.go +++ b/pve_exporter.go @@ -74,6 +74,7 @@ type lxc struct { SwapUsed json.Number `json:"swap"` NetIn json.Number `json:"netin"` NetOut json.Number `json:"netout"` + CpuUsage json.Number `json:"cpu"` } type lxcResponse struct { @@ -93,6 +94,7 @@ type qemu struct { RamUsed json.Number `json:"mem"` NetIn json.Number `json:"netin"` NetOut json.Number `json:"netout"` + CpuUsage json.Number `json:"cpu"` } type qemuResponse struct { @@ -361,6 +363,11 @@ var ( "Total CPU count for each LXC", []string{"node", "lxc"}, nil, ) + clusterLxcCpuUsage = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "lxc", "cpu_seconds"), + "Current CPU usage", + []string{"node", "lxc"}, nil, + ) clusterLxcDiskTotal = prometheus.NewDesc( prometheus.BuildFQName(nameSpace, "lxc", "disk_total"), "Disk size for each LXC", @@ -441,6 +448,11 @@ var ( "Total CPU count for each QEMU VM", []string{"node", "qemu"}, nil, ) + clusterQemuCpuUsage = prometheus.NewDesc( + prometheus.BuildFQName(nameSpace, "qemu", "cpu_seconds"), + "Current CPU usage", + []string{"node", "qemu"}, nil, + ) clusterQemuDiskTotal = prometheus.NewDesc( prometheus.BuildFQName(nameSpace, "qemu", "disk_total"), "Disk size for each QEMU VM", @@ -609,6 +621,9 @@ func (e Exporter) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( clusterQemuCpuCount, prometheus.GaugeValue, jNumberToFloat(qVM.CpuCount), node.Name, qVM.Name, ) + ch <- prometheus.MustNewConstMetric( + clusterQemuCpuUsage, prometheus.GaugeValue, jNumberToFloat(qVM.CpuUsage), node.Name, qVM.Name, + ) ch <- prometheus.MustNewConstMetric( clusterQemuDiskTotal, prometheus.GaugeValue, jNumberToFloat(qVM.DiskTotal), node.Name, qVM.Name, ) @@ -663,6 +678,9 @@ func (e Exporter) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( clusterLxcCpuCount, prometheus.GaugeValue, jNumberToFloat(lxc.CpuCount), node.Name, lxc.Name, ) + ch <- prometheus.MustNewConstMetric( + clusterLxcCpuUsage, prometheus.GaugeValue, jNumberToFloat(lxc.CpuUsage), node.Name, lxc.Name, + ) ch <- prometheus.MustNewConstMetric( clusterLxcDiskTotal, prometheus.GaugeValue, jNumberToFloat(lxc.DiskTotal), node.Name, lxc.Name, )