Implement local prometheus

README.rst

@@ -59,6 +59,7 @@ Configure prometheus server
       alert:
         PrometheusTargetDown:
           if: 'up != 1'
+          for: 5m
           labels:
             severity: down
           annotations:
@@ -72,6 +73,8 @@ Configure prometheus server
           num_fingerprint_mutexes: 4096
       alertmanager:
         notification_queue_capacity: 10000
+        host: localhost
+        port: 9093
       config:
         global:
           scrape_interval: "15s"
@@ -135,6 +138,16 @@ Configure pushgateway
       enabled: true
       external_port: 15012
 
+.. code-block:: yaml
+
+  prometheus:
+    pushgateway:
+      enabled: true
+      telemetry_path: /metrics
+      bind:
+        address: 0.0.0.0
+        port: 9091
+
 Documentation and Bugs
 ======================
 

metadata/service/collectd_exporter/single.yml

@@ -0,0 +1,7 @@
+applications:
+  - collect_exporter
+parameters:
+  prometheus:
+    collectd_exporter:
+      enabled: true
+      push_path: 'collectd'

metadata/service/pushgateway/local.yml

@@ -0,0 +1,9 @@
+applications:
+  - pushgateway
+parameters:
+  prometheus:
+    pushgateway:
+      enabled: true
+      bind:
+        address: localhost
+        port: 9091

metadata/service/pushgateway/single.yml

@@ -0,0 +1,9 @@
+applications:
+  - pushgateway
+parameters:
+  prometheus:
+    pushgateway:
+      enabled: true
+      bind:
+        address: 0.0.0.0
+        port: 9091

prometheus/alertmanager.sls

@@ -1,12 +1,62 @@
 {% from "prometheus/map.jinja" import alertmanager with context %}
 {%- if alertmanager.enabled %}
 
-{%- if pillar.docker is defined and pillar.docker.host is defined %}
-
 {{alertmanager.dir.config}}/alertmanager.yml:
   file.managed:
   - source: salt://prometheus/files/alertmanager.yml
   - template: jinja
 
+{%- if not (pillar.docker is defined and pillar.docker.host is defined) %}
+
+include:
+  - prometheus.common
+
+alertmanager_tarball:
+  archive.extracted:
+    - name: {{ alertmanager.dir.install }}
+    - source: {{ alertmanager.source }}
+    - source_hash: sha1={{ alertmanager.source_hash }}
+    - archive_format: tar
+    - if_missing: {{ alertmanager.dir.version_path }}
+
+alertmanager_bin_link:
+  file.symlink:
+    - name: /usr/bin/alertmanager
+    - target: {{ alertmanager.dir.version_path }}/alertmanager
+    - require:
+      - archive: alertmanager_tarball
+
+alertmanager_defaults:
+  file.managed:
+    - name: /etc/default/alertmanager
+    - source: salt://prometheus/files/default-alertmanager.jinja
+    - template: jinja
+
+alertmanager_service_unit:
+  file.managed:
+{%- if grains.get('init') == 'systemd' %}
+    - name: /etc/systemd/system/alertmanager.service
+    - source: salt://prometheus/files/alertmanager.systemd.jinja
+{%- elif grains.get('init') == 'upstart' %}
+    - name: /etc/init/alertmanager.conf
+    - source: salt://prometheus/files/alertmanager.upstart.jinja
+{%- endif %}
+    - watch:
+      - file: alertmanager_defaults
+    - require_in:
+      - file: alertmanager_service
+
+alertmanager_service:
+  service.running:
+    - name: alertmanager
+    - enable: True
+    - reload: True
+    - watch:
+      - file: alertmanager_service_unit
+      - file: {{alertmanager.dir.config}}/alertmanager.yml
+      - file: alertmanager_bin_link
+
 {%- endif %}
+
+
 {%- endif %}

prometheus/collectd_exporter.sls

@@ -0,0 +1,56 @@
+{% from "prometheus/map.jinja" import collectd_exporter with context %}
+{%- if collectd_exporter.enabled %}
+
+{%- if not (pillar.docker is defined and pillar.docker.host is defined) %}
+
+include:
+  - prometheus.common
+
+collectd_exporter_tarball:
+  archive.extracted:
+    - name: {{ collectd_exporter.dir.install }}
+    - source: {{ collectd_exporter.source }}
+    - source_hash: sha1={{ collectd_exporter.source_hash }}
+    - archive_format: tar
+    - if_missing: {{ collectd_exporter.dir.version_path }}
+
+collectd_exporter_bin_link:
+  file.symlink:
+    - name: /usr/bin/collectd_exporter
+    - target: {{ collectd_exporter.dir.version_path }}/collectd_exporter
+    - require:
+      - archive: collectd_exporter_tarball
+
+collectd_exporter_defaults:
+  file.managed:
+    - name: /etc/default/collectd_exporter
+    - source: salt://prometheus/files/default-collectd_exporter.jinja
+    - template: jinja
+
+collectd_exporter_service_unit:
+  file.managed:
+{%- if grains.get('init') == 'systemd' %}
+    - name: /etc/systemd/system/collectd_exporter.service
+    - source: salt://prometheus/files/collectd_exporter.systemd.jinja
+{%- elif grains.get('init') == 'upstart' %}
+    - name: /etc/init/collectd_exporter.conf
+    - source: salt://prometheus/files/collectd_exporter.upstart.jinja
+{%- endif %}
+    - watch:
+      - file: collectd_exporter_defaults
+    - require_in:
+      - file: collectd_exporter_service
+
+collectd_exporter_service:
+  service.running:
+    - name: collectd_exporter
+    - enable: True
+    - reload: True
+    - watch:
+      - file: collectd_exporter_service_unit
+      - file: collectd_exporter_bin_link
+
+{%- endif %}
+
+
+{%- endif %}

prometheus/common.sls

@@ -0,0 +1,20 @@
+{% from "prometheus/map.jinja" import server with context %}
+
+prometheus_user:
+  user.present:
+  - name: prometheus
+  - shell: /bin/bash
+  - system: true
+  - home: /srv/prometheus
+
+prometheus_dirs:
+  file.directory:
+  - names:
+    - /var/log/prometheus
+    - {{ server.dir.storage }}
+    - {{ server.dir.config }}
+  - makedirs: true
+  - group: prometheus
+  - user: prometheus
+  - require:
+    - user: prometheus

prometheus/files/alertmanager.systemd.jinja

@@ -0,0 +1,18 @@
+[Unit]
+Description=The Alertmanager handles alerts sent by client applications such as the Prometheus server
+Documentation=https://prometheus.io/docs/alerting/alertmanager/
+Wants=basic.target
+After=basic.target network.target
+
+[Service]
+User=prometheus
+Group=prometheus
+EnvironmentFile=/etc/default/alertmanager
+ExecStart=/usr/bin/alertmanager $ARGS
+ExecReload=/bin/kill -HUP $MAINPID
+KillMode=process
+Restart=always
+RestartSec=42s
+
+[Install]
+WantedBy=multi-user.target

prometheus/files/alertmanager.upstart.jinja

@@ -0,0 +1,30 @@
+# Prometheus Alert Manager (Upstart unit)
+description "The Alertmanager handles alerts sent by client applications such as the Prometheus server."
+start on runlevel [2345]
+stop on runlevel [06]
+
+env ALERTMANAGER=/usr/bin/alertmanager
+env USER=prometheus
+env GROUP=prometheus
+env DEFAULTS=/etc/default/alertmanager
+env RUNDIR=/var/run/alertmanager
+env PID_FILE=/var/run/alertmanager/alertmanager.pid
+pre-start script
+  [ -e $DEFAULTS ] && . $DEFAULTS
+
+  mkdir -p $RUNDIR           || true
+  chmod 0750 $RUNDIR         || true
+  chown $USER:$GROUP $RUNDIR || true
+end script
+
+script
+    # read settings like GOMAXPROCS from "/etc/default/alertmanager", if available.
+    [ -e $DEFAULTS ] && . $DEFAULTS
+
+    export GOMAXPROCS=${GOMAXPROCS:-2}
+    exec start-stop-daemon -c $USER -g $GROUP -p $PID_FILE -x $ALERTMANAGER -S -- $ARGS
+end script
+
+respawn
+respawn limit 10 10
+kill timeout 10

prometheus/files/alerts.yml

@@ -31,7 +31,7 @@
   {%- if alert.get('enabled', True) %}
     ALERT {{ alertname }}
       IF {{ alert.if }}
-      {%- if alert.for is defined %}FOR {{ alert.for }}{%- endif %}
+      {% if alert.for is defined %}FOR {{ alert.for }}{% endif %}
       {%- if alert.labels is defined %}
       LABELS {
         {%- for name, value in alert.labels.iteritems() %}

prometheus/files/collectd_exporter.systemd.jinja

@@ -0,0 +1,18 @@
+[Unit]
+Description="An exporter for collectd. It accepts collectd's binary network protocol as sent by collectd's network plugin and metrics in JSON format via HTTP POST as sent by collectd's write_http plugin, and transforms and exposes them for consumption by Prometheus."
+Documentation=https://prometheus.io/docs/alerting/collectd_exporter/
+Wants=basic.target
+After=basic.target network.target
+
+[Service]
+User=prometheus
+Group=prometheus
+EnvironmentFile=/etc/default/collectd_exporter
+ExecStart=/usr/bin/collectd_exporter $ARGS
+ExecReload=/bin/kill -HUP $MAINPID
+KillMode=process
+Restart=always
+RestartSec=42s
+
+[Install]
+WantedBy=multi-user.target

prometheus/files/collectd_exporter.upstart.jinja

@@ -0,0 +1,30 @@
+# Prometheus Alert Manager (Upstart unit)
+description "An exporter for collectd. It accepts collectd's binary network protocol as sent by collectd's network plugin and metrics in JSON format via HTTP POST as sent by collectd's write_http plugin, and transforms and exposes them for consumption by Prometheus."
+start on runlevel [2345]
+stop on runlevel [06]
+
+env ALERTMANAGER=/usr/bin/collectd_exporter
+env USER=prometheus
+env GROUP=prometheus
+env DEFAULTS=/etc/default/collectd_exporter
+env RUNDIR=/var/run/collectd_exporter
+env PID_FILE=/var/run/collectd_exporter/collectd_exporter.pid
+pre-start script
+  [ -e $DEFAULTS ] && . $DEFAULTS
+
+  mkdir -p $RUNDIR           || true
+  chmod 0750 $RUNDIR         || true
+  chown $USER:$GROUP $RUNDIR || true
+end script
+
+script
+    # read settings like GOMAXPROCS from "/etc/default/collectd_exporter", if available.
+    [ -e $DEFAULTS ] && . $DEFAULTS
+
+    export GOMAXPROCS=${GOMAXPROCS:-2}
+    exec start-stop-daemon -c $USER -g $GROUP -p $PID_FILE -x $ALERTMANAGER -S -- $ARGS
+end script
+
+respawn
+respawn limit 10 10
+kill timeout 10

prometheus/files/default-alertmanager.jinja

@@ -0,0 +1,26 @@
+{%- from "prometheus/map.jinja" import alertmanager with context %}
+# Set the command-line arguments to pass to the server.
+ARGS="-config.file {{alertmanager.dir.config}}/alertmanager.yml -storage.path {{alertmanager.dir.storage}}"
+
+# The alert manager supports the following options:
+#  -config.file string
+#       Alertmanager configuration file name.
+#       (default "/etc/prometheus/alertmanager.yml")
+#  -log.level value
+#       Only log messages with the given severity or above.
+#       Valid levels: [debug, info, warn, error, fatal]. (default info)
+#  -storage.path string
+#       Base path for data storage.
+#       (default "/var/lib/prometheus/alertmanager/")
+#  -web.external-url string
+#       The URL under which Alertmanager is externally reachable (for example,
+#       if Alertmanager is served via a reverse proxy). Used for generating
+#       relative and absolute links back to Alertmanager itself. If the URL has
+#       a path portion, it will be used to prefix all HTTP endpoints served by
+#       Alertmanager. If omitted, relevant URL components will be derived
+#       automatically.
+#  -web.listen-address string
+#       Address to listen on for the web interface and API. (default ":9093")
+#  -web.local-assets string
+#       Path to static assets/templates directory.
+#       (default "/usr/share/prometheus/alertmanager/")

prometheus/files/default-collectd_exporter.jinja

@@ -0,0 +1,3 @@
+{%- from "prometheus/map.jinja" import collectd_exporter with context %}
+# Set the command-line arguments to pass to the server.
+ARGS="-collectd.listen-address {{ collectd_exporter.bind.address }}:{{ collectd_exporter.bind.port }} -web.collectd-push-path {{ collectd_exporter.push_path }} "