services/telegraf: refactor and add mongodb

linyinfeng · Sep 23, 2024 · 5f69e6f · 5f69e6f
1 parent d2a35b4
commit 5f69e6f
Show file tree

Hide file tree

Showing 9 changed files with 138 additions and 32 deletions.
diff --git a/nixos/profiles/services/minio/default.nix b/nixos/profiles/services/minio/default.nix
@@ -47,6 +47,9 @@ in
   };
 
   # metrics
+  services.telegraf.extraConfig.outputs.influxdb_v2 = [
+    (config.lib.telegraf.mkMainInfluxdbOutput "minio")
+  ];
   services.telegraf.extraConfig = {
     inputs.prometheus = [
       {

diff --git a/nixos/profiles/services/mongodb/default.nix b/nixos/profiles/services/mongodb/default.nix
@@ -1,22 +1,104 @@
-{ config, ... }:
-{
-  services.mongodb = {
-    enable = true;
-    enableAuth = true;
-    extraConfig = ''
-      net.port: ${toString config.ports.mongodb}
+{ config, lib, ... }:
+lib.mkMerge [
+  # main
+  {
+    services.mongodb = {
+      enable = true;
+      enableAuth = true;
+      extraConfig = ''
+        net.port: ${toString config.ports.mongodb}
+      '';
+      initialRootPassword = "temporary"; # will be replaced in initialScript
+      initialScript = config.sops.templates."mongodb-init.js".path;
+    };
+    sops.templates."mongodb-init.js" = {
+      content = ''
+        db.changeUserPassword("root", "${config.sops.placeholder."mongodb_admin_password"}")
+      '';
+      owner = config.services.mongodb.user;
+    };
+    sops.secrets."mongodb_admin_password" = {
+      terraformOutput.enable = true;
+      restartUnits = [ ]; # needs manual rotation
+    };
+  }
+
+  # monitoring
+  {
+    systemd.services.mongodb-monitor-setup = {
+      script = ''
+        mongodb_admin_password="$(cat "$CREDENTIALS_DIRECTORY/mongodb-admin-password")"
+        mongo --username root --password "$mongodb_admin_password" admin "$CREDENTIALS_DIRECTORY/mongodb-init.js"
+      '';
+      requires = [
+        "mongodb.service"
+      ];
+      after = [
+        "mongodb.service"
+      ];
+      path = [
+        config.services.mongodb.package
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        LoadCredential = [
+          "mongodb-admin-password:${config.sops.secrets."mongodb_admin_password".path}"
+          "mongodb-init.js:${config.sops.templates."mongodb-monitor-init.js".path}"
+        ];
+      };
+      restartTriggers = [
+        config.sops.templates."mongodb-monitor-init.js".content
+      ];
+    };
+    sops.templates."mongodb-monitor-init.js".content = ''
+      if (db.getUser("monitor") == null) {
+        db.createUser({
+          user: "monitor",
+          pwd: "temporary",
+          roles: []
+        });
+      };
+      db.updateUser("monitor", {
+        roles: [ { role: "clusterMonitor", db: "admin" } ]
+      });
+      db.changeUserPassword("monitor", "${config.sops.placeholder."mongodb_monitor_password"}");
     '';
-    initialRootPassword = "temporary"; # will be replaced in initialScript
-    initialScript = config.sops.templates."mongodb-init.js".path;
-  };
-  sops.templates."mongodb-init.js" = {
-    content = ''
-      db.changeUserPassword("root", "${config.sops.placeholder."mongodb_admin_password"}")
+
+    services.telegraf.extraConfig.outputs.influxdb_v2 = [
+      (config.lib.telegraf.mkMainInfluxdbOutput "mongodb")
+    ];
+    services.telegraf.extraConfig = {
+      inputs = {
+        mongodb = [
+          {
+            servers = [
+              "mongodb://monitor:\${MONGODB_MONITOR_PASSWORD}@localhost:${toString config.ports.mongodb}/?connect=direct"
+            ];
+            tags.output_bucket = "mongodb";
+          }
+        ];
+      };
+    };
+    services.telegraf.environmentFiles = [
+      config.sops.templates."telegraf-mongodb-env".path
+    ];
+    systemd.services.telegraf = {
+      requires = [ "mongodb-monitor-setup.service" ];
+      after = [ "mongodb-monitor-setup.service" ];
+      restartTriggers = [
+        config.sops.templates."telegraf-mongodb-env".content
+      ];
+    };
+    sops.templates."telegraf-mongodb-env".content = ''
+      MONGODB_MONITOR_PASSWORD=${config.sops.placeholder."mongodb_monitor_password"}
     '';
-    owner = config.services.mongodb.user;
-  };
-  sops.secrets."mongodb_admin_password" = {
-    terraformOutput.enable = true;
-    restartUnits = [ ]; # needs manual rotation
-  };
-}
+    sops.secrets."mongodb_monitor_password" = {
+      terraformOutput.enable = true;
+      restartUnits = [
+        "mongodb-monitor-setup.service"
+        "telegraf.service"
+      ];
+    };
+  }
+]
diff --git a/nixos/profiles/services/sicp-staging/default.nix b/nixos/profiles/services/sicp-staging/default.nix
@@ -140,6 +140,9 @@ in
       mongodb_admin_password="$(cat "$CREDENTIALS_DIRECTORY/mongodb-admin-password")"
       mongo --username root --password "$mongodb_admin_password" admin "$CREDENTIALS_DIRECTORY/mongodb-init.js"
     '';
+    requires = [
+      "mongodb.service"
+    ];
     after = [
       "mongodb.service"
     ];
@@ -190,7 +193,8 @@ in
     if (db.getUser("sicp_staging") == null) {
       db.createUser({
         user: "sicp_staging",
-        pwd: "temporary"
+        pwd: "temporary",
+        roles: []
       });
     };
     db.updateUser("sicp_staging", {

diff --git a/nixos/profiles/services/telegraf-http/default.nix b/nixos/profiles/services/telegraf-http/default.nix
@@ -176,6 +176,9 @@ in
       }
     )
   ];
+  services.telegraf.extraConfig.outputs.influxdb_v2 = [
+    (config.lib.telegraf.mkMainInfluxdbOutput "http")
+  ];
   services.telegraf.extraConfig = {
     inputs = {
       http_response = lib.lists.map (code: {

diff --git a/nixos/profiles/services/telegraf-system/default.nix b/nixos/profiles/services/telegraf-system/default.nix
@@ -1,5 +1,8 @@
-{ pkgs, ... }:
+{ config, pkgs, ... }:
 {
+  services.telegraf.extraConfig.outputs.influxdb_v2 = [
+    (config.lib.telegraf.mkMainInfluxdbOutput "system")
+  ];
   services.telegraf.extraConfig = {
     inputs = {
       cpu = [

diff --git a/nixos/profiles/services/telegraf/default.nix b/nixos/profiles/services/telegraf/default.nix
@@ -1,6 +1,6 @@
 { config, ... }:
 let
-  mainInfluxdb = bucket: {
+  mkMainInfluxdbOutput = bucket: {
     urls = [ config.lib.self.data.influxdb_url ];
     token = "$INFLUX_TOKEN";
     organization = "main-org";
@@ -23,13 +23,13 @@ in
         flush_jitter = "5s";
       };
       outputs.influxdb_v2 = [
-        (mainInfluxdb "system")
-        (mainInfluxdb "minio")
-        (mainInfluxdb "minecraft")
-        (mainInfluxdb "http")
+        # make using config.lib.telegraf.mkMainInfluxdbOutput
       ];
     };
   };
+  lib.telegraf = {
+    inherit mkMainInfluxdbOutput;
+  };
   sops.secrets."influxdb_token" = {
     terraformOutput.enable = true;
     restartUnits = [ "telegraf.service" ];

diff --git a/secrets/terraform/hosts/mtl0.yaml b/secrets/terraform/hosts/mtl0.yaml
@@ -24,6 +24,7 @@ minio_pastebin_key_id: ENC[AES256_GCM,data:MbjOoxOlKYU=,iv:L8Eh5WMfnUbTBmFmN2qKd
 minio_sicp_staging_access_key: ENC[AES256_GCM,data:OFbTlTwBfwooV0uxRd2Mxnf2Ve4L1KrfjWEMo/ujHwN8iDp5Q3pYiPke+9sKOhYe3kbS0pMbUME=,iv:vo6cFNIjZN58wrsfPEtGgQJl8s1WiFsNi/j3gMxjMcg=,tag:kcfJmQ/x/zwI9wo8vJ7zdg==,type:str]
 minio_sicp_staging_key_id: ENC[AES256_GCM,data:PiHXEg6I6z+zhSPV,iv:/904fBXuoRnUkw0osCnl/LJ4a0y9HKZ4rzWUpiB76qg=,tag:SxU0lHZ3DI32kg67O2xyDw==,type:str]
 mongodb_admin_password: ENC[AES256_GCM,data:Jb/tl1qrfFuZvw5Syk6pS6rCzN3G/jEry2/5iEkMfdiXvJaeyKHGXv0kFwr9Yd1/RT3uufbG4rqYUfenrR982A==,iv:E7fjypvS39MucA4k+WtcR4OjytRkhD0VxkkJ49SFzMo=,tag:pNlkjTxy6hVEFjacTXv73Q==,type:str]
+mongodb_monitor_password: ENC[AES256_GCM,data:nzx6NhjK7mLAMDu6Y8h4VVSyLqyCyYx0Ne+Gi5NKxCy2dnUHs77lXYikGukPjL7ONZ8bXTprtWTFSwrvjN/ZCw==,iv:YfHebW0IohQnba0v0Y0uQeCfN5qs9xc0LipvZHRQ1ak=,tag:FyI3kZjznbY38RvCA75nnA==,type:str]
 mongodb_sicp_staging_password: ENC[AES256_GCM,data:3AIHGpGpPqK42eWi+I5gsxMPxUgVtnCvz+LwTKrh8buItjO/eOjFVZc7XQ2pR5/6uipwOPIIHo2D1J+kTi70YA==,iv:cAOUn0wK78PXl9+RWIV7Q6GNUNmVBcKEV9EYWVVqNvY=,tag:fMojF//ywGWdvN5fjAsNqg==,type:str]
 rabbitmq_sicp_staging_password: ENC[AES256_GCM,data:vDYZ7rVu6OHprYTk7cHcmWi/2F9knEXYPU5bnefIey8=,iv:b6Yn4iToOvuT6xXSgouTJJImyjRKCz0cym3BJcVct/8=,tag:03Awa2oS3V0QqEAFEGkRdg==,type:str]
 restic_password: ENC[AES256_GCM,data:vaWjEKDxLcRDf++2yshq5WynYCkuSGYX/RxDx4mCc2I=,iv:/a41CRAQzBjq6z31UeJ01Zmqr8FfSmmAHrAZibY4Q8Y=,tag:I1TcOu83wfuRETUY3QksFg==,type:str]
@@ -64,8 +65,8 @@ sops:
         V2hRNEhCdHYzMUhvMGFYMG5vcFVmTDAKJfGXQKrLecTN7vTSpNmTXzyJWLEFs5g8
         l8iDsxeSySYsd23aJ0MNwxDOx7xHE90iOuFqnhdGQl2B2wF6HDfm1w==
         -----END AGE ENCRYPTED FILE-----
-  lastmodified: "2024-09-22T11:13:00Z"
-  mac: ENC[AES256_GCM,data:3d9LuSAoo+ASUs9qgXjlT4+GO6uBsZZHd24QxVHrF7WiY3rik/kNwVW1MRVZEeT3j/qoOg4V1QVdT1nU6QMyvRvOaeiSOIm96uSGF6r1UoVDE87DB1ryyxo/em6DSFglraLdFv+FGGXClysIEsbgbq3g7g4rTTo2jdMQ1Gh3Ank=,iv:nqJtyXP8ZO5epaNWLD4IOeWazStuorShqId+3GfhavI=,tag:Ma2y782z7/zvp2QfYLJWow==,type:str]
+  lastmodified: "2024-09-23T06:28:51Z"
+  mac: ENC[AES256_GCM,data:E7YhTmX0mgfw3vpAKZLkSy3CgTN5Nk6k8DtAWX8qPTbkMCoZoQrM6Tbf4963a8Q7LbFwey+liV8JfIYJDEKLIt0r1ZrKONpAvQI+RjeFeeUG/npVXxlGnV+38QzphIe0qWdJ0NUMpHP9RHl29LdV8zVxLKEnGtkWADI1UCCTJ/E=,iv:u2XPQWMScbHpsUsG6cnBDzr2JhAcJIMSPqE+z8PBFwU=,tag:q47h4fYx5ZOyvO2fPgXkyw==,type:str]
   pgp:
     - created_at: "2023-05-11T12:18:58Z"
       enc: |-

diff --git a/terraform/influxdb.tf b/terraform/influxdb.tf
@@ -1,6 +1,6 @@
 locals {
   # currently use self-hosted influxdb
-  # influxdb_url is too expensive
+  # influxdb cloud is too expensive
   influxdb_url = "https://influxdb.li7g.com"
 }
 output "influxdb_url" {

diff --git a/terraform/passwords.tf b/terraform/passwords.tf
@@ -344,8 +344,6 @@ output "gnome_remote_desktop_password" {
   sensitive = true
 }
 
-# SICP staging
-
 resource "random_password" "mongodb_admin" {
   length  = 64
   special = false
@@ -355,6 +353,18 @@ output "mongodb_admin_password" {
   sensitive = true
 }
 
+resource "random_password" "mongodb_monitor" {
+  length  = 64
+  special = false
+}
+output "mongodb_monitor_password" {
+  value     = random_password.mongodb_monitor.result
+  sensitive = true
+}
+
+
+# SICP staging
+
 resource "random_password" "mongodb_sicp_staging" {
   length  = 64
   special = false