From 5fe7ff043449c8c4b42b34428e59cb161590150e Mon Sep 17 00:00:00 2001
From: xinyangli <lixinyang411@gmail.com>
Date: Wed, 31 Jul 2024 14:27:03 +0800
Subject: [PATCH] modules/prometheus: split exporters and alerts to seperate
 files

---
 modules/nixos/default.nix               |   2 +-
 modules/nixos/prometheus.nix            | 245 ------------------------
 modules/nixos/prometheus/caddy.nix      |  45 +++++
 modules/nixos/prometheus/default.nix    | 194 +++++++++++++++++++
 modules/nixos/prometheus/gotosocial.nix |  19 ++
 modules/nixos/prometheus/ntfy-sh.nix    |  17 ++
 modules/nixos/prometheus/restic.nix     |  41 ++++
 7 files changed, 317 insertions(+), 246 deletions(-)
 delete mode 100644 modules/nixos/prometheus.nix
 create mode 100644 modules/nixos/prometheus/caddy.nix
 create mode 100644 modules/nixos/prometheus/default.nix
 create mode 100644 modules/nixos/prometheus/gotosocial.nix
 create mode 100644 modules/nixos/prometheus/ntfy-sh.nix
 create mode 100644 modules/nixos/prometheus/restic.nix

diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix
index 7908b49..0b31ac1 100644
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@@ -5,7 +5,7 @@
     ./common-settings/nix-conf.nix
     ./restic.nix
     ./vaultwarden.nix
-    ./prometheus.nix
+    ./prometheus
     ./hedgedoc.nix
     ./sing-box.nix
     ./kanidm-client.nix
diff --git a/modules/nixos/prometheus.nix b/modules/nixos/prometheus.nix
deleted file mode 100644
index d7e23fc..0000000
--- a/modules/nixos/prometheus.nix
+++ /dev/null
@@ -1,245 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with lib;
-
-let
-  cfg = config.custom.prometheus;
-in
-{
-  options = {
-    custom.prometheus = {
-      enable = mkEnableOption "Prometheus instance";
-      exporters = {
-        enable = mkOption {
-          type = types.bool;
-          default = false;
-          description = "Enable Prometheus exporter on every supported services";
-        };
-      };
-      grafana = {
-        enable = mkEnableOption "Grafana Cloud";
-        password_file = mkOption {
-          type = types.path;
-        };
-      };
-    };
-  };
-
-  config = mkIf cfg.enable (mkMerge [
-    {
-      services.tailscale = {
-        enable = true;
-        permitCertUid = config.services.caddy.user;
-      };
-
-      services.caddy = {
-        enable = true;
-        virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
-          reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
-        '';
-      };
-
-      services.caddy.globalConfig = ''
-        servers {
-          metrics
-        }
-      '';
-      services.restic.server.prometheus = cfg.enable;
-      services.gotosocial.settings = mkIf cfg.enable {
-        metrics-enabled = true;
-      };
-      services.ntfy-sh.settings.enable-metrics = true;
-
-      services.prometheus = mkIf cfg.enable
-        {
-          enable = true;
-          port = 9091;
-          globalConfig.external_labels = { hostname = config.networking.hostName; };
-          remoteWrite = mkIf cfg.grafana.enable [
-            {
-              name = "grafana";
-              url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
-              basic_auth = {
-                username = "1340065";
-                password_file = cfg.grafana.password_file;
-              };
-            }
-          ];
-          exporters = {
-            node = {
-              enable = true;
-              enabledCollectors = [
-                "conntrack"
-                "diskstats"
-                "entropy"
-                "filefd"
-                "filesystem"
-                "loadavg"
-                "meminfo"
-                "netdev"
-                "netstat"
-                "stat"
-                "time"
-                "vmstat"
-                "systemd"
-                "logind"
-                "interrupts"
-                "ksmd"
-              ];
-              port = 9100;
-            };
-          };
-          scrapeConfigs = [
-            {
-              job_name = "prometheus";
-              static_configs = [
-                { targets = [ "localhost:${toString config.services.prometheus.port}" ]; }
-              ];
-            }
-            {
-              job_name = "node";
-              static_configs = [
-                { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
-              ];
-            }
-          ];
-
-          alertmanager = {
-            enable = true;
-            listenAddress = "127.0.0.1";
-            logLevel = "debug";
-            configuration = {
-              route = {
-                receiver = "ntfy";
-              };
-              receivers = [
-                {
-                  name = "ntfy";
-                  webhook_configs = [
-                    {
-                      url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
-                        Alert {{.status}}
-                        {{range .alerts}}-----{{range $k,$v := .labels}}
-                        {{$k}}={{$v}}{{end}}
-                        {{end}}
-                      ''}";
-                      send_resolved = true;
-                    }
-                  ];
-                }
-              ];
-            };
-          };
-
-          alertmanagers = [
-            {
-              scheme = "http";
-              static_configs = [
-                {
-                  targets = [
-                    "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
-                  ];
-                }
-              ];
-            }
-          ];
-
-          rules = let mkRule = condition: { ... }@rule: (if condition then [ rule ] else [ ]); in [
-            (lib.generators.toYAML { } {
-              groups = (mkRule true
-                {
-                  name = "system_alerts";
-                  rules = [
-                    {
-                      alert = "SystemdFailedUnits";
-                      expr = "node_systemd_unit_state{state=\"failed\"} > 0";
-                      for = "5m";
-                      labels = { severity = "critical"; };
-                      annotations = { summary = "Systemd has failed units on {{ $labels.instance }}"; description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"; };
-                    }
-                    {
-                      alert = "HighLoadAverage";
-                      expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
-                      for = "1m";
-                      labels = { severity = "warning"; };
-                      annotations = { summary = "High load average detected on {{ $labels.instance }}"; description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."; };
-                    }
-                    {
-                      alert = "HighTransmitTraffic";
-                      expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
-                      for = "1m";
-                      labels = { severity = "warning"; };
-                      annotations = { summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"; description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."; };
-                    }
-                  ];
-                }) ++ (mkRule config.services.restic.server.enable {
-                name = "restic_alerts";
-                rules = [
-                  {
-                    alert = "ResticCheckFailed";
-                    expr = "restic_check_success == 0";
-                    for = "5m";
-                    labels = { severity = "critical"; };
-                    annotations = { summary = "Restic check failed (instance {{ $labels.instance }})"; description = "Restic check failed\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}"; };
-                  }
-                  {
-                    alert = "ResticOutdatedBackup";
-                    expr = "time() - restic_backup_timestamp > 518400";
-                    for = "0m";
-                    labels = { severity = "critical"; };
-                    annotations = { summary = "Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated"; description = "Restic backup is outdated\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}"; };
-                  }
-                ];
-              }) ++ (mkRule config.services.caddy.enable {
-                name = "caddy_alerts";
-                rules = [
-                  {
-                    alert = "UpstreamHealthy";
-                    expr = "caddy_reverse_proxy_upstreams_healthy == 0";
-                    for = "5m";
-                    labels = { severity = "critical"; };
-                    annotations = { summary = "Upstream {{ $labels.unstream }} not healthy"; };
-                  }
-                  {
-                    alert = "HighRequestLatency";
-                    expr = "histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 5";
-                    for = "2m";
-                    labels = { severity = "warning"; };
-                    annotations = { summary = "High request latency on {{ $labels.instance }}"; description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; };
-                  }
-                ];
-              });
-            })
-          ];
-        };
-    }
-    {
-      services.prometheus.scrapeConfigs = [
-        (mkIf config.services.caddy.enable {
-          job_name = "caddy";
-          static_configs = [
-            { targets = [ "localhost:2019" ]; }
-          ];
-        })
-        (mkIf config.services.restic.server.enable {
-          job_name = "restic";
-          static_configs = [
-            { targets = [ config.services.restic.server.listenAddress ]; }
-          ];
-        })
-        (mkIf config.services.gotosocial.enable {
-          job_name = "gotosocial";
-          static_configs = [
-            { targets = [ "localhost:${toString config.services.gotosocial.settings.port}" ]; }
-          ];
-        })
-        (mkIf config.services.ntfy-sh.enable {
-          job_name = "ntfy-sh";
-          static_configs = [
-            { targets = [ "auth.xinyang.life" ]; }
-          ];
-        })
-      ];
-    }
-  ]);
-}
diff --git a/modules/nixos/prometheus/caddy.nix b/modules/nixos/prometheus/caddy.nix
new file mode 100644
index 0000000..a62b639
--- /dev/null
+++ b/modules/nixos/prometheus/caddy.nix
@@ -0,0 +1,45 @@
+{ config, lib, ... }:
+let
+  cfg = config.custom.prometheus;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.caddy.globalConfig = lib.mkIf cfg.exporters.caddy.enable ''
+      servers {
+        metrics
+      }
+    '';
+
+    services.prometheus.scrapeConfigs = [
+      (lib.mkIf cfg.exporters.caddy.enable {
+        job_name = "caddy";
+        static_configs = [
+          { targets = [ "127.0.0.1:2019" ]; }
+        ];
+      })
+    ];
+
+    custom.prometheus.ruleModules = [
+      (lib.mkIf cfg.exporters.caddy.enable {
+        name = "caddy_alerts";
+        rules = [
+          {
+            alert = "UpstreamHealthy";
+            expr = "caddy_reverse_proxy_upstreams_healthy != 1";
+            for = "5m";
+            labels = { severity = "critical"; };
+            annotations = { summary = "Upstream {{ $labels.unstream }} not healthy"; };
+          }
+          {
+            alert = "HighRequestLatency";
+            expr = "histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 5";
+            for = "2m";
+            labels = { severity = "warning"; };
+            annotations = { summary = "High request latency on {{ $labels.instance }}"; description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; };
+          }
+        ];
+      })
+    ];
+  };
+
+}
diff --git a/modules/nixos/prometheus/default.nix b/modules/nixos/prometheus/default.nix
new file mode 100644
index 0000000..803b9aa
--- /dev/null
+++ b/modules/nixos/prometheus/default.nix
@@ -0,0 +1,194 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.custom.prometheus;
+  exporterCfg = config.custom.prometheus.exporters;
+  mkExporterOption = enableOption: (mkOption {
+    type = types.bool;
+    default = enableOption;
+    description = "Enable this exporter";
+  });
+
+  mkRulesOption = mkOption {
+    type = types.listOf (types.submodule {
+      options = {
+        name = mkOption {
+          type = lib.types.str;
+        };
+        rules = mkOption {
+          type = lib.types.listOf lib.types.attrs;
+        };
+      };
+    });
+  };
+in
+{
+  imports = [
+    ./caddy.nix
+    ./gotosocial.nix
+    ./ntfy-sh.nix
+    ./restic.nix
+  ];
+
+  options = {
+    custom.prometheus = {
+      enable = mkEnableOption "Prometheus instance";
+      exporters = {
+        enable = mkOption {
+          type = types.bool;
+          default = false;
+          description = "Enable Prometheus exporter on every supported services";
+        };
+
+        restic.enable = mkExporterOption config.services.restic.server.enable;
+        caddy.enable = mkExporterOption config.services.caddy.enable;
+        gotosocial.enable = mkExporterOption config.services.gotosocial.enable;
+        ntfy-sh.enable = mkExporterOption config.services.gotosocial.enable;
+      };
+      grafana = {
+        enable = mkEnableOption "Grafana Cloud";
+        password_file = mkOption {
+          type = types.path;
+        };
+      };
+      ruleModules = mkRulesOption;
+    };
+  };
+
+  config = mkIf cfg.enable
+    {
+      services.tailscale = {
+        enable = true;
+        permitCertUid = config.services.caddy.user;
+        openFirewall = true;
+      };
+
+      services.caddy = {
+        enable = true;
+        virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
+          reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
+        '';
+      };
+
+      services.prometheus = mkIf cfg.enable
+        {
+          enable = true;
+          port = 9091;
+          globalConfig.external_labels = { hostname = config.networking.hostName; };
+          remoteWrite = mkIf cfg.grafana.enable [
+            {
+              name = "grafana";
+              url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
+              basic_auth = {
+                username = "1340065";
+                password_file = cfg.grafana.password_file;
+              };
+            }
+          ];
+          exporters = {
+            node = {
+              enable = true;
+              enabledCollectors = [
+                "diskstats"
+                "loadavg"
+                "time"
+                "systemd"
+              ];
+              port = 9100;
+            };
+          };
+          scrapeConfigs = [
+            {
+              job_name = "prometheus";
+              static_configs = [
+                { targets = [ "localhost:${toString config.services.prometheus.port}" ]; }
+              ];
+            }
+            {
+              job_name = "node";
+              static_configs = [
+                { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
+              ];
+            }
+          ];
+
+          alertmanager = {
+            enable = true;
+            listenAddress = "127.0.0.1";
+            logLevel = "debug";
+            configuration = {
+              route = {
+                receiver = "ntfy";
+              };
+              receivers = [
+                {
+                  name = "ntfy";
+                  webhook_configs = [
+                    {
+                      url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
+                        Alert {{.status}}
+                        {{range .alerts}}-----{{range $k,$v := .labels}}
+                        {{$k}}={{$v}}{{end}}
+                        {{end}}
+                      ''}";
+                      send_resolved = true;
+                    }
+                  ];
+                }
+              ];
+            };
+          };
+
+          alertmanagers = [
+            {
+              scheme = "http";
+              static_configs = [
+                {
+                  targets = [
+                    "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
+                  ];
+                }
+              ];
+            }
+          ];
+          rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
+        };
+      custom.prometheus.ruleModules = [
+        {
+          name = "system_alerts";
+          rules = [
+            {
+              alert = "SystemdFailedUnits";
+              expr = "node_systemd_unit_state{state=\"failed\"} > 0";
+              for = "5m";
+              labels = { severity = "critical"; };
+              annotations = { summary = "Systemd has failed units on {{ $labels.instance }}"; description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"; };
+            }
+            {
+              alert = "HighLoadAverage";
+              expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
+              for = "1m";
+              labels = { severity = "warning"; };
+              annotations = { summary = "High load average detected on {{ $labels.instance }}"; description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."; };
+            }
+            {
+              alert = "HighTransmitTraffic";
+              expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
+              for = "1m";
+              labels = { severity = "warning"; };
+              annotations = { summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"; description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."; };
+            }
+            {
+              alert = "NetworkTrafficExceedLimit";
+              expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
+              for = "0m";
+              labels = { severity = "critical"; };
+              annotations = { summary = "Outbound network traffic exceed 300GB for last 30 day"; };
+            }
+          ];
+        }
+      ];
+    };
+}
diff --git a/modules/nixos/prometheus/gotosocial.nix b/modules/nixos/prometheus/gotosocial.nix
new file mode 100644
index 0000000..4870e88
--- /dev/null
+++ b/modules/nixos/prometheus/gotosocial.nix
@@ -0,0 +1,19 @@
+{ config, lib, ... }:
+let
+  cfg = config.custom.prometheus;
+in
+{
+  config = lib.mkIf cfg.exporters.gotosocial.enable {
+    services.gotosocial.settings = lib.mkIf cfg.exporters.gotosocial.enable {
+      metrics-enabled = true;
+    };
+    services.prometheus.scrapeConfigs = [
+      {
+        job_name = "gotosocial";
+        static_configs = [
+          { targets = [ "localhost:8080" ]; }
+        ];
+      }
+    ];
+  };
+}
diff --git a/modules/nixos/prometheus/ntfy-sh.nix b/modules/nixos/prometheus/ntfy-sh.nix
new file mode 100644
index 0000000..35d62ff
--- /dev/null
+++ b/modules/nixos/prometheus/ntfy-sh.nix
@@ -0,0 +1,17 @@
+{ config, lib, ... }:
+let
+  cfg = config.custom.prometheus;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.ntfy-sh.settings.enable-metrics = true;
+    services.prometheus.scrapeConfigs = [
+      (lib.mkIf cfg.exporters.ntfy-sh.enable {
+        job_name = "ntfy-sh";
+        static_configs = [
+          { targets = [ "ntfy.xinyang.life" ]; }
+        ];
+      })
+    ];
+  };
+}
diff --git a/modules/nixos/prometheus/restic.nix b/modules/nixos/prometheus/restic.nix
new file mode 100644
index 0000000..80f0316
--- /dev/null
+++ b/modules/nixos/prometheus/restic.nix
@@ -0,0 +1,41 @@
+{ config, lib, ... }:
+let
+  cfg = config.custom.prometheus;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.restic.server.prometheus = true;
+
+    services.prometheus.scrapeConfigs = [
+      (lib.mkIf cfg.exporters.restic.enable {
+        job_name = "restic";
+        static_configs = [
+          { targets = [ config.services.restic.server.listenAddress ]; }
+        ];
+      })
+    ];
+
+    custom.prometheus.ruleModules = [
+      (lib.mkIf cfg.exporters.restic.enable {
+        name = "restic_alerts";
+        rules = [
+          {
+            alert = "ResticCheckFailed";
+            expr = "restic_check_success == 0";
+            for = "5m";
+            labels = { severity = "critical"; };
+            annotations = { summary = "Restic check failed (instance {{ $labels.instance }})"; description = "Restic check failed\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}"; };
+          }
+          {
+            alert = "ResticOutdatedBackup";
+            expr = "time() - restic_backup_timestamp > 518400";
+            for = "0m";
+            labels = { severity = "critical"; };
+            annotations = { summary = "Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated"; description = "Restic backup is outdated\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}"; };
+          }
+        ];
+      })
+    ];
+  };
+
+}