nixos-config/modules/nixos/prometheus/default.nix

{
  config,
  pkgs,
  lib,
  ...
}:

with lib;

let
  cfg = config.custom.prometheus;
  mkExporterOption =
    enableOption:
    (mkOption {
      type = types.bool;
      default = enableOption;
      description = "Enable this exporter";
    });

  mkRulesOption = mkOption {
    type = types.listOf (
      types.submodule {
        options = {
          name = mkOption { type = lib.types.str; };
          rules = mkOption { type = lib.types.listOf lib.types.attrs; };
        };
      }
    );
  };
in
{
  imports = [
    ./blackbox.nix
    ./caddy.nix
    ./gotosocial.nix
    ./immich.nix
    ./miniflux.nix
    ./ntfy-sh.nix
    ./restic.nix
  ];

  options = {
    custom.prometheus = {
      enable = mkEnableOption "Prometheus instance";
      exporters = {
        enable = mkOption {
          type = types.bool;
          default = false;
          description = "Enable Prometheus exporter on every supported services";
        };

        restic.enable = mkExporterOption config.services.restic.server.enable;
        blackbox.enable = mkExporterOption false;
        caddy.enable = mkExporterOption config.services.caddy.enable;
        gotosocial.enable = mkExporterOption config.services.gotosocial.enable;
        immich.enable = mkExporterOption config.services.immich.enable;
        miniflux.enable = mkExporterOption config.services.miniflux.enable;
        ntfy-sh.enable = mkExporterOption config.services.ntfy-sh.enable;
      };
      grafana = {
        enable = mkEnableOption "Grafana Cloud";
        password_file = mkOption { type = types.path; };
      };
      ruleModules = mkRulesOption;
    };
  };

  config = mkIf cfg.enable {
    services.tailscale = {
      enable = true;
      permitCertUid = config.services.caddy.user;
      openFirewall = true;
    };

    services.caddy = {
      enable = true;
      virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
        reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
      '';
    };

    services.prometheus = mkIf cfg.enable {
      enable = true;
      port = 9091;
      globalConfig.external_labels = {
        hostname = config.networking.hostName;
      };
      remoteWrite = mkIf cfg.grafana.enable [
        {
          name = "grafana";
          url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
          basic_auth = {
            username = "1340065";
            password_file = cfg.grafana.password_file;
          };
        }
      ];
      exporters = {
        node = {
          enable = true;
          enabledCollectors = [
            "loadavg"
            "time"
            "systemd"
          ];
          listenAddress = "127.0.0.1";
          port = 9100;
        };
      };
      scrapeConfigs = [
        {
          job_name = "prometheus";
          static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
        }
        {
          job_name = "node";
          static_configs = [
            { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
          ];
        }
      ];

      alertmanager = {
        enable = true;
        listenAddress = "127.0.0.1";
        logLevel = "debug";
        configuration = {
          route = {
            receiver = "ntfy";
          };
          receivers = [
            {
              name = "ntfy";
              webhook_configs = [
                {
                  url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
                    Alert {{.status}}
                    {{range .alerts}}-----{{range $k,$v := .labels}}
                    {{$k}}={{$v}}{{end}}
                    {{end}}
                  ''}";
                  send_resolved = true;
                }
              ];
            }
          ];
        };
      };

      alertmanagers = [
        {
          scheme = "http";
          static_configs = [
            {
              targets = [
                "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
              ];
            }
          ];
        }
      ];
      rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
    };
    custom.prometheus.ruleModules = [
      {
        name = "system_alerts";
        rules = [
          {
            alert = "SystemdFailedUnits";
            expr = "node_systemd_unit_state{state=\"failed\"} > 0";
            for = "5m";
            labels = {
              severity = "critical";
            };
            annotations = {
              summary = "Systemd has failed units on {{ $labels.instance }}";
              description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!";
            };
          }
          {
            alert = "HighLoadAverage";
            expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
            for = "1m";
            labels = {
              severity = "warning";
            };
            annotations = {
              summary = "High load average detected on {{ $labels.instance }}";
              description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs.";
            };
          }
          {
            alert = "HighTransmitTraffic";
            expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
            for = "1m";
            labels = {
              severity = "warning";
            };
            annotations = {
              summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})";
              description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute.";
            };
          }
          {
            alert = "NetworkTrafficExceedLimit";
            expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
            for = "0m";
            labels = {
              severity = "critical";
            };
            annotations = {
              summary = "Outbound network traffic exceed 300GB for last 30 day";
            };
          }
          {
            alert = "JobDown";
            expr = "up == 0";
            for = "1m";
            labels = {
              severity = "critical";
            };
            annotations = {
              summary = "Job {{ $labels.job }} down for 1m.";
            };
          }
        ];
      }
    ];
  };
}