nixos-config/modules/nixos/prometheus/default.nix

230 lines
6.6 KiB
Nix

{
config,
pkgs,
lib,
...
}:
with lib;
let
cfg = config.custom.prometheus;
mkExporterOption =
enableOption:
(mkOption {
type = types.bool;
default = enableOption;
description = "Enable this exporter";
});
mkRulesOption = mkOption {
type = types.listOf (
types.submodule {
options = {
name = mkOption { type = lib.types.str; };
rules = mkOption { type = lib.types.listOf lib.types.attrs; };
};
}
);
};
in
{
imports = [
./blackbox.nix
./caddy.nix
./gotosocial.nix
./immich.nix
./miniflux.nix
./ntfy-sh.nix
./restic.nix
];
options = {
custom.prometheus = {
enable = mkEnableOption "Prometheus instance";
exporters = {
enable = mkOption {
type = types.bool;
default = false;
description = "Enable Prometheus exporter on every supported services";
};
restic.enable = mkExporterOption config.services.restic.server.enable;
blackbox.enable = mkExporterOption false;
caddy.enable = mkExporterOption config.services.caddy.enable;
gotosocial.enable = mkExporterOption config.services.gotosocial.enable;
immich.enable = mkExporterOption config.services.immich.enable;
miniflux.enable = mkExporterOption config.services.miniflux.enable;
ntfy-sh.enable = mkExporterOption config.services.ntfy-sh.enable;
};
grafana = {
enable = mkEnableOption "Grafana Cloud";
password_file = mkOption { type = types.path; };
};
ruleModules = mkRulesOption;
};
};
config = mkIf cfg.enable {
services.tailscale = {
enable = true;
permitCertUid = config.services.caddy.user;
openFirewall = true;
};
services.caddy = {
enable = true;
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
'';
};
services.prometheus = mkIf cfg.enable {
enable = true;
port = 9091;
globalConfig.external_labels = {
hostname = config.networking.hostName;
};
remoteWrite = mkIf cfg.grafana.enable [
{
name = "grafana";
url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
basic_auth = {
username = "1340065";
password_file = cfg.grafana.password_file;
};
}
];
exporters = {
node = {
enable = true;
enabledCollectors = [
"loadavg"
"time"
"systemd"
];
listenAddress = "127.0.0.1";
port = 9100;
};
};
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
}
{
job_name = "node";
static_configs = [
{ targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
];
}
];
alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
logLevel = "debug";
configuration = {
route = {
receiver = "ntfy";
};
receivers = [
{
name = "ntfy";
webhook_configs = [
{
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
Alert {{.status}}
{{range .alerts}}-----{{range $k,$v := .labels}}
{{$k}}={{$v}}{{end}}
{{end}}
''}";
send_resolved = true;
}
];
}
];
};
};
alertmanagers = [
{
scheme = "http";
static_configs = [
{
targets = [
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
];
}
];
}
];
rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
};
custom.prometheus.ruleModules = [
{
name = "system_alerts";
rules = [
{
alert = "SystemdFailedUnits";
expr = "node_systemd_unit_state{state=\"failed\"} > 0";
for = "5m";
labels = {
severity = "critical";
};
annotations = {
summary = "Systemd has failed units on {{ $labels.instance }}";
description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!";
};
}
{
alert = "HighLoadAverage";
expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
for = "1m";
labels = {
severity = "warning";
};
annotations = {
summary = "High load average detected on {{ $labels.instance }}";
description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs.";
};
}
{
alert = "HighTransmitTraffic";
expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
for = "1m";
labels = {
severity = "warning";
};
annotations = {
summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})";
description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute.";
};
}
{
alert = "NetworkTrafficExceedLimit";
expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
for = "0m";
labels = {
severity = "critical";
};
annotations = {
summary = "Outbound network traffic exceed 300GB for last 30 day";
};
}
{
alert = "JobDown";
expr = "up == 0";
for = "1m";
labels = {
severity = "critical";
};
annotations = {
summary = "Job {{ $labels.job }} down for 1m.";
};
}
];
}
];
};
}