feat: better prometheus integration

This commit is contained in:
xinyangli 2024-07-30 15:56:02 +08:00
parent 1830ab192c
commit 745ce62f88
Signed by: xin
SSH key fingerprint: SHA256:qZ/tzd8lYRtUFSrfBDBMcUqV4GHKxqeqRA3huItgvbk

View file

@ -32,15 +32,16 @@ in
} }
''; '';
services.restic.server.prometheus = cfg.enable; services.restic.server.prometheus = cfg.enable;
services.gotosocial.settings = { services.gotosocial.settings = mkIf cfg.enable {
metrics-enable = true; metrics-enabled = true;
}; };
services.prometheus = mkIf cfg.enable { services.prometheus = mkIf cfg.enable {
enable = true; enable = true;
port = 9091; port = 9091;
globalConfig.external_labels = { hostname = config.networking.hostName; }; globalConfig.external_labels = { hostname = config.networking.hostName; };
remoteWrite = mkIf cfg.grafana.enable [ remoteWrite = mkIf cfg.grafana.enable [
{ name = "grafana"; {
name = "grafana";
url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push"; url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
basic_auth = { basic_auth = {
username = "1340065"; username = "1340065";
@ -51,45 +52,140 @@ in
exporters = { exporters = {
node = { node = {
enable = true; enable = true;
enabledCollectors = [ "systemd" ]; enabledCollectors = [
"conntrack"
"diskstats"
"entropy"
"filefd"
"filesystem"
"loadavg"
"meminfo"
"netdev"
"netstat"
"stat"
"time"
"vmstat"
"systemd"
"logind"
"interrupts"
"ksmd"
];
port = 9100; port = 9100;
}; };
}; };
scrapeConfigs = [ scrapeConfigs = [
{ job_name = "prometheus"; {
job_name = "prometheus";
static_configs = [ static_configs = [
{ targets = [ "localhost:${toString config.services.prometheus.port}" ]; } { targets = [ "localhost:${toString config.services.prometheus.port}" ]; }
]; ];
} }
{ job_name = "node"; {
job_name = "node";
static_configs = [ static_configs = [
{ targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; } { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
]; ];
} }
]; ];
alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
extraFlags = [
"--cluster.advertise-address=127.0.0.1:9093"
];
configuration = {
route = {
receiver = "ntfy";
};
receivers = [
{
name = "ntfy";
webhook_configs = [
{
url = "https://ntfy.xinyang.life/prometheus-alerts";
send_resolved = true;
}
];
}
];
};
};
alertmanagers = [
{
scheme = "http";
path_prefix = "/alertmanager";
static_configs = [
{
targets = [
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
];
}
];
}
];
rules = [
''
groups:
- name: system_alerts
rules:
- alert: SystemdFailedUnits
expr: node_systemd_unit_state{state="failed"} > 0
for: 5m
labels:
severity: critical
annotations:
summary: "Systemd has failed units on {{ $labels.instance }}"
description: "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"
- alert: HighLoadAverage
expr: node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode="idle"})
for: 1m
labels:
severity: warning
annotations:
summary: "High load average detected on {{ $labels.instance }}"
description: "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."
- alert: HighTransmitTraffic
expr: rate(node_network_transmit_bytes_total{device!="lo"}[5m]) > 100000000
for: 1m
labels:
severity: warning
annotations:
summary: "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"
description: "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."
''
(if config.services.restic.server.enable then
''
groups:
- name: restic_alerts
'' else "")
];
}; };
} }
{ {
services.prometheus.scrapeConfigs = [ services.prometheus.scrapeConfigs = [
( mkIf config.services.caddy.enable { (mkIf config.services.caddy.enable {
job_name = "caddy"; job_name = "caddy";
static_configs = [ static_configs = [
{ targets = [ "localhost:2019" ]; } { targets = [ "localhost:2019" ]; }
]; ];
}) })
( mkIf config.services.restic.server.enable { (mkIf config.services.restic.server.enable {
job_name = "restic"; job_name = "restic";
static_configs = [ static_configs = [
{ targets = [ config.services.restic.server.listenAddress ]; } { targets = [ config.services.restic.server.listenAddress ]; }
]; ];
}) })
( mkIf config.services.gotosocial.enable { (mkIf config.services.gotosocial.enable {
job_name = "gotosocial"; job_name = "gotosocial";
static_configs = [ static_configs = [
{ targets = [ "localhost:${toString config.services.gotosocial.settings.port}" ]; } { targets = [ "localhost:${toString config.services.gotosocial.settings.port}" ]; }
]; ];
}) })
]; ];
} }]);
]);
} }