fix
This commit is contained in:
parent
c88f86eec2
commit
013d87afdf
SSH key fingerprint:
SHA256:qZ/tzd8lYRtUFSrfBDBMcUqV4GHKxqeqRA3huItgvbk
1 changed files with 187 additions and 193 deletions
|
|
@ -25,207 +25,194 @@ in
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
config = mkIf cfg.enable (mkMerge [{
|
config = mkIf cfg.enable (mkMerge [
|
||||||
services.tailscale = {
|
{
|
||||||
enable = true;
|
services.tailscale = {
|
||||||
permitCertUid = config.services.caddy.user;
|
|
||||||
};
|
|
||||||
|
|
||||||
services.caddy = {
|
|
||||||
enable = true;
|
|
||||||
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
|
|
||||||
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
services.caddy.globalConfig = ''
|
|
||||||
servers {
|
|
||||||
metrics
|
|
||||||
}
|
|
||||||
'';
|
|
||||||
services.restic.server.prometheus = cfg.enable;
|
|
||||||
services.gotosocial.settings = mkIf cfg.enable {
|
|
||||||
metrics-enabled = true;
|
|
||||||
};
|
|
||||||
services.prometheus = mkIf cfg.enable {
|
|
||||||
enable = true;
|
|
||||||
port = 9091;
|
|
||||||
globalConfig.external_labels = { hostname = config.networking.hostName; };
|
|
||||||
remoteWrite = mkIf cfg.grafana.enable [
|
|
||||||
{
|
|
||||||
name = "grafana";
|
|
||||||
url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
|
|
||||||
basic_auth = {
|
|
||||||
username = "1340065";
|
|
||||||
password_file = cfg.grafana.password_file;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
];
|
|
||||||
exporters = {
|
|
||||||
node = {
|
|
||||||
enable = true;
|
|
||||||
enabledCollectors = [
|
|
||||||
"conntrack"
|
|
||||||
"diskstats"
|
|
||||||
"entropy"
|
|
||||||
"filefd"
|
|
||||||
"filesystem"
|
|
||||||
"loadavg"
|
|
||||||
"meminfo"
|
|
||||||
"netdev"
|
|
||||||
"netstat"
|
|
||||||
"stat"
|
|
||||||
"time"
|
|
||||||
"vmstat"
|
|
||||||
"systemd"
|
|
||||||
"logind"
|
|
||||||
"interrupts"
|
|
||||||
"ksmd"
|
|
||||||
];
|
|
||||||
port = 9100;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
scrapeConfigs = [
|
|
||||||
{
|
|
||||||
job_name = "prometheus";
|
|
||||||
static_configs = [
|
|
||||||
{ targets = [ "localhost:${toString config.services.prometheus.port}" ]; }
|
|
||||||
];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
job_name = "node";
|
|
||||||
static_configs = [
|
|
||||||
{ targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
|
|
||||||
];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
alertmanager = {
|
|
||||||
enable = true;
|
enable = true;
|
||||||
listenAddress = "127.0.0.1";
|
permitCertUid = config.services.caddy.user;
|
||||||
extraFlags = [
|
};
|
||||||
"--cluster.advertise-address=127.0.0.1:9093"
|
|
||||||
];
|
services.caddy = {
|
||||||
configuration = {
|
enable = true;
|
||||||
route = {
|
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
|
||||||
receiver = "ntfy";
|
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
|
||||||
};
|
'';
|
||||||
receivers = [
|
};
|
||||||
|
|
||||||
|
services.caddy.globalConfig = ''
|
||||||
|
servers {
|
||||||
|
metrics
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
services.restic.server.prometheus = cfg.enable;
|
||||||
|
services.gotosocial.settings = mkIf cfg.enable {
|
||||||
|
metrics-enabled = true;
|
||||||
|
};
|
||||||
|
services.ntfy-sh.settings.enable-metrics = true;
|
||||||
|
|
||||||
|
services.prometheus = mkIf cfg.enable
|
||||||
|
{
|
||||||
|
enable = true;
|
||||||
|
port = 9091;
|
||||||
|
globalConfig.external_labels = { hostname = config.networking.hostName; };
|
||||||
|
remoteWrite = mkIf cfg.grafana.enable [
|
||||||
{
|
{
|
||||||
name = "ntfy";
|
name = "grafana";
|
||||||
webhook_configs = [
|
url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
|
||||||
|
basic_auth = {
|
||||||
|
username = "1340065";
|
||||||
|
password_file = cfg.grafana.password_file;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
exporters = {
|
||||||
|
node = {
|
||||||
|
enable = true;
|
||||||
|
enabledCollectors = [
|
||||||
|
"conntrack"
|
||||||
|
"diskstats"
|
||||||
|
"entropy"
|
||||||
|
"filefd"
|
||||||
|
"filesystem"
|
||||||
|
"loadavg"
|
||||||
|
"meminfo"
|
||||||
|
"netdev"
|
||||||
|
"netstat"
|
||||||
|
"stat"
|
||||||
|
"time"
|
||||||
|
"vmstat"
|
||||||
|
"systemd"
|
||||||
|
"logind"
|
||||||
|
"interrupts"
|
||||||
|
"ksmd"
|
||||||
|
];
|
||||||
|
port = 9100;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
scrapeConfigs = [
|
||||||
|
{
|
||||||
|
job_name = "prometheus";
|
||||||
|
static_configs = [
|
||||||
|
{ targets = [ "localhost:${toString config.services.prometheus.port}" ]; }
|
||||||
|
];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
job_name = "node";
|
||||||
|
static_configs = [
|
||||||
|
{ targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
alertmanager = {
|
||||||
|
enable = true;
|
||||||
|
listenAddress = "127.0.0.1";
|
||||||
|
logLevel = "debug";
|
||||||
|
configuration = {
|
||||||
|
route = {
|
||||||
|
receiver = "ntfy";
|
||||||
|
};
|
||||||
|
receivers = [
|
||||||
{
|
{
|
||||||
url = "https://ntfy.xinyang.life/prometheus-alerts";
|
name = "ntfy";
|
||||||
send_resolved = true;
|
webhook_configs = [
|
||||||
|
{
|
||||||
|
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
|
||||||
|
Alert {{.status}}
|
||||||
|
{{range .alerts}}-----{{range $k,$v := .labels}}
|
||||||
|
{{$k}}={{$v}}{{end}}
|
||||||
|
{{end}}
|
||||||
|
''}";
|
||||||
|
send_resolved = true;
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
alertmanagers = [
|
||||||
|
{
|
||||||
|
scheme = "http";
|
||||||
|
static_configs = [
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
|
||||||
|
];
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
alertmanagers = [
|
rules = let mkRule = condition: { ... }@rule: (if condition then [ rule ] else [ ]); in [
|
||||||
{
|
(lib.generators.toYAML { } {
|
||||||
scheme = "http";
|
groups = (mkRule true
|
||||||
path_prefix = "/alertmanager";
|
{
|
||||||
static_configs = [
|
name = "system_alerts";
|
||||||
{
|
rules = [
|
||||||
targets = [
|
{
|
||||||
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
|
alert = "SystemdFailedUnits";
|
||||||
];
|
expr = "node_systemd_unit_state{state=\"failed\"} > 0";
|
||||||
}
|
for = "5m";
|
||||||
|
labels = { severity = "critical"; };
|
||||||
|
annotations = { summary = "Systemd has failed units on {{ $labels.instance }}"; description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"; };
|
||||||
|
}
|
||||||
|
{
|
||||||
|
alert = "HighLoadAverage";
|
||||||
|
expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
|
||||||
|
for = "1m";
|
||||||
|
labels = { severity = "warning"; };
|
||||||
|
annotations = { summary = "High load average detected on {{ $labels.instance }}"; description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."; };
|
||||||
|
}
|
||||||
|
{
|
||||||
|
alert = "HighTransmitTraffic";
|
||||||
|
expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
|
||||||
|
for = "1m";
|
||||||
|
labels = { severity = "warning"; };
|
||||||
|
annotations = { summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"; description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."; };
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}) ++ (mkRule config.services.restic.server.enable {
|
||||||
|
name = "restic_alerts";
|
||||||
|
rules = [
|
||||||
|
{
|
||||||
|
alert = "ResticCheckFailed";
|
||||||
|
expr = "restic_check_success == 0";
|
||||||
|
for = "5m";
|
||||||
|
labels = { severity = "critical"; };
|
||||||
|
annotations = { summary = "Restic check failed (instance {{ $labels.instance }})"; description = "Restic check failed\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; };
|
||||||
|
}
|
||||||
|
{
|
||||||
|
alert = "ResticOutdatedBackup";
|
||||||
|
expr = "time() - restic_backup_timestamp > 518400";
|
||||||
|
for = "0m";
|
||||||
|
labels = { severity = "critical"; };
|
||||||
|
annotations = { summary = "Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated"; description = "Restic backup is outdated\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; };
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}) ++ (mkRule config.services.caddy.enable {
|
||||||
|
name = "caddy_alerts";
|
||||||
|
rules = [
|
||||||
|
{
|
||||||
|
alert = "UpstreamHealthy";
|
||||||
|
expr = "caddy_reverse_proxy_upstreams_healthy == 0";
|
||||||
|
for = "5m";
|
||||||
|
labels = { severity = "critical"; };
|
||||||
|
annotations = { summary = "Upstream {{ $labels.unstream }} not healthy"; };
|
||||||
|
}
|
||||||
|
{
|
||||||
|
alert = "HighRequestLatency";
|
||||||
|
expr = "histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 0.5";
|
||||||
|
for = "2m";
|
||||||
|
labels = { severity = "warning"; };
|
||||||
|
annotations = { summary = "High request latency on {{ $labels.instance }}"; description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; };
|
||||||
|
}
|
||||||
|
];
|
||||||
|
});
|
||||||
|
})
|
||||||
];
|
];
|
||||||
}
|
};
|
||||||
];
|
}
|
||||||
|
|
||||||
rules = [
|
|
||||||
''
|
|
||||||
groups:
|
|
||||||
- name: system_alerts
|
|
||||||
rules:
|
|
||||||
- alert: SystemdFailedUnits
|
|
||||||
expr: node_systemd_unit_state{state="failed"} > 0
|
|
||||||
for: 5m
|
|
||||||
labels:
|
|
||||||
severity: critical
|
|
||||||
annotations:
|
|
||||||
summary: "Systemd has failed units on {{ $labels.instance }}"
|
|
||||||
description: "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"
|
|
||||||
|
|
||||||
- alert: HighLoadAverage
|
|
||||||
expr: node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode="idle"})
|
|
||||||
for: 1m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
annotations:
|
|
||||||
summary: "High load average detected on {{ $labels.instance }}"
|
|
||||||
description: "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."
|
|
||||||
|
|
||||||
- alert: HighTransmitTraffic
|
|
||||||
expr: rate(node_network_transmit_bytes_total{device!="lo"}[5m]) > 100000000
|
|
||||||
for: 1m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
annotations:
|
|
||||||
summary: "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"
|
|
||||||
description: "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."
|
|
||||||
''
|
|
||||||
(if config.services.restic.server.enable then
|
|
||||||
''
|
|
||||||
groups:
|
|
||||||
- name: restic_alerts
|
|
||||||
rules:
|
|
||||||
- alert: ResticCheckFailed
|
|
||||||
expr: restic_check_success == 0
|
|
||||||
for: 5m
|
|
||||||
labels:
|
|
||||||
severity: critical
|
|
||||||
annotations:
|
|
||||||
summary: Restic check failed (instance {{ $labels.instance }})
|
|
||||||
description: Restic check failed\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
|
|
||||||
|
|
||||||
- alert: ResticOutdatedBackup
|
|
||||||
# 1209600 = 15 days
|
|
||||||
expr: time() - restic_backup_timestamp > 518400
|
|
||||||
for: 0m
|
|
||||||
labels:
|
|
||||||
severity: critical
|
|
||||||
annotations:
|
|
||||||
summary: Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated
|
|
||||||
description: Restic backup is outdated\n VALUE = {{ $value }}\n LABELS = {{ $labels }}
|
|
||||||
'' else "")
|
|
||||||
(if config.services.caddy.enable then ''
|
|
||||||
groups:
|
|
||||||
- name: caddy_alerts
|
|
||||||
rules:
|
|
||||||
- alert: HighHttpErrorRate
|
|
||||||
expr: rate(caddy_http_request_duration_seconds_count{status_code=~"5.."}[5m]) / rate(caddy_http_request_duration_seconds_count[5m]) > 0.01
|
|
||||||
for: 10m
|
|
||||||
labels:
|
|
||||||
severity: critical
|
|
||||||
annotations:
|
|
||||||
summary: "High error rate on {{ $labels.instance }}"
|
|
||||||
description: "More than 1% of HTTP requests are errors over the last 10 minutes."
|
|
||||||
- alert: CaddyDown
|
|
||||||
expr: up{job="caddy"} == 0
|
|
||||||
for: 5m
|
|
||||||
labels:
|
|
||||||
severity: critical
|
|
||||||
annotations:
|
|
||||||
summary: "Caddy server down on {{ $labels.instance }}"
|
|
||||||
description: "Caddy server is down for more than 5 minutes."
|
|
||||||
- alert: HighRequestLatency
|
|
||||||
expr: histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 0.5
|
|
||||||
for: 2m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
annotations:
|
|
||||||
summary: "High request latency on {{ $labels.instance }}"
|
|
||||||
description: "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."
|
|
||||||
'' else "")
|
|
||||||
];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
{
|
{
|
||||||
services.prometheus.scrapeConfigs = [
|
services.prometheus.scrapeConfigs = [
|
||||||
(mkIf config.services.caddy.enable {
|
(mkIf config.services.caddy.enable {
|
||||||
|
|
@ -246,6 +233,13 @@ in
|
||||||
{ targets = [ "localhost:${toString config.services.gotosocial.settings.port}" ]; }
|
{ targets = [ "localhost:${toString config.services.gotosocial.settings.port}" ]; }
|
||||||
];
|
];
|
||||||
})
|
})
|
||||||
|
(mkIf config.services.ntfy-sh.enable {
|
||||||
|
job_name = "ntfy-sh";
|
||||||
|
static_configs = [
|
||||||
|
{ targets = [ "auth.xinyang.life" ]; }
|
||||||
|
];
|
||||||
|
})
|
||||||
];
|
];
|
||||||
}]);
|
}
|
||||||
|
]);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue