Compare commits

...

2 commits

9 changed files with 345 additions and 246 deletions

View file

@ -61,6 +61,7 @@
hedgedoc = {
displayName = "HedgeDoc";
originUrl = "https://docs.xinyang.life/";
originLanding = "https://docs.xinyang.life/auth/oauth2";
allowInsecureClientDisablePkce = true;
scopeMaps = {
hedgedoc-users = [ "openid" "email" "profile" ];

View file

@ -62,6 +62,19 @@ in
group = "kanidm";
};
};
services.ntfy-sh = {
enable = true;
group = "caddy";
settings = {
listen-unix = "/var/run/ntfy-sh/ntfy.sock";
listen-unix-mode = 432; # octal 0660
base-url = "https://ntfy.xinyang.life";
};
};
systemd.services.ntfy-sh.serviceConfig.RuntimeDirectory = "ntfy-sh";
services.kanidm = {
package = pkgs.kanidm.withSecretProvisioning;
enableServer = true;
@ -161,6 +174,11 @@ in
};
users.groups.git = { };
users.users = {
${config.services.caddy.user}.extraGroups = [
config.services.ntfy-sh.group
];
};
services.caddy = {
enable = true;
@ -191,5 +209,14 @@ in
}
}
'';
virtualHosts."https://ntfy.xinyang.life".extraConfig = ''
reverse_proxy unix/${config.services.ntfy-sh.settings.listen-unix}
@httpget {
protocol http
method GET
path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/)
}
redir @httpget https://{host}{uri}
'';
};
}

View file

@ -5,7 +5,7 @@
./common-settings/nix-conf.nix
./restic.nix
./vaultwarden.nix
./prometheus.nix
./prometheus
./hedgedoc.nix
./sing-box.nix
./kanidm-client.nix

View file

@ -1,245 +0,0 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.custom.prometheus;
in
{
options = {
custom.prometheus = {
enable = mkEnableOption "Prometheus instance";
exporters = {
enable = mkOption {
type = types.bool;
default = false;
description = "Enable Prometheus exporter on every supported services";
};
};
grafana = {
enable = mkEnableOption "Grafana Cloud";
password_file = mkOption {
type = types.path;
};
};
};
};
config = mkIf cfg.enable (mkMerge [
{
services.tailscale = {
enable = true;
permitCertUid = config.services.caddy.user;
};
services.caddy = {
enable = true;
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
'';
};
services.caddy.globalConfig = ''
servers {
metrics
}
'';
services.restic.server.prometheus = cfg.enable;
services.gotosocial.settings = mkIf cfg.enable {
metrics-enabled = true;
};
services.ntfy-sh.settings.enable-metrics = true;
services.prometheus = mkIf cfg.enable
{
enable = true;
port = 9091;
globalConfig.external_labels = { hostname = config.networking.hostName; };
remoteWrite = mkIf cfg.grafana.enable [
{
name = "grafana";
url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
basic_auth = {
username = "1340065";
password_file = cfg.grafana.password_file;
};
}
];
exporters = {
node = {
enable = true;
enabledCollectors = [
"conntrack"
"diskstats"
"entropy"
"filefd"
"filesystem"
"loadavg"
"meminfo"
"netdev"
"netstat"
"stat"
"time"
"vmstat"
"systemd"
"logind"
"interrupts"
"ksmd"
];
port = 9100;
};
};
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [
{ targets = [ "localhost:${toString config.services.prometheus.port}" ]; }
];
}
{
job_name = "node";
static_configs = [
{ targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
];
}
];
alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
logLevel = "debug";
configuration = {
route = {
receiver = "ntfy";
};
receivers = [
{
name = "ntfy";
webhook_configs = [
{
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
Alert {{.status}}
{{range .alerts}}-----{{range $k,$v := .labels}}
{{$k}}={{$v}}{{end}}
{{end}}
''}";
send_resolved = true;
}
];
}
];
};
};
alertmanagers = [
{
scheme = "http";
static_configs = [
{
targets = [
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
];
}
];
}
];
rules = let mkRule = condition: { ... }@rule: (if condition then [ rule ] else [ ]); in [
(lib.generators.toYAML { } {
groups = (mkRule true
{
name = "system_alerts";
rules = [
{
alert = "SystemdFailedUnits";
expr = "node_systemd_unit_state{state=\"failed\"} > 0";
for = "5m";
labels = { severity = "critical"; };
annotations = { summary = "Systemd has failed units on {{ $labels.instance }}"; description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"; };
}
{
alert = "HighLoadAverage";
expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
for = "1m";
labels = { severity = "warning"; };
annotations = { summary = "High load average detected on {{ $labels.instance }}"; description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."; };
}
{
alert = "HighTransmitTraffic";
expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
for = "1m";
labels = { severity = "warning"; };
annotations = { summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"; description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."; };
}
];
}) ++ (mkRule config.services.restic.server.enable {
name = "restic_alerts";
rules = [
{
alert = "ResticCheckFailed";
expr = "restic_check_success == 0";
for = "5m";
labels = { severity = "critical"; };
annotations = { summary = "Restic check failed (instance {{ $labels.instance }})"; description = "Restic check failed\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; };
}
{
alert = "ResticOutdatedBackup";
expr = "time() - restic_backup_timestamp > 518400";
for = "0m";
labels = { severity = "critical"; };
annotations = { summary = "Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated"; description = "Restic backup is outdated\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; };
}
];
}) ++ (mkRule config.services.caddy.enable {
name = "caddy_alerts";
rules = [
{
alert = "UpstreamHealthy";
expr = "caddy_reverse_proxy_upstreams_healthy == 0";
for = "5m";
labels = { severity = "critical"; };
annotations = { summary = "Upstream {{ $labels.unstream }} not healthy"; };
}
{
alert = "HighRequestLatency";
expr = "histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 0.5";
for = "2m";
labels = { severity = "warning"; };
annotations = { summary = "High request latency on {{ $labels.instance }}"; description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; };
}
];
});
})
];
};
}
{
services.prometheus.scrapeConfigs = [
(mkIf config.services.caddy.enable {
job_name = "caddy";
static_configs = [
{ targets = [ "localhost:2019" ]; }
];
})
(mkIf config.services.restic.server.enable {
job_name = "restic";
static_configs = [
{ targets = [ config.services.restic.server.listenAddress ]; }
];
})
(mkIf config.services.gotosocial.enable {
job_name = "gotosocial";
static_configs = [
{ targets = [ "localhost:${toString config.services.gotosocial.settings.port}" ]; }
];
})
(mkIf config.services.ntfy-sh.enable {
job_name = "ntfy-sh";
static_configs = [
{ targets = [ "auth.xinyang.life" ]; }
];
})
];
}
]);
}

View file

@ -0,0 +1,45 @@
{ config, lib, ... }:
let
cfg = config.custom.prometheus;
in
{
config = lib.mkIf cfg.enable {
services.caddy.globalConfig = lib.mkIf cfg.exporters.caddy.enable ''
servers {
metrics
}
'';
services.prometheus.scrapeConfigs = [
(lib.mkIf cfg.exporters.caddy.enable {
job_name = "caddy";
static_configs = [
{ targets = [ "127.0.0.1:2019" ]; }
];
})
];
custom.prometheus.ruleModules = [
(lib.mkIf cfg.exporters.caddy.enable {
name = "caddy_alerts";
rules = [
{
alert = "UpstreamHealthy";
expr = "caddy_reverse_proxy_upstreams_healthy != 1";
for = "5m";
labels = { severity = "critical"; };
annotations = { summary = "Upstream {{ $labels.unstream }} not healthy"; };
}
{
alert = "HighRequestLatency";
expr = "histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 5";
for = "2m";
labels = { severity = "warning"; };
annotations = { summary = "High request latency on {{ $labels.instance }}"; description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; };
}
];
})
];
};
}

View file

@ -0,0 +1,194 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.custom.prometheus;
exporterCfg = config.custom.prometheus.exporters;
mkExporterOption = enableOption: (mkOption {
type = types.bool;
default = enableOption;
description = "Enable this exporter";
});
mkRulesOption = mkOption {
type = types.listOf (types.submodule {
options = {
name = mkOption {
type = lib.types.str;
};
rules = mkOption {
type = lib.types.listOf lib.types.attrs;
};
};
});
};
in
{
imports = [
./caddy.nix
./gotosocial.nix
./ntfy-sh.nix
./restic.nix
];
options = {
custom.prometheus = {
enable = mkEnableOption "Prometheus instance";
exporters = {
enable = mkOption {
type = types.bool;
default = false;
description = "Enable Prometheus exporter on every supported services";
};
restic.enable = mkExporterOption config.services.restic.server.enable;
caddy.enable = mkExporterOption config.services.caddy.enable;
gotosocial.enable = mkExporterOption config.services.gotosocial.enable;
ntfy-sh.enable = mkExporterOption config.services.gotosocial.enable;
};
grafana = {
enable = mkEnableOption "Grafana Cloud";
password_file = mkOption {
type = types.path;
};
};
ruleModules = mkRulesOption;
};
};
config = mkIf cfg.enable
{
services.tailscale = {
enable = true;
permitCertUid = config.services.caddy.user;
openFirewall = true;
};
services.caddy = {
enable = true;
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
'';
};
services.prometheus = mkIf cfg.enable
{
enable = true;
port = 9091;
globalConfig.external_labels = { hostname = config.networking.hostName; };
remoteWrite = mkIf cfg.grafana.enable [
{
name = "grafana";
url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
basic_auth = {
username = "1340065";
password_file = cfg.grafana.password_file;
};
}
];
exporters = {
node = {
enable = true;
enabledCollectors = [
"diskstats"
"loadavg"
"time"
"systemd"
];
port = 9100;
};
};
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [
{ targets = [ "localhost:${toString config.services.prometheus.port}" ]; }
];
}
{
job_name = "node";
static_configs = [
{ targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
];
}
];
alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
logLevel = "debug";
configuration = {
route = {
receiver = "ntfy";
};
receivers = [
{
name = "ntfy";
webhook_configs = [
{
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
Alert {{.status}}
{{range .alerts}}-----{{range $k,$v := .labels}}
{{$k}}={{$v}}{{end}}
{{end}}
''}";
send_resolved = true;
}
];
}
];
};
};
alertmanagers = [
{
scheme = "http";
static_configs = [
{
targets = [
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
];
}
];
}
];
rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
};
custom.prometheus.ruleModules = [
{
name = "system_alerts";
rules = [
{
alert = "SystemdFailedUnits";
expr = "node_systemd_unit_state{state=\"failed\"} > 0";
for = "5m";
labels = { severity = "critical"; };
annotations = { summary = "Systemd has failed units on {{ $labels.instance }}"; description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"; };
}
{
alert = "HighLoadAverage";
expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
for = "1m";
labels = { severity = "warning"; };
annotations = { summary = "High load average detected on {{ $labels.instance }}"; description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."; };
}
{
alert = "HighTransmitTraffic";
expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
for = "1m";
labels = { severity = "warning"; };
annotations = { summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"; description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."; };
}
{
alert = "NetworkTrafficExceedLimit";
expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
for = "0m";
labels = { severity = "critical"; };
annotations = { summary = "Outbound network traffic exceed 300GB for last 30 day"; };
}
];
}
];
};
}

View file

@ -0,0 +1,19 @@
{ config, lib, ... }:
let
cfg = config.custom.prometheus;
in
{
config = lib.mkIf cfg.exporters.gotosocial.enable {
services.gotosocial.settings = lib.mkIf cfg.exporters.gotosocial.enable {
metrics-enabled = true;
};
services.prometheus.scrapeConfigs = [
{
job_name = "gotosocial";
static_configs = [
{ targets = [ "localhost:8080" ]; }
];
}
];
};
}

View file

@ -0,0 +1,17 @@
{ config, lib, ... }:
let
cfg = config.custom.prometheus;
in
{
config = lib.mkIf cfg.enable {
services.ntfy-sh.settings.enable-metrics = true;
services.prometheus.scrapeConfigs = [
(lib.mkIf cfg.exporters.ntfy-sh.enable {
job_name = "ntfy-sh";
static_configs = [
{ targets = [ "ntfy.xinyang.life" ]; }
];
})
];
};
}

View file

@ -0,0 +1,41 @@
{ config, lib, ... }:
let
cfg = config.custom.prometheus;
in
{
config = lib.mkIf cfg.enable {
services.restic.server.prometheus = true;
services.prometheus.scrapeConfigs = [
(lib.mkIf cfg.exporters.restic.enable {
job_name = "restic";
static_configs = [
{ targets = [ config.services.restic.server.listenAddress ]; }
];
})
];
custom.prometheus.ruleModules = [
(lib.mkIf cfg.exporters.restic.enable {
name = "restic_alerts";
rules = [
{
alert = "ResticCheckFailed";
expr = "restic_check_success == 0";
for = "5m";
labels = { severity = "critical"; };
annotations = { summary = "Restic check failed (instance {{ $labels.instance }})"; description = "Restic check failed\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; };
}
{
alert = "ResticOutdatedBackup";
expr = "time() - restic_backup_timestamp > 518400";
for = "0m";
labels = { severity = "critical"; };
annotations = { summary = "Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated"; description = "Restic backup is outdated\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; };
}
];
})
];
};
}