diff --git a/machines/calcite/configuration.nix b/machines/calcite/configuration.nix index 4b35351..d42c585 100644 --- a/machines/calcite/configuration.nix +++ b/machines/calcite/configuration.nix @@ -208,7 +208,6 @@ element-desktop tdesktop qq - feishu # Password manager bitwarden @@ -265,6 +264,11 @@ custom.forgejo-actions-runner.enable = true; custom.forgejo-actions-runner.tokenFile = config.sops.secrets.gitea_env.path; + custom.prometheus = { + enable = true; + exporters.enable = true; + }; + # MTP support services.gvfs.enable = true; diff --git a/machines/dolomite/default.nix b/machines/dolomite/default.nix index 3965655..5bf2979 100644 --- a/machines/dolomite/default.nix +++ b/machines/dolomite/default.nix @@ -14,6 +14,12 @@ in config = { isBandwagon = builtins.elem config.networking.hostName bwgHosts; isLightsail = builtins.elem config.networking.hostName awsHosts; + + commonSettings = { + auth.enable = true; + nix.enable = true; + }; + sops = { secrets = { wg_private_key = { @@ -43,6 +49,8 @@ in networking.firewall.allowedTCPPorts = [ 80 8080 ]; networking.firewall.allowedUDPPorts = [ ] ++ (lib.range 6311 6314); + services.tailscale.enable = true; + custom.prometheus = { enable = false; exporters.enable = false; @@ -52,33 +60,6 @@ in }; }; - custom.kanidm-client = { - enable = true; - uri = "https://auth.xinyang.life/"; - asSSHAuth = { - enable = true; - allowedGroups = [ "linux_users" ]; - }; - sudoers = [ "xin@auth.xinyang.life" ]; - }; - - services.openssh = { - settings = { - PasswordAuthentication = false; - KbdInteractiveAuthentication = false; - PermitRootLogin = lib.mkForce "no"; - GSSAPIAuthentication = "no"; - KerberosAuthentication = "no"; - }; - }; - services.fail2ban.enable = true; - programs.mosh.enable = true; - - security.sudo = { - execWheelOnly = true; - wheelNeedsPassword = false; - }; - services.sing-box = let singTls = { enabled = true; diff --git a/machines/massicot/default.nix b/machines/massicot/default.nix index 56cbfe5..5ac8151 100644 --- a/machines/massicot/default.nix +++ b/machines/massicot/default.nix @@ -7,6 +7,11 @@ ./networking.nix ./services.nix ]; + + commonSettings = { + auth.enable = true; + nix.enable = true; + }; sops = { defaultSopsFile = ./secrets.yaml; @@ -52,34 +57,6 @@ networking = { hostName = "massicot"; }; - - custom.kanidm-client = { - enable = true; - uri = "https://auth.xinyang.life/"; - asSSHAuth = { - enable = true; - allowedGroups = [ "linux_users" ]; - }; - sudoers = [ "xin@auth.xinyang.life" ]; - }; - - security.sudo = { - execWheelOnly = true; - wheelNeedsPassword = false; - }; - - services.openssh = { - enable = true; - settings = { - PasswordAuthentication = false; - KbdInteractiveAuthentication = false; - PermitRootLogin = "no"; - GSSAPIAuthentication = "no"; - KerberosAuthentication = "no"; - }; - }; - services.fail2ban.enable = true; - programs.mosh.enable = true; systemd.services.sshd.wantedBy = pkgs.lib.mkForce [ "multi-user.target" ]; } diff --git a/modules/nixos/prometheus.nix b/modules/nixos/prometheus.nix index 40035f3..9d27980 100644 --- a/modules/nixos/prometheus.nix +++ b/modules/nixos/prometheus.nix @@ -26,6 +26,18 @@ in }; config = mkIf cfg.enable (mkMerge [{ + services.tailscale = { + enable = true; + permitCertUid = config.services.caddy.user; + }; + + services.caddy = { + enable = true; + virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = '' + reverse_proxy 127.0.0.1:${toString config.services.prometheus.port} + ''; + }; + services.caddy.globalConfig = '' servers { metrics @@ -103,7 +115,7 @@ in name = "ntfy"; webhook_configs = [ { - url = "${config.services.ntfy-sh.settings.base-url}/prometheus-alerts"; + url = "https://ntfy.xinyang.life/prometheus-alerts"; send_resolved = true; } ]; @@ -162,7 +174,55 @@ in '' groups: - name: restic_alerts + rules: + - alert: ResticCheckFailed + expr: restic_check_success == 0 + for: 5m + labels: + severity: critical + annotations: + summary: Restic check failed (instance {{ $labels.instance }}) + description: Restic check failed\n VALUE = {{ $value }}\n LABELS = {{ $labels }} + + - alert: ResticOutdatedBackup + # 1209600 = 15 days + expr: time() - restic_backup_timestamp > 518400 + for: 0m + labels: + severity: critical + annotations: + summary: Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated + description: Restic backup is outdated\n VALUE = {{ $value }}\n LABELS = {{ $labels }} '' else "") + (if config.services.caddy.enable then '' + groups: + - name: caddy_alerts + rules: + - alert: HighHttpErrorRate + expr: rate(caddy_http_request_duration_seconds_count{status_code=~"5.."}[5m]) / rate(caddy_http_request_duration_seconds_count[5m]) > 0.01 + for: 10m + labels: + severity: critical + annotations: + summary: "High error rate on {{ $labels.instance }}" + description: "More than 1% of HTTP requests are errors over the last 10 minutes." + - alert: CaddyDown + expr: up{job="caddy"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Caddy server down on {{ $labels.instance }}" + description: "Caddy server is down for more than 5 minutes." + - alert: HighRequestLatency + expr: histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 0.5 + for: 2m + labels: + severity: warning + annotations: + summary: "High request latency on {{ $labels.instance }}" + description: "95th percentile of request latency is above 0.5 seconds for the last 2 minutes." + '' else "") ]; }; }