feat: enable tailscale on servers

machines/calcite/configuration.nix

@@ -208,7 +208,6 @@
     element-desktop
     tdesktop
     qq
-    feishu
 
     # Password manager
     bitwarden
@@ -265,6 +264,11 @@
   custom.forgejo-actions-runner.enable = true;
   custom.forgejo-actions-runner.tokenFile = config.sops.secrets.gitea_env.path;
 
+  custom.prometheus = {
+    enable = true;
+    exporters.enable = true;
+  };
+
   # MTP support
   services.gvfs.enable = true;
 

machines/dolomite/default.nix

@@ -14,6 +14,12 @@ in
   config = {
     isBandwagon = builtins.elem config.networking.hostName bwgHosts;
     isLightsail = builtins.elem config.networking.hostName awsHosts;
+
+    commonSettings = {
+      auth.enable = true;
+      nix.enable = true;
+    };
+
     sops = {
       secrets = {
         wg_private_key = {
@@ -43,6 +49,8 @@ in
     networking.firewall.allowedTCPPorts = [ 80 8080 ];
     networking.firewall.allowedUDPPorts = [ ] ++ (lib.range 6311 6314);
 
+    services.tailscale.enable = true;
+
     custom.prometheus = {
       enable = false;
       exporters.enable = false;
@@ -52,33 +60,6 @@ in
       };
     };
 
-    custom.kanidm-client = {
-      enable = true;
-      uri = "https://auth.xinyang.life/";
-      asSSHAuth = {
-        enable = true;
-        allowedGroups = [ "linux_users" ];
-      };
-      sudoers = [ "xin@auth.xinyang.life" ];
-    };
-
-    services.openssh = {
-      settings = {
-        PasswordAuthentication = false;
-        KbdInteractiveAuthentication = false;
-        PermitRootLogin = lib.mkForce "no";
-        GSSAPIAuthentication = "no";
-        KerberosAuthentication = "no";
-      };
-    };
-    services.fail2ban.enable = true;
-    programs.mosh.enable = true;
-
-    security.sudo = {
-      execWheelOnly = true;
-      wheelNeedsPassword = false;
-    };
-
     services.sing-box = let
       singTls = {
         enabled = true;

machines/massicot/default.nix

@@ -8,6 +8,11 @@
     ./services.nix
   ];
 
+  commonSettings = {
+    auth.enable = true;
+    nix.enable = true;
+  };
+  
   sops = {
     defaultSopsFile = ./secrets.yaml;
     age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
@@ -53,33 +58,5 @@
     hostName = "massicot";
   };
   
-  custom.kanidm-client = {
-    enable = true;
-    uri = "https://auth.xinyang.life/";
-    asSSHAuth = {
-      enable = true;
-      allowedGroups = [ "linux_users" ];
-    };
-    sudoers = [ "xin@auth.xinyang.life" ];
-  };
-
-  security.sudo = {
-      execWheelOnly = true;
-      wheelNeedsPassword = false;
-    };
-
-  services.openssh = {
-    enable = true;
-    settings = {
-      PasswordAuthentication = false;
-      KbdInteractiveAuthentication = false;
-      PermitRootLogin = "no";
-      GSSAPIAuthentication = "no";
-      KerberosAuthentication = "no";
-    };
-  };
-  services.fail2ban.enable = true;
-  programs.mosh.enable = true;
-  
   systemd.services.sshd.wantedBy = pkgs.lib.mkForce [ "multi-user.target" ];
 }

modules/nixos/prometheus.nix

@@ -26,6 +26,18 @@ in
   };
 
   config = mkIf cfg.enable (mkMerge [{
+    services.tailscale = {
+      enable = true;
+      permitCertUid = config.services.caddy.user;
+    };
+
+    services.caddy = {
+      enable = true;
+      virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
+        reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
+      '';
+    };
+
     services.caddy.globalConfig = ''
       servers {
         metrics
@@ -103,7 +115,7 @@ in
               name = "ntfy";
               webhook_configs = [
                 {
-                  url = "${config.services.ntfy-sh.settings.base-url}/prometheus-alerts";
+                  url = "https://ntfy.xinyang.life/prometheus-alerts";
                   send_resolved = true;
                 }
               ];
@@ -162,6 +174,54 @@ in
           ''
             groups:
               - name: restic_alerts
+                rules:
+                  - alert: ResticCheckFailed
+                    expr: restic_check_success == 0
+                    for: 5m
+                    labels:
+                      severity: critical
+                    annotations:
+                      summary: Restic check failed (instance {{ $labels.instance }})
+                      description: Restic check failed\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}
+
+                  - alert: ResticOutdatedBackup
+                    # 1209600 = 15 days
+                    expr: time() - restic_backup_timestamp > 518400
+                    for: 0m
+                    labels:
+                      severity: critical
+                    annotations:
+                      summary: Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated
+                      description: Restic backup is outdated\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}
+          '' else "")
+        (if config.services.caddy.enable then ''
+            groups:
+              - name: caddy_alerts
+                rules:
+                  - alert: HighHttpErrorRate
+                    expr: rate(caddy_http_request_duration_seconds_count{status_code=~"5.."}[5m]) / rate(caddy_http_request_duration_seconds_count[5m]) > 0.01
+                    for: 10m
+                    labels:
+                      severity: critical
+                    annotations:
+                      summary: "High error rate on {{ $labels.instance }}"
+                      description: "More than 1% of HTTP requests are errors over the last 10 minutes."
+                  - alert: CaddyDown
+                    expr: up{job="caddy"} == 0
+                    for: 5m
+                    labels:
+                      severity: critical
+                    annotations:
+                      summary: "Caddy server down on {{ $labels.instance }}"
+                      description: "Caddy server is down for more than 5 minutes."
+                  - alert: HighRequestLatency
+                    expr: histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 0.5
+                    for: 2m
+                    labels:
+                      severity: warning
+                    annotations:
+                      summary: "High request latency on {{ $labels.instance }}"
+                      description: "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."
         '' else "")
       ];
     };