all: add prometheus

.sops.yaml

@@ -39,6 +39,7 @@ creation_rules:
       - *host-raspite
       - *host-sgp-00
       - *host-tok-00
+      - *host-massicot
   - path_regex: home/xin/secrets.yaml
     key_groups:
     - age:

flake.nix

@@ -104,6 +104,7 @@
 
           sgp-00 = { name, nodes, pkgs, ... }: with inputs; {
               imports = [
+                self.nixosModules.default
                 machines/dolomite
               ];
               nixpkgs.system = "x86_64-linux";
@@ -118,6 +119,7 @@
 
           tok-00 = { name, nodes, pkgs, ... }: with inputs; {
               imports = [
+                self.nixosModules.default
                 machines/dolomite
               ];
               nixpkgs.system = "x86_64-linux";

machines/dolomite/default.nix

@@ -37,6 +37,15 @@
     networking.firewall.allowedTCPPorts = [ 80 8080 ];
     networking.firewall.allowedUDPPorts = [ ] ++ (lib.range 6311 6314);
 
+    custom.prometheus = {
+      enable = true;
+      exporters.enable = true;
+      grafana = {
+        enable = true;
+        password_file = config.sops.secrets.grafana_cloud_api.path;
+      };
+    };
+
     services.sing-box = let
       singTls = {
         enabled = true;

machines/massicot/default.nix

@@ -18,6 +18,10 @@
       gts_env = {
         owner = "gotosocial";
       };
+      grafana_cloud_api = {
+        owner = "prometheus";
+        sopsFile = ../secrets.yaml;
+      };
     };
   };
 

machines/massicot/services.nix

@@ -11,6 +11,15 @@ in
     domain = "vaultwarden.xinyang.life";
   };
 
+  custom.prometheus = {
+    enable = true;
+    exporters.enable = true;
+    grafana = {
+      enable = true;
+      password_file = config.sops.secrets.grafana_cloud_api.path;
+    };
+  };
+
   fileSystems = builtins.listToAttrs (map (share: {
     name = "/mnt/storage/${share}";
     value = { 

machines/secrets.yaml

@@ -6,6 +6,7 @@ singbox_sg_server: ENC[AES256_GCM,data:5rogqKm5yiy5Yvz4Vo1a6Q==,iv:Vx9wNTdVHkReu
 singbox_jp_server: ENC[AES256_GCM,data:xKTcxkcu1WIsT/wlMpEoqGJK,iv:nXetY339YuOi2jFEb3xkPTglHRMk/quIrQL4ko+8MxY=,tag:+Nwsx65/gdrDhL1ZurR5Ng==,type:str]
 singbox_password: ENC[AES256_GCM,data:0tBIzwtNSQqbGlD+CDnQfJigbFVBChEL,iv:W2HaHeSkvmS6jHSnfOJ6tD2QXuUq1A+mfZf7sEXB++E=,tag:5BtYAv1NO70IL4m/uG8QKA==,type:str]
 singbox_uuid: ENC[AES256_GCM,data:ufN+vDl/rDASoQL23tHwlr3ybMyrlC/Kd7bT0c5+SP+bc6Zj,iv:+uwt/N9LpFaJK6MjoczyrZ039MDZn4kRmtEoq4OvdFU=,tag:6Yma9+yrISwQoSRDgUbuwA==,type:str]
+grafana_cloud_api: ENC[AES256_GCM,data:Pz+tE09dcJa+ZEWS3vtpOtitGCA9Cg/+gOd/0FsF8ooxzPyN9/UMuTcP02aIPW5v7yZCkGJOAXufIyechNf0crgAV/KmwGGwixH7I+1f3sDtGiFZEMnQgrysyfJo0KIrIZ8XP0SyXDs3vKjDU8cUI4+IyucHacWQ1kWdEtINjcPNHRPS2yaMUIvsRn0z8Cs2byMD3ghUHHHOz40CuO6r4A==,iv:cHvbeCmLFmJPNKsl1BBYx9WJP7ZJWi+8c9yHZWc6FTs=,tag:yWXtPokYE4frCmzzzyEqEg==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -15,50 +16,59 @@ sops:
         - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBweTlPTGVRbUlndTdES0s2
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0MUxIZHJTYk9YS0lPOGZK
-            SVM2N2FUMnozQk11cDk0cTFEb1l6YldkVHc4CmhnNzJyY1VKRWhpc0tTbFNKeDBD
+            VUJhQ1liNEtXZ3ZYaCtqQWVBTGVJclVVRER3CmJUcS9yY2x1TFFYMkpZOWxZeW5w
-            a0hzMi93Ly9zY2Fjd1RCdjV6WnVmOU0KLS0tIFh6NVFteWxxNithMGM0dnJiNE9X
+            WFk0WTNoWmphdG12dTdHaW9tYVRjS1UKLS0tIHd4enVwalRDaHQwK0U1RFNHOEVI
-            dGovQ2ZMZWx1djVkb0Y4ZVNLRDJPRncKz0N/zP3mN97BpLaDgE9hx/zooGyHAnvC
+            N0UrRjRxTWJRanI4VnRjWlhzQS8zSGsKSJJnFuEp7yO8bIh2LpSvgjsYAK05u2TE
-            D8iH/1PZ21uMYeUQq83B8mDKbv+qAltA/vD+ZNnb4ULjYLmVn5p/hQ==
+            a+UBiu6xQQaUnL02CAau4xHqBn9GZxeqlVAjVSJITArLR/uQkkUM6g==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKUkxVTUtYZ0RWUFVxY0Rl
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZT3ZES3BHWWpDekt0VEYz
-            UFFadVlzUFJVMGpzRVd5bHVDQmQycVlNSkcwCkMvcUJMRFVWTzNHZ3pxemRLelJP
+            emUvUTQ3WUFWd0w2VlVSWHMrd3ZvZjYvYlJZCkcyRjBZWEdGTXJZVENyZ1U2YTV2
-            K3pQMFdURmpRUVRuL1lzT09FVVdBd3MKLS0tIE9LY0NHSW1UWUJpbWdNQW1CVUlD
+            eU1MS3NCQzZ3Y3ZhOG4rRVByU1ZlRU0KLS0tIFdGVTliOFpSTWl0YlV6OTVUbk9O
-            b1FmZnVjOFFCMDVXdFBtZzZWdkt6RVUKvLoHmEhkyeKHlstRoT3duTIQTojxzcFI
+            SjBoUnNOVTB1QWFDYnVwWkhaN3d0VGMKjNiW597mLAogPyDBUhEDYd/VyePXesL7
-            NapIBB3/6Qqho+kYc8/hLWb61EsSX9yqO9C6f6FpFrwi0696OvP3mA==
+            kzyV/e8t/5zHs3/I17ZUd8bxdCjbrrXI1g4Swx31yCgZOk8uKAuLRQ==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4ZFNoMmNXV2F1U2E2bUhv
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQaTlNTjVXTHFzNS9GUk1S
-            K3lGTCs2KzZYbXVlWEdVelNDTS80SW85c0J3CkszNGkrbFVKWks4dmwyYlpQMkpW
+            bVMxeWdwSUlmN3B6QlovejI3SlNuc2dJMjFVClF2VFRVNjFrQldRcHNLeWhpWFE1
-            Zm02cG41ZlpwcEdCbzFkSHpjWHpCdG8KLS0tIHlrNXp6TTI5ZnhGTUNMWTZ0ekVS
+            UDRvY3RTZHZCa2RDZ1RmVWRHb2ttUVUKLS0tIEI0QS9SL3lTeXVITVgvcHVCNmdW
-            VExPWk1zeVExYXdaL2o1WVB5NlhsNFkK3vsnc4qE08W13ttzt+YCHbQh2c/mOxFZ
+            cVl6T3NWWEVkWExuTldqQU5CUzFTM1UKFYD1jdEQfFRNBkRyL+1gZzCdpJHN7QqU
-            DneXTgOjkyBaY5JDFKlzlIN3m8QRBG5vPOuSKXaoFmY8E68RzNey3w==
+            4CVOsIeVl6ufWG4D2FfP4Zow5uhnvDXmWqBCmpJ/iVKnu3klihlndA==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvdHA5WHA2V2RNTTZXNVVT
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxRGZ5WVFJQzFSWlR6dDMv
-            Wks2a2tqT045ZkJFYTN2RHhmdkZxMjlPRDNFCm1HaHhLNkp6NWZxNUYvOTRybE1Z
+            bXJsNlZLeVVpK1RuaVpySkcreHE1SkNMSjA4CkxGMzVvZHZ4ZTdRdzh6K3V6OVQ0
-            Y1l5eDFkcXRWSko3ODhqV2htb3pzcDQKLS0tIGI3YlI4dCtMbGl1aHFZdDBic0Jv
+            RkI3bWg5ZUw5RFlQN05zdC9HVkdjYlUKLS0tIGdibTdwbnRhMmZEZ2VPelF6a3Aw
-            LzV3NWhFQTlaZ1Y3R0paaEZPZDNpZzgK3/ZE3+F+mq574MfiF7PRlKmAU6mUTiGF
+            U1dGQmxOTklFTmFaMTc1MGQvRVB1TzgKkhxjImoj1lxpvBMjKJJOiM2eC2bQ73Ay
-            Ffqh0kQumHH7nBuunD0L7Zp2j15hMjUs/oxX558jY9BNl+rN2VWO0Q==
+            Rket8CjZnfRhYDD9YoOWBNswONQoVY8/dSXgLDObtfFxbnjZ1pj63A==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBueFhiQzdMaU1zR2VtOEtO
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3RWRsdXNTQkNJWXFTODY4
-            WFVtdVJLU3B3TzRSSENodUpuUm03TnBHQnhBCmRrdjJScEVsS0JTQmthZWIzVFlv
+            WVNYb2xKZHJWWTUvZmlMS3VkYnhWQkVaZHpFCjJjY2JzeFQza3llNHZFYWVVK0Ri
-            TVY3TUo0VllPWElua21mczZvT3YxYjAKLS0tIFpDcE0wSXdSRXFGY2tLd1orVE9L
+            K2ZJNUlZMWxFbGdhQ2pxRlh4VjVITFkKLS0tIGFHSDI5aW5aTUdFTEJOMnNjVXlm
-            Y2MyZUhOaEVVZU9Hc0xHbWtMdG1Ca2cKHU7pgODnNVDiMFF6be07a320a9HWKIdO
+            SVlDVk9Xdnc0WVpFN2VmSlZIajJielkKz8xnfxIArN9PLjUorYPzakmLx7/bsoq0
-            OKFA9R6WX1TFhKBKNDqK/mokJBTxu4nR16ewHSWOU13O/M8aKCQhug==
+            EfoiB6ZpuWMeNEmfHygTEUPTC7eWw42EIYk964vI6LySFQyO3Z8p5g==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-12-19T12:56:28Z"
+        - recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
-    mac: ENC[AES256_GCM,data:v7Rn7dPOzfcgab2MhiU7h0CXjkAbkpBX7l7iLdnw3RUIjxulTXVuPpgenojF5yVqFCPgm2LKBKniD+cvtMvVhb00a1tnDNM/tfjH9GjBYNZH9xtPWJED7GLASd6nIF5BZhANKhH8yphAi5VJ/4cyEdMFbWu+2gO8GyQxJQYhgY8=,iv:bbbZ8vF+Vbwq/6PXN/7qvRO62M/eDZ591v4gXc1fs+g=,tag:dyt9LVU32hnbVT12C/Afqw==,type:str]
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2WFIzVEZPUmFBclpweDZR
+            WXZFb0FjcWxDRTNpQmFRaU9BY0lPTzAxNWhvClk5UmxFQllGQ29VOGIxeS9xMmV2
+            SUdEaFJ3bFZPSjVjQ1JnVS9jSWxXaWcKLS0tIGs0ZE0wMUZDeGNWNlhoN3JOMmlG
+            c1E1Sld1ejZhTStKTU5teEJKT2JwVXcKuEQnA6b1WJ+RNqmrZ8t3joiEZ57Oq9M1
+            P4tMGerB12A1myTJlt5Ss2OCTBUV7ooVRNsyPjyvJy/YTyjqZ5xmxg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2023-12-20T01:34:00Z"
+    mac: ENC[AES256_GCM,data:6MLBRPA5g2r3yy/i7DSxjWaYhHH/4GiAqL/pRIvYyIrKQWYvfviWlTX9dqHVzzCXjueEXUM5dXFb2B+Sds68EGgBuBlZvBchtstHUOtMLE3pttC+xCzerQFyrPDrXbnpfdDYPHWxvhhhFpWu8G5RSfzSgkgp7+cx9iZHq/g1k/Q=,iv:8yFIOgHtBiCtbamufrXXHrjIq5DV3MIJbTJPtXlgpPg=,tag:CVOIojTN2KkXJsDVyiZjMQ==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1

machines/sops.nix

@@ -24,6 +24,9 @@
       singbox_uuid = {
          owner = "root";
       };
+      grafana_cloud_api = {
+         owner = "prometheus";
+      };
     };
   };
 }

modules/nixos/prometheus.nix

@@ -1,4 +1,3 @@
-
 { config, pkgs, lib, ... }:
 
 with lib;
@@ -17,13 +16,38 @@ in
           description = "Enable Prometheus exporter on every supported services";
         };
       };
+      grafana = {
+        enable = mkEnableOption "Grafana Cloud";
+        password_file = mkOption {
+          type = types.path;
+        };
+      };
     };
   };
 
-  config = {
+  config = mkMerge [{
+    services.caddy.globalConfig = ''
+      servers {
+        metrics
+      }
+    '';
+    services.restic.server.prometheus = cfg.enable;
+    services.gotosocial.settings = {
+      metrics-enable = true;
+    };
     services.prometheus = mkIf cfg.enable {
       enable = true;
       port = 9091;
+      globalConfig.external_labels = { hostname = config.networking.hostName; };
+      remoteWrite = mkIf cfg.grafana.enable [
+        { name = "grafana";
+          url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
+          basic_auth = {
+            username = "1340065";
+            password_file = cfg.grafana.password_file;
+          };
+        }
+      ];
       exporters = {
         node = {
           enable = true;
@@ -44,5 +68,28 @@ in
         }
       ];
     };
-  };
+  }
+  {
+    services.prometheus.scrapeConfigs = [
+      ( mkIf config.services.caddy.enable {
+        job_name = "caddy";
+        static_configs = [
+          { targets = [ "localhost:2019" ]; }
+        ];
+      })
+      ( mkIf config.services.restic.server.enable {
+        job_name = "restic";
+        static_configs = [
+          { targets = [ config.services.restic.server.listenAddress ]; }
+        ];
+      })
+      ( mkIf config.services.gotosocial.enable {
+        job_name = "gotosocial";
+        static_configs = [
+          { targets = [ "localhost:${toString config.services.gotosocial.settings.port}" ]; }
+        ];
+      })
+    ];
+  }
+  ];
 }