all: add prometheus

2023-12-20 11:13:20 +08:00 · 2023-12-20 11:13:20 +08:00 · b9eebc2a7e
commit b9eebc2a7e
parent fcdc65d8ce
8 changed files with 116 additions and 31 deletions
--- a/.sops.yaml
+++ b/.sops.yaml
@ -39,6 +39,7 @@ creation_rules:
      - *host-raspite
      - *host-sgp-00
      - *host-tok-00
+      - *host-massicot
  - path_regex: home/xin/secrets.yaml
    key_groups:
    - age:
--- a/flake.nix
+++ b/flake.nix
@ -104,6 +104,7 @@

          sgp-00 = { name, nodes, pkgs, ... }: with inputs; {
              imports = [
+                self.nixosModules.default
                machines/dolomite
              ];
              nixpkgs.system = "x86_64-linux";
@ -118,6 +119,7 @@

          tok-00 = { name, nodes, pkgs, ... }: with inputs; {
              imports = [
+                self.nixosModules.default
                machines/dolomite
              ];
              nixpkgs.system = "x86_64-linux";
--- a/machines/dolomite/default.nix
+++ b/machines/dolomite/default.nix
@ -37,6 +37,15 @@
    networking.firewall.allowedTCPPorts = [ 80 8080 ];
    networking.firewall.allowedUDPPorts = [ ] ++ (lib.range 6311 6314);

+    custom.prometheus = {
+      enable = true;
+      exporters.enable = true;
+      grafana = {
+        enable = true;
+        password_file = config.sops.secrets.grafana_cloud_api.path;
+      };
+    };
+
    services.sing-box = let
      singTls = {
        enabled = true;
--- a/machines/massicot/default.nix
+++ b/machines/massicot/default.nix
@ -18,6 +18,10 @@
      gts_env = {
        owner = "gotosocial";
      };
+      grafana_cloud_api = {
+        owner = "prometheus";
+        sopsFile = ../secrets.yaml;
+      };
    };
  };

--- a/machines/massicot/services.nix
+++ b/machines/massicot/services.nix
@ -11,6 +11,15 @@ in
    domain = "vaultwarden.xinyang.life";
  };

+  custom.prometheus = {
+    enable = true;
+    exporters.enable = true;
+    grafana = {
+      enable = true;
+      password_file = config.sops.secrets.grafana_cloud_api.path;
+    };
+  };
+
  fileSystems = builtins.listToAttrs (map (share: {
    name = "/mnt/storage/${share}";
    value = { 
--- a/machines/secrets.yaml
+++ b/machines/secrets.yaml
@ -6,6 +6,7 @@ singbox_sg_server: ENC[AES256_GCM,data:5rogqKm5yiy5Yvz4Vo1a6Q==,iv:Vx9wNTdVHkReu
 singbox_jp_server: ENC[AES256_GCM,data:xKTcxkcu1WIsT/wlMpEoqGJK,iv:nXetY339YuOi2jFEb3xkPTglHRMk/quIrQL4ko+8MxY=,tag:+Nwsx65/gdrDhL1ZurR5Ng==,type:str]
 singbox_password: ENC[AES256_GCM,data:0tBIzwtNSQqbGlD+CDnQfJigbFVBChEL,iv:W2HaHeSkvmS6jHSnfOJ6tD2QXuUq1A+mfZf7sEXB++E=,tag:5BtYAv1NO70IL4m/uG8QKA==,type:str]
 singbox_uuid: ENC[AES256_GCM,data:ufN+vDl/rDASoQL23tHwlr3ybMyrlC/Kd7bT0c5+SP+bc6Zj,iv:+uwt/N9LpFaJK6MjoczyrZ039MDZn4kRmtEoq4OvdFU=,tag:6Yma9+yrISwQoSRDgUbuwA==,type:str]
+grafana_cloud_api: ENC[AES256_GCM,data:Pz+tE09dcJa+ZEWS3vtpOtitGCA9Cg/+gOd/0FsF8ooxzPyN9/UMuTcP02aIPW5v7yZCkGJOAXufIyechNf0crgAV/KmwGGwixH7I+1f3sDtGiFZEMnQgrysyfJo0KIrIZ8XP0SyXDs3vKjDU8cUI4+IyucHacWQ1kWdEtINjcPNHRPS2yaMUIvsRn0z8Cs2byMD3ghUHHHOz40CuO6r4A==,iv:cHvbeCmLFmJPNKsl1BBYx9WJP7ZJWi+8c9yHZWc6FTs=,tag:yWXtPokYE4frCmzzzyEqEg==,type:str]
 sops:
    kms: []
    gcp_kms: []
@ -15,50 +16,59 @@ sops:
        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBweTlPTGVRbUlndTdES0s2
-            SVM2N2FUMnozQk11cDk0cTFEb1l6YldkVHc4CmhnNzJyY1VKRWhpc0tTbFNKeDBD
-            a0hzMi93Ly9zY2Fjd1RCdjV6WnVmOU0KLS0tIFh6NVFteWxxNithMGM0dnJiNE9X
-            dGovQ2ZMZWx1djVkb0Y4ZVNLRDJPRncKz0N/zP3mN97BpLaDgE9hx/zooGyHAnvC
-            D8iH/1PZ21uMYeUQq83B8mDKbv+qAltA/vD+ZNnb4ULjYLmVn5p/hQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0MUxIZHJTYk9YS0lPOGZK
+            VUJhQ1liNEtXZ3ZYaCtqQWVBTGVJclVVRER3CmJUcS9yY2x1TFFYMkpZOWxZeW5w
+            WFk0WTNoWmphdG12dTdHaW9tYVRjS1UKLS0tIHd4enVwalRDaHQwK0U1RFNHOEVI
+            N0UrRjRxTWJRanI4VnRjWlhzQS8zSGsKSJJnFuEp7yO8bIh2LpSvgjsYAK05u2TE
+            a+UBiu6xQQaUnL02CAau4xHqBn9GZxeqlVAjVSJITArLR/uQkkUM6g==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKUkxVTUtYZ0RWUFVxY0Rl
-            UFFadVlzUFJVMGpzRVd5bHVDQmQycVlNSkcwCkMvcUJMRFVWTzNHZ3pxemRLelJP
-            K3pQMFdURmpRUVRuL1lzT09FVVdBd3MKLS0tIE9LY0NHSW1UWUJpbWdNQW1CVUlD
-            b1FmZnVjOFFCMDVXdFBtZzZWdkt6RVUKvLoHmEhkyeKHlstRoT3duTIQTojxzcFI
-            NapIBB3/6Qqho+kYc8/hLWb61EsSX9yqO9C6f6FpFrwi0696OvP3mA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZT3ZES3BHWWpDekt0VEYz
+            emUvUTQ3WUFWd0w2VlVSWHMrd3ZvZjYvYlJZCkcyRjBZWEdGTXJZVENyZ1U2YTV2
+            eU1MS3NCQzZ3Y3ZhOG4rRVByU1ZlRU0KLS0tIFdGVTliOFpSTWl0YlV6OTVUbk9O
+            SjBoUnNOVTB1QWFDYnVwWkhaN3d0VGMKjNiW597mLAogPyDBUhEDYd/VyePXesL7
+            kzyV/e8t/5zHs3/I17ZUd8bxdCjbrrXI1g4Swx31yCgZOk8uKAuLRQ==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4ZFNoMmNXV2F1U2E2bUhv
-            K3lGTCs2KzZYbXVlWEdVelNDTS80SW85c0J3CkszNGkrbFVKWks4dmwyYlpQMkpW
-            Zm02cG41ZlpwcEdCbzFkSHpjWHpCdG8KLS0tIHlrNXp6TTI5ZnhGTUNMWTZ0ekVS
-            VExPWk1zeVExYXdaL2o1WVB5NlhsNFkK3vsnc4qE08W13ttzt+YCHbQh2c/mOxFZ
-            DneXTgOjkyBaY5JDFKlzlIN3m8QRBG5vPOuSKXaoFmY8E68RzNey3w==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQaTlNTjVXTHFzNS9GUk1S
+            bVMxeWdwSUlmN3B6QlovejI3SlNuc2dJMjFVClF2VFRVNjFrQldRcHNLeWhpWFE1
+            UDRvY3RTZHZCa2RDZ1RmVWRHb2ttUVUKLS0tIEI0QS9SL3lTeXVITVgvcHVCNmdW
+            cVl6T3NWWEVkWExuTldqQU5CUzFTM1UKFYD1jdEQfFRNBkRyL+1gZzCdpJHN7QqU
+            4CVOsIeVl6ufWG4D2FfP4Zow5uhnvDXmWqBCmpJ/iVKnu3klihlndA==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvdHA5WHA2V2RNTTZXNVVT
-            Wks2a2tqT045ZkJFYTN2RHhmdkZxMjlPRDNFCm1HaHhLNkp6NWZxNUYvOTRybE1Z
-            Y1l5eDFkcXRWSko3ODhqV2htb3pzcDQKLS0tIGI3YlI4dCtMbGl1aHFZdDBic0Jv
-            LzV3NWhFQTlaZ1Y3R0paaEZPZDNpZzgK3/ZE3+F+mq574MfiF7PRlKmAU6mUTiGF
-            Ffqh0kQumHH7nBuunD0L7Zp2j15hMjUs/oxX558jY9BNl+rN2VWO0Q==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxRGZ5WVFJQzFSWlR6dDMv
+            bXJsNlZLeVVpK1RuaVpySkcreHE1SkNMSjA4CkxGMzVvZHZ4ZTdRdzh6K3V6OVQ0
+            RkI3bWg5ZUw5RFlQN05zdC9HVkdjYlUKLS0tIGdibTdwbnRhMmZEZ2VPelF6a3Aw
+            U1dGQmxOTklFTmFaMTc1MGQvRVB1TzgKkhxjImoj1lxpvBMjKJJOiM2eC2bQ73Ay
+            Rket8CjZnfRhYDD9YoOWBNswONQoVY8/dSXgLDObtfFxbnjZ1pj63A==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBueFhiQzdMaU1zR2VtOEtO
-            WFVtdVJLU3B3TzRSSENodUpuUm03TnBHQnhBCmRrdjJScEVsS0JTQmthZWIzVFlv
-            TVY3TUo0VllPWElua21mczZvT3YxYjAKLS0tIFpDcE0wSXdSRXFGY2tLd1orVE9L
-            Y2MyZUhOaEVVZU9Hc0xHbWtMdG1Ca2cKHU7pgODnNVDiMFF6be07a320a9HWKIdO
-            OKFA9R6WX1TFhKBKNDqK/mokJBTxu4nR16ewHSWOU13O/M8aKCQhug==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3RWRsdXNTQkNJWXFTODY4
+            WVNYb2xKZHJWWTUvZmlMS3VkYnhWQkVaZHpFCjJjY2JzeFQza3llNHZFYWVVK0Ri
+            K2ZJNUlZMWxFbGdhQ2pxRlh4VjVITFkKLS0tIGFHSDI5aW5aTUdFTEJOMnNjVXlm
+            SVlDVk9Xdnc0WVpFN2VmSlZIajJielkKz8xnfxIArN9PLjUorYPzakmLx7/bsoq0
+            EfoiB6ZpuWMeNEmfHygTEUPTC7eWw42EIYk964vI6LySFQyO3Z8p5g==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-12-19T12:56:28Z"
-    mac: ENC[AES256_GCM,data:v7Rn7dPOzfcgab2MhiU7h0CXjkAbkpBX7l7iLdnw3RUIjxulTXVuPpgenojF5yVqFCPgm2LKBKniD+cvtMvVhb00a1tnDNM/tfjH9GjBYNZH9xtPWJED7GLASd6nIF5BZhANKhH8yphAi5VJ/4cyEdMFbWu+2gO8GyQxJQYhgY8=,iv:bbbZ8vF+Vbwq/6PXN/7qvRO62M/eDZ591v4gXc1fs+g=,tag:dyt9LVU32hnbVT12C/Afqw==,type:str]
+        - recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2WFIzVEZPUmFBclpweDZR
+            WXZFb0FjcWxDRTNpQmFRaU9BY0lPTzAxNWhvClk5UmxFQllGQ29VOGIxeS9xMmV2
+            SUdEaFJ3bFZPSjVjQ1JnVS9jSWxXaWcKLS0tIGs0ZE0wMUZDeGNWNlhoN3JOMmlG
+            c1E1Sld1ejZhTStKTU5teEJKT2JwVXcKuEQnA6b1WJ+RNqmrZ8t3joiEZ57Oq9M1
+            P4tMGerB12A1myTJlt5Ss2OCTBUV7ooVRNsyPjyvJy/YTyjqZ5xmxg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2023-12-20T01:34:00Z"
+    mac: ENC[AES256_GCM,data:6MLBRPA5g2r3yy/i7DSxjWaYhHH/4GiAqL/pRIvYyIrKQWYvfviWlTX9dqHVzzCXjueEXUM5dXFb2B+Sds68EGgBuBlZvBchtstHUOtMLE3pttC+xCzerQFyrPDrXbnpfdDYPHWxvhhhFpWu8G5RSfzSgkgp7+cx9iZHq/g1k/Q=,iv:8yFIOgHtBiCtbamufrXXHrjIq5DV3MIJbTJPtXlgpPg=,tag:CVOIojTN2KkXJsDVyiZjMQ==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
    version: 3.8.1
--- a/machines/sops.nix
+++ b/machines/sops.nix
@ -24,6 +24,9 @@
      singbox_uuid = {
         owner = "root";
      };
+      grafana_cloud_api = {
+         owner = "prometheus";
+      };
    };
  };
 }
--- a/modules/nixos/prometheus.nix
+++ b/modules/nixos/prometheus.nix
@ -1,4 +1,3 @@
-
 { config, pkgs, lib, ... }:

 with lib;
@ -17,13 +16,38 @@ in
          description = "Enable Prometheus exporter on every supported services";
        };
      };
+      grafana = {
+        enable = mkEnableOption "Grafana Cloud";
+        password_file = mkOption {
+          type = types.path;
+        };
+      };
    };
  };

-  config = {
+  config = mkMerge [{
+    services.caddy.globalConfig = ''
+      servers {
+        metrics
+      }
+    '';
+    services.restic.server.prometheus = cfg.enable;
+    services.gotosocial.settings = {
+      metrics-enable = true;
+    };
    services.prometheus = mkIf cfg.enable {
      enable = true;
      port = 9091;
+      globalConfig.external_labels = { hostname = config.networking.hostName; };
+      remoteWrite = mkIf cfg.grafana.enable [
+        { name = "grafana";
+          url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
+          basic_auth = {
+            username = "1340065";
+            password_file = cfg.grafana.password_file;
+          };
+        }
+      ];
      exporters = {
        node = {
          enable = true;
@ -44,5 +68,28 @@ in
        }
      ];
    };
-  };
-}
+  }
+  {
+    services.prometheus.scrapeConfigs = [
+      ( mkIf config.services.caddy.enable {
+        job_name = "caddy";
+        static_configs = [
+          { targets = [ "localhost:2019" ]; }
+        ];
+      })
+      ( mkIf config.services.restic.server.enable {
+        job_name = "restic";
+        static_configs = [
+          { targets = [ config.services.restic.server.listenAddress ]; }
+        ];
+      })
+      ( mkIf config.services.gotosocial.enable {
+        job_name = "gotosocial";
+        static_configs = [
+          { targets = [ "localhost:${toString config.services.gotosocial.settings.port}" ]; }
+        ];
+      })
+    ];
+  }
+  ];
+}