From b9eebc2a7ef94e3752fabd9660e4e7be97aa44da Mon Sep 17 00:00:00 2001 From: xinyangli Date: Wed, 20 Dec 2023 11:13:20 +0800 Subject: [PATCH] all: add prometheus --- .sops.yaml | 1 + flake.nix | 2 ++ machines/dolomite/default.nix | 9 +++++ machines/massicot/default.nix | 4 +++ machines/massicot/services.nix | 9 +++++ machines/secrets.yaml | 64 ++++++++++++++++++++-------------- machines/sops.nix | 3 ++ modules/nixos/prometheus.nix | 55 ++++++++++++++++++++++++++--- 8 files changed, 116 insertions(+), 31 deletions(-) diff --git a/.sops.yaml b/.sops.yaml index baadf5e..dac73f2 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -39,6 +39,7 @@ creation_rules: - *host-raspite - *host-sgp-00 - *host-tok-00 + - *host-massicot - path_regex: home/xin/secrets.yaml key_groups: - age: diff --git a/flake.nix b/flake.nix index f7e2e10..d89b9ae 100644 --- a/flake.nix +++ b/flake.nix @@ -104,6 +104,7 @@ sgp-00 = { name, nodes, pkgs, ... }: with inputs; { imports = [ + self.nixosModules.default machines/dolomite ]; nixpkgs.system = "x86_64-linux"; @@ -118,6 +119,7 @@ tok-00 = { name, nodes, pkgs, ... }: with inputs; { imports = [ + self.nixosModules.default machines/dolomite ]; nixpkgs.system = "x86_64-linux"; diff --git a/machines/dolomite/default.nix b/machines/dolomite/default.nix index 5dd6073..f03d8b4 100644 --- a/machines/dolomite/default.nix +++ b/machines/dolomite/default.nix @@ -37,6 +37,15 @@ networking.firewall.allowedTCPPorts = [ 80 8080 ]; networking.firewall.allowedUDPPorts = [ ] ++ (lib.range 6311 6314); + custom.prometheus = { + enable = true; + exporters.enable = true; + grafana = { + enable = true; + password_file = config.sops.secrets.grafana_cloud_api.path; + }; + }; + services.sing-box = let singTls = { enabled = true; diff --git a/machines/massicot/default.nix b/machines/massicot/default.nix index ab6a5f3..7ffd7b6 100644 --- a/machines/massicot/default.nix +++ b/machines/massicot/default.nix @@ -18,6 +18,10 @@ gts_env = { owner = "gotosocial"; }; + grafana_cloud_api = { + owner = "prometheus"; + sopsFile = ../secrets.yaml; + }; }; }; diff --git a/machines/massicot/services.nix b/machines/massicot/services.nix index 410d546..e0b00bd 100644 --- a/machines/massicot/services.nix +++ b/machines/massicot/services.nix @@ -11,6 +11,15 @@ in domain = "vaultwarden.xinyang.life"; }; + custom.prometheus = { + enable = true; + exporters.enable = true; + grafana = { + enable = true; + password_file = config.sops.secrets.grafana_cloud_api.path; + }; + }; + fileSystems = builtins.listToAttrs (map (share: { name = "/mnt/storage/${share}"; value = { diff --git a/machines/secrets.yaml b/machines/secrets.yaml index fba11c5..46b1575 100644 --- a/machines/secrets.yaml +++ b/machines/secrets.yaml @@ -6,6 +6,7 @@ singbox_sg_server: ENC[AES256_GCM,data:5rogqKm5yiy5Yvz4Vo1a6Q==,iv:Vx9wNTdVHkReu singbox_jp_server: ENC[AES256_GCM,data:xKTcxkcu1WIsT/wlMpEoqGJK,iv:nXetY339YuOi2jFEb3xkPTglHRMk/quIrQL4ko+8MxY=,tag:+Nwsx65/gdrDhL1ZurR5Ng==,type:str] singbox_password: ENC[AES256_GCM,data:0tBIzwtNSQqbGlD+CDnQfJigbFVBChEL,iv:W2HaHeSkvmS6jHSnfOJ6tD2QXuUq1A+mfZf7sEXB++E=,tag:5BtYAv1NO70IL4m/uG8QKA==,type:str] singbox_uuid: ENC[AES256_GCM,data:ufN+vDl/rDASoQL23tHwlr3ybMyrlC/Kd7bT0c5+SP+bc6Zj,iv:+uwt/N9LpFaJK6MjoczyrZ039MDZn4kRmtEoq4OvdFU=,tag:6Yma9+yrISwQoSRDgUbuwA==,type:str] +grafana_cloud_api: ENC[AES256_GCM,data:Pz+tE09dcJa+ZEWS3vtpOtitGCA9Cg/+gOd/0FsF8ooxzPyN9/UMuTcP02aIPW5v7yZCkGJOAXufIyechNf0crgAV/KmwGGwixH7I+1f3sDtGiFZEMnQgrysyfJo0KIrIZ8XP0SyXDs3vKjDU8cUI4+IyucHacWQ1kWdEtINjcPNHRPS2yaMUIvsRn0z8Cs2byMD3ghUHHHOz40CuO6r4A==,iv:cHvbeCmLFmJPNKsl1BBYx9WJP7ZJWi+8c9yHZWc6FTs=,tag:yWXtPokYE4frCmzzzyEqEg==,type:str] sops: kms: [] gcp_kms: [] @@ -15,50 +16,59 @@ sops: - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBweTlPTGVRbUlndTdES0s2 - SVM2N2FUMnozQk11cDk0cTFEb1l6YldkVHc4CmhnNzJyY1VKRWhpc0tTbFNKeDBD - a0hzMi93Ly9zY2Fjd1RCdjV6WnVmOU0KLS0tIFh6NVFteWxxNithMGM0dnJiNE9X - dGovQ2ZMZWx1djVkb0Y4ZVNLRDJPRncKz0N/zP3mN97BpLaDgE9hx/zooGyHAnvC - D8iH/1PZ21uMYeUQq83B8mDKbv+qAltA/vD+ZNnb4ULjYLmVn5p/hQ== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0MUxIZHJTYk9YS0lPOGZK + VUJhQ1liNEtXZ3ZYaCtqQWVBTGVJclVVRER3CmJUcS9yY2x1TFFYMkpZOWxZeW5w + WFk0WTNoWmphdG12dTdHaW9tYVRjS1UKLS0tIHd4enVwalRDaHQwK0U1RFNHOEVI + N0UrRjRxTWJRanI4VnRjWlhzQS8zSGsKSJJnFuEp7yO8bIh2LpSvgjsYAK05u2TE + a+UBiu6xQQaUnL02CAau4xHqBn9GZxeqlVAjVSJITArLR/uQkkUM6g== -----END AGE ENCRYPTED FILE----- - recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKUkxVTUtYZ0RWUFVxY0Rl - UFFadVlzUFJVMGpzRVd5bHVDQmQycVlNSkcwCkMvcUJMRFVWTzNHZ3pxemRLelJP - K3pQMFdURmpRUVRuL1lzT09FVVdBd3MKLS0tIE9LY0NHSW1UWUJpbWdNQW1CVUlD - b1FmZnVjOFFCMDVXdFBtZzZWdkt6RVUKvLoHmEhkyeKHlstRoT3duTIQTojxzcFI - NapIBB3/6Qqho+kYc8/hLWb61EsSX9yqO9C6f6FpFrwi0696OvP3mA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZT3ZES3BHWWpDekt0VEYz + emUvUTQ3WUFWd0w2VlVSWHMrd3ZvZjYvYlJZCkcyRjBZWEdGTXJZVENyZ1U2YTV2 + eU1MS3NCQzZ3Y3ZhOG4rRVByU1ZlRU0KLS0tIFdGVTliOFpSTWl0YlV6OTVUbk9O + SjBoUnNOVTB1QWFDYnVwWkhaN3d0VGMKjNiW597mLAogPyDBUhEDYd/VyePXesL7 + kzyV/e8t/5zHs3/I17ZUd8bxdCjbrrXI1g4Swx31yCgZOk8uKAuLRQ== -----END AGE ENCRYPTED FILE----- - recipient: age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4ZFNoMmNXV2F1U2E2bUhv - K3lGTCs2KzZYbXVlWEdVelNDTS80SW85c0J3CkszNGkrbFVKWks4dmwyYlpQMkpW - Zm02cG41ZlpwcEdCbzFkSHpjWHpCdG8KLS0tIHlrNXp6TTI5ZnhGTUNMWTZ0ekVS - VExPWk1zeVExYXdaL2o1WVB5NlhsNFkK3vsnc4qE08W13ttzt+YCHbQh2c/mOxFZ - DneXTgOjkyBaY5JDFKlzlIN3m8QRBG5vPOuSKXaoFmY8E68RzNey3w== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQaTlNTjVXTHFzNS9GUk1S + bVMxeWdwSUlmN3B6QlovejI3SlNuc2dJMjFVClF2VFRVNjFrQldRcHNLeWhpWFE1 + UDRvY3RTZHZCa2RDZ1RmVWRHb2ttUVUKLS0tIEI0QS9SL3lTeXVITVgvcHVCNmdW + cVl6T3NWWEVkWExuTldqQU5CUzFTM1UKFYD1jdEQfFRNBkRyL+1gZzCdpJHN7QqU + 4CVOsIeVl6ufWG4D2FfP4Zow5uhnvDXmWqBCmpJ/iVKnu3klihlndA== -----END AGE ENCRYPTED FILE----- - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvdHA5WHA2V2RNTTZXNVVT - Wks2a2tqT045ZkJFYTN2RHhmdkZxMjlPRDNFCm1HaHhLNkp6NWZxNUYvOTRybE1Z - Y1l5eDFkcXRWSko3ODhqV2htb3pzcDQKLS0tIGI3YlI4dCtMbGl1aHFZdDBic0Jv - LzV3NWhFQTlaZ1Y3R0paaEZPZDNpZzgK3/ZE3+F+mq574MfiF7PRlKmAU6mUTiGF - Ffqh0kQumHH7nBuunD0L7Zp2j15hMjUs/oxX558jY9BNl+rN2VWO0Q== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxRGZ5WVFJQzFSWlR6dDMv + bXJsNlZLeVVpK1RuaVpySkcreHE1SkNMSjA4CkxGMzVvZHZ4ZTdRdzh6K3V6OVQ0 + RkI3bWg5ZUw5RFlQN05zdC9HVkdjYlUKLS0tIGdibTdwbnRhMmZEZ2VPelF6a3Aw + U1dGQmxOTklFTmFaMTc1MGQvRVB1TzgKkhxjImoj1lxpvBMjKJJOiM2eC2bQ73Ay + Rket8CjZnfRhYDD9YoOWBNswONQoVY8/dSXgLDObtfFxbnjZ1pj63A== -----END AGE ENCRYPTED FILE----- - recipient: age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBueFhiQzdMaU1zR2VtOEtO - WFVtdVJLU3B3TzRSSENodUpuUm03TnBHQnhBCmRrdjJScEVsS0JTQmthZWIzVFlv - TVY3TUo0VllPWElua21mczZvT3YxYjAKLS0tIFpDcE0wSXdSRXFGY2tLd1orVE9L - Y2MyZUhOaEVVZU9Hc0xHbWtMdG1Ca2cKHU7pgODnNVDiMFF6be07a320a9HWKIdO - OKFA9R6WX1TFhKBKNDqK/mokJBTxu4nR16ewHSWOU13O/M8aKCQhug== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3RWRsdXNTQkNJWXFTODY4 + WVNYb2xKZHJWWTUvZmlMS3VkYnhWQkVaZHpFCjJjY2JzeFQza3llNHZFYWVVK0Ri + K2ZJNUlZMWxFbGdhQ2pxRlh4VjVITFkKLS0tIGFHSDI5aW5aTUdFTEJOMnNjVXlm + SVlDVk9Xdnc0WVpFN2VmSlZIajJielkKz8xnfxIArN9PLjUorYPzakmLx7/bsoq0 + EfoiB6ZpuWMeNEmfHygTEUPTC7eWw42EIYk964vI6LySFQyO3Z8p5g== -----END AGE ENCRYPTED FILE----- - lastmodified: "2023-12-19T12:56:28Z" - mac: ENC[AES256_GCM,data:v7Rn7dPOzfcgab2MhiU7h0CXjkAbkpBX7l7iLdnw3RUIjxulTXVuPpgenojF5yVqFCPgm2LKBKniD+cvtMvVhb00a1tnDNM/tfjH9GjBYNZH9xtPWJED7GLASd6nIF5BZhANKhH8yphAi5VJ/4cyEdMFbWu+2gO8GyQxJQYhgY8=,iv:bbbZ8vF+Vbwq/6PXN/7qvRO62M/eDZ591v4gXc1fs+g=,tag:dyt9LVU32hnbVT12C/Afqw==,type:str] + - recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2WFIzVEZPUmFBclpweDZR + WXZFb0FjcWxDRTNpQmFRaU9BY0lPTzAxNWhvClk5UmxFQllGQ29VOGIxeS9xMmV2 + SUdEaFJ3bFZPSjVjQ1JnVS9jSWxXaWcKLS0tIGs0ZE0wMUZDeGNWNlhoN3JOMmlG + c1E1Sld1ejZhTStKTU5teEJKT2JwVXcKuEQnA6b1WJ+RNqmrZ8t3joiEZ57Oq9M1 + P4tMGerB12A1myTJlt5Ss2OCTBUV7ooVRNsyPjyvJy/YTyjqZ5xmxg== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2023-12-20T01:34:00Z" + mac: ENC[AES256_GCM,data:6MLBRPA5g2r3yy/i7DSxjWaYhHH/4GiAqL/pRIvYyIrKQWYvfviWlTX9dqHVzzCXjueEXUM5dXFb2B+Sds68EGgBuBlZvBchtstHUOtMLE3pttC+xCzerQFyrPDrXbnpfdDYPHWxvhhhFpWu8G5RSfzSgkgp7+cx9iZHq/g1k/Q=,iv:8yFIOgHtBiCtbamufrXXHrjIq5DV3MIJbTJPtXlgpPg=,tag:CVOIojTN2KkXJsDVyiZjMQ==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.8.1 diff --git a/machines/sops.nix b/machines/sops.nix index 1a8aa50..64cc721 100644 --- a/machines/sops.nix +++ b/machines/sops.nix @@ -24,6 +24,9 @@ singbox_uuid = { owner = "root"; }; + grafana_cloud_api = { + owner = "prometheus"; + }; }; }; } diff --git a/modules/nixos/prometheus.nix b/modules/nixos/prometheus.nix index ac0a976..5234e76 100644 --- a/modules/nixos/prometheus.nix +++ b/modules/nixos/prometheus.nix @@ -1,4 +1,3 @@ - { config, pkgs, lib, ... }: with lib; @@ -17,13 +16,38 @@ in description = "Enable Prometheus exporter on every supported services"; }; }; + grafana = { + enable = mkEnableOption "Grafana Cloud"; + password_file = mkOption { + type = types.path; + }; + }; }; }; - config = { + config = mkMerge [{ + services.caddy.globalConfig = '' + servers { + metrics + } + ''; + services.restic.server.prometheus = cfg.enable; + services.gotosocial.settings = { + metrics-enable = true; + }; services.prometheus = mkIf cfg.enable { enable = true; port = 9091; + globalConfig.external_labels = { hostname = config.networking.hostName; }; + remoteWrite = mkIf cfg.grafana.enable [ + { name = "grafana"; + url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push"; + basic_auth = { + username = "1340065"; + password_file = cfg.grafana.password_file; + }; + } + ]; exporters = { node = { enable = true; @@ -44,5 +68,28 @@ in } ]; }; - }; -} \ No newline at end of file + } + { + services.prometheus.scrapeConfigs = [ + ( mkIf config.services.caddy.enable { + job_name = "caddy"; + static_configs = [ + { targets = [ "localhost:2019" ]; } + ]; + }) + ( mkIf config.services.restic.server.enable { + job_name = "restic"; + static_configs = [ + { targets = [ config.services.restic.server.listenAddress ]; } + ]; + }) + ( mkIf config.services.gotosocial.enable { + job_name = "gotosocial"; + static_configs = [ + { targets = [ "localhost:${toString config.services.gotosocial.settings.port}" ]; } + ]; + }) + ]; + } + ]; +}