From ced05f99fcbab5516138d6f86dccfff389a5ba6c Mon Sep 17 00:00:00 2001 From: xinyangli Date: Thu, 1 Aug 2024 17:01:53 +0800 Subject: [PATCH] prometheus: enable every where --- flake.lock | 36 ++++++------- machines/dolomite/default.nix | 6 ++- machines/massicot/default.nix | 3 ++ machines/massicot/kanidm-provision.nix | 36 ++++++++++++- machines/massicot/secrets.yaml | 7 +-- machines/massicot/services.nix | 73 ++++++++++++++++++++------ machines/weilite/default.nix | 4 ++ modules/nixos/prometheus/caddy.nix | 7 --- modules/nixos/prometheus/default.nix | 2 + modules/nixos/prometheus/immich.nix | 26 +++++++++ 10 files changed, 154 insertions(+), 46 deletions(-) create mode 100644 modules/nixos/prometheus/immich.nix diff --git a/flake.lock b/flake.lock index 70b6d93..d78098f 100644 --- a/flake.lock +++ b/flake.lock @@ -99,11 +99,11 @@ ] }, "locked": { - "lastModified": 1722203588, - "narHash": "sha256-91V5FMSQ4z9bkhTCf0f86Zjw0bh367daSf0mzCIW0vU=", + "lastModified": 1722462338, + "narHash": "sha256-ss0G8t8RJVDewA3MyqgAlV951cWRK6EtVhVKEZ7J5LU=", "owner": "nix-community", "repo": "home-manager", - "rev": "792757f643cedc13f02098d8ed506d82e19ec1da", + "rev": "6e090576c4824b16e8759ebca3958c5b09659ee8", "type": "github" }, "original": { @@ -143,11 +143,11 @@ ] }, "locked": { - "lastModified": 1722302960, - "narHash": "sha256-byZl18UZCHy3vLhxrXp8THzlzmwNfil93ZQLY30i7/Q=", + "lastModified": 1722476581, + "narHash": "sha256-dCNcvjaOTu+cPin3VUym9pglsghWYJe5oUpKTuAgiiU=", "owner": "nix-community", "repo": "nix-vscode-extensions", - "rev": "e1a1e6cabd0140ed353e173290e6d92510f5fd66", + "rev": "1fe57eaf074d28246ec310486fe3db4ae44d0451", "type": "github" }, "original": { @@ -158,11 +158,11 @@ }, "nixos-hardware": { "locked": { - "lastModified": 1722278305, - "narHash": "sha256-xLBAegsn9wbj+pQfbX07kykd5VBV3Ywk3IbObVAAlWA=", + "lastModified": 1722332872, + "narHash": "sha256-2xLM4sc5QBfi0U/AANJAW21Bj4ZX479MHPMPkB+eKBU=", "owner": "NixOS", "repo": "nixos-hardware", - "rev": "eab049fe178c11395d65a858ba1b56461ba9652d", + "rev": "14c333162ba53c02853add87a0000cbd7aa230c2", "type": "github" }, "original": { @@ -174,11 +174,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1722307517, - "narHash": "sha256-QTsnr7l9MlOVMASsv6w1luxAKqR32RJceBYQlg5bpkM=", + "lastModified": 1722489601, + "narHash": "sha256-sB37J92AwEcmzg0GgxdI1TU6M+psUpbo0iYLFJBmsfo=", "owner": "xinyangli", "repo": "nixpkgs", - "rev": "ebd00a4a357b00eb56b5d11f57aeb2b1fca9be34", + "rev": "eee3d54e62749dfd0f263e3903ca0ec1ebdbe72b", "type": "github" }, "original": { @@ -190,11 +190,11 @@ }, "nixpkgs-stable": { "locked": { - "lastModified": 1722087241, - "narHash": "sha256-2ShmEaFi0kJVOEEu5gmlykN5dwjWYWYUJmlRTvZQRpU=", + "lastModified": 1722221733, + "narHash": "sha256-sga9SrrPb+pQJxG1ttJfMPheZvDOxApFfwXCFO0H9xw=", "owner": "nixos", "repo": "nixpkgs", - "rev": "8c50662509100d53229d4be607f1a3a31157fa12", + "rev": "12bf09802d77264e441f48e25459c10c93eada2e", "type": "github" }, "original": { @@ -222,11 +222,11 @@ }, "nur": { "locked": { - "lastModified": 1722309060, - "narHash": "sha256-lJ5auEUvSI0H0GwW5yWLgizvJ2A+N4aL2u2Xqa6JVCc=", + "lastModified": 1722485061, + "narHash": "sha256-opkrX6noshjk2V3PKBiksA8+M6K7cu3EuiuAWL04pNs=", "owner": "nix-community", "repo": "NUR", - "rev": "e491266f3f0e1fee7709c4d3d68130b5500dcd46", + "rev": "3bf06551d5922d420607091f5a3321e712ece307", "type": "github" }, "original": { diff --git a/machines/dolomite/default.nix b/machines/dolomite/default.nix index 9bdb8e0..3a5406f 100644 --- a/machines/dolomite/default.nix +++ b/machines/dolomite/default.nix @@ -44,7 +44,7 @@ in networking.firewall.allowedUDPPorts = [ ] ++ (lib.range 6311 6314); custom.prometheus = { - enable = false; + enable = true; exporters.blackbox.enable = true; }; @@ -161,6 +161,10 @@ in outbound = "dns-out"; protocol = "dns"; } + { + inbound = "sg0"; + outbound = "direct"; + } { inbound = "sg4"; outbound = "direct"; diff --git a/machines/massicot/default.nix b/machines/massicot/default.nix index 56cbfe5..2e7597f 100644 --- a/machines/massicot/default.nix +++ b/machines/massicot/default.nix @@ -25,6 +25,9 @@ owner = "prometheus"; sopsFile = ../secrets.yaml; }; + grafana_oauth_secret = { + owner = "grafana"; + }; }; }; diff --git a/machines/massicot/kanidm-provision.nix b/machines/massicot/kanidm-provision.nix index 9eb10dd..3bbf1ca 100644 --- a/machines/massicot/kanidm-provision.nix +++ b/machines/massicot/kanidm-provision.nix @@ -18,7 +18,19 @@ members = [ "xin" ]; }; immich-users = { - members = [ "xin" "zhuo" ]; + members = [ "xin" "zhuo" "ycm" ]; + }; + grafana-superadmins = { + members = [ "xin" ]; + }; + grafana-admins = { + members = [ "xin" ]; + }; + grafana-editors = { + members = [ "xin" ]; + }; + grafana-users = { + members = [ "xin" ]; }; }; persons = { @@ -31,6 +43,11 @@ displayName = "Zhuo"; mailAddresses = [ "13681104320@163.com" ]; }; + + ycm = { + displayName = "Chunming"; + mailAddresses = [ "chunmingyou@gmail.com" ]; + }; }; systems.oauth2 = { forgejo = { @@ -75,5 +92,22 @@ immich-users = [ "openid" "email" "profile" ]; }; }; + grafana = { + displayName = "Grafana"; + originUrl = "https://grafana.xinyang.life/"; + scopeMaps = { + grafana-users = [ "openid" "email" "profile" "groups" ]; + }; + claimMaps = { + grafana_role = { + joinType = "array"; + valuesByGroup = { + grafana-superadmins = [ "GrafanaAdmin" ]; + grafana-admins = [ "Admin" ]; + grafana-editors = [ "Editor" ]; + }; + }; + }; + }; }; } \ No newline at end of file diff --git a/machines/massicot/secrets.yaml b/machines/massicot/secrets.yaml index 5e5d0fe..c1dbf8e 100644 --- a/machines/massicot/secrets.yaml +++ b/machines/massicot/secrets.yaml @@ -1,6 +1,7 @@ storage_box_mount: ENC[AES256_GCM,data:9lOAL3tkfB0pN4/cuM4SX0xoMrW0UUEzTN8spw3MQ3BWrfsRc3Stsce3puXz1sRf,iv:7Q9wzpBgQ3tqcfy0n/c6Ya84Kg60nhR/e2H0pVntWsY=,tag:9a0xvNBGQpCvhxgmV3hrww==,type:str] gts_env: ENC[AES256_GCM,data:CKFKHXCJvTD0HFkVrBWhabcl/cloCT03qcZIc5JymiIAu+o6wef6gsQlkKP81vxC9S3XMYtLgXQ03D7Jetkfg+7nafF1+ogN,iv:/axRqZIatwYL++/KmBIievPPyKRkHGmVpgRe2Eet+fg=,tag:gwxyuePOYiD1vlSyq3yjXA==,type:str] hedgedoc_env: ENC[AES256_GCM,data:zwAA+zKSJT0tZyYArCaa1lfL0y8DNHDp/thS11DrVxNvjmk38o0ydsKArfZKzFYye+qNBzz1B4sPCdW4cFgQUNgbM+n9AvoMB8CssdmQ+sALKmozA5aEV23q+khZSGlHocP6WA==,iv:SgZruOS1nanK64Ex1dvgoD1HzbGbNa4DFSBuVoaNgEc=,tag:R+I8m1AloDCXs5PdpEpS0w==,type:str] +grafana_oauth_secret: ENC[AES256_GCM,data:2dSgxeWXNtlvbrgW9whCVuM6tfzd4lVhynwQTSPbBJndhI8scpJle7LjI1+b14FS9boBsuYO+ym4Pf1I8/jJtKkj6X6I0BmXFBC/SfpCpo+ZGrxacg==,iv:N8iTPqMagKP3hWc7n0bjgYKvaFaw11ITvDn9lUkkAPY=,tag:Cz59fA2Zq3jVvhfxaFuGAA==,type:str] sops: kms: [] gcp_kms: [] @@ -25,8 +26,8 @@ sops: dnFBa0lDWWZtS1BHdzBoVzNTaGNkSEEKi/W1n7RT8NpTp00SBMwxsUJAPDhumJ/i V2VnaSNwouD3SswTcoBzqQpBP9XrqzjIYGke90ZODFQbMY9WDQ+O0g== -----END AGE ENCRYPTED FILE----- - lastmodified: "2023-12-22T08:05:27Z" - mac: ENC[AES256_GCM,data:CiXU49arW+3w4/Lkh4l+6VjopyP7XNCU4AmuwZmnmQ7Vv4RCt84fC6lM6o4HiCc5jB07QY+2WZ5LvWz9zgSt636UpnCMgbG1w2Lxae38fW02RHJv90rn+cyyddB5kSucr5/P5NKBOZut54Cf4zVW9BaqajpQMxe4hEOn+xXpXz8=,iv:beWRlUvb6OUOK+mUXdvpvmM8S7xK0QIkIA2Bk9QA35c=,tag:KrBXqsAdBAhtwygdEHnUqQ==,type:str] + lastmodified: "2024-07-31T09:24:12Z" + mac: ENC[AES256_GCM,data:/TIuK0O0e3Kkb9yjVE4GEPLRRFo1wQEzfcuCcX/hS4eGSgVPu8p52meEzVW7Z9GLiKsmgSW+L5fW4k+kXGcOfKr1BarjfHa0pGcfoW/gb8BV2TFmX9rQk9ioh5m5NT97pv5KgrpPIU+HjUEe5ORebVZh5sW/R3Vh3PCyagINcIs=,iv:mU4P7BUnMjA/hIhX9SUImOuazoccPdnmeNIPGJUXaLw=,tag:EMXAVLgFZk3Mgv2O1rgibg==,type:str] pgp: [] unencrypted_suffix: _unencrypted - version: 3.8.1 + version: 3.9.0 diff --git a/machines/massicot/services.nix b/machines/massicot/services.nix index d5d5c13..2db1118 100644 --- a/machines/massicot/services.nix +++ b/machines/massicot/services.nix @@ -1,4 +1,4 @@ -{ config, pkgs, inputs, ... }: +{ config, pkgs, ... }: let kanidm_listen_port = 5324; in @@ -31,15 +31,16 @@ in exporters.blackbox.enable = true; }; - systemd.mounts = map (share: { - what = "//u380335-sub1.your-storagebox.de/u380335-sub1/${share}"; - where = "/mnt/storage/${share}"; - type = "cifs"; - options = "rw,uid=${share},gid=${share},credentials=${config.sops.secrets.storage_box_mount.path},_netdev,fsc"; - before = [ "${share}.service" ]; - after = [ "cachefilesd.service" ]; - wantedBy = [ "${share}.service" ]; - }) [ "forgejo" "gotosocial" "conduit" "hedgedoc" ]; + systemd.mounts = map + (share: { + what = "//u380335-sub1.your-storagebox.de/u380335-sub1/${share}"; + where = "/mnt/storage/${share}"; + type = "cifs"; + options = "rw,uid=${share},gid=${share},credentials=${config.sops.secrets.storage_box_mount.path},_netdev,fsc"; + before = [ "${share}.service" ]; + after = [ "cachefilesd.service" ]; + wantedBy = [ "${share}.service" ]; + }) [ "forgejo" "gotosocial" "conduit" "hedgedoc" ]; services.cachefilesd.enable = true; @@ -53,9 +54,9 @@ in security.acme = { acceptTerms = true; certs."auth.xinyang.life" = { - email = "lixinyang411@gmail.com"; - listenHTTP = "127.0.0.1:1360"; - group = "kanidm"; + email = "lixinyang411@gmail.com"; + listenHTTP = "127.0.0.1:1360"; + group = "kanidm"; }; }; @@ -162,6 +163,38 @@ in }; }; + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "127.0.0.1"; + http_port = 3003; + root_url = "https://grafana.xinyang.life"; + domain = "grafana.xinyang.life"; + }; + "auth.generic_oauth" = { + enabled = true; + name = "Kanidm"; + client_id = "grafana"; + scopes = "openid,profile,email,groups"; + auth_url = "https://auth.xinyang.life/ui/oauth2"; + token_url = "https://auth.xinyang.life/oauth2/token"; + api_url = "https://auth.xinyang.life/oauth2/openid/grafana/userinfo"; + use_pkce = true; + use_refresh_token = true; + allow_sign_up = true; + login_attribute_path = "preferred_username"; + groups_attribute_path = "groups"; + role_attribute_path = "contains(grafana_role[*], 'GrafanaAdmin') && 'GrafanaAdmin' || contains(grafana_role[*], 'Admin') && 'Admin' || contains(grafana_role[*], 'Editor') && 'Editor' || 'Viewer'"; + allow_assign_grafana_admin = true; + auto_login = true; + }; + "auth" = { disable_login_form = true; }; + }; + }; + + systemd.services.grafana.serviceConfig.EnvironmentFile = config.sops.secrets.grafana_oauth_secret.path; + users.users.git = { isSystemUser = true; useDefaultShell = true; @@ -192,9 +225,9 @@ in virtualHosts."https://git.xinyang.life:443".extraConfig = '' reverse_proxy http://${config.services.gitea.settings.server.DOMAIN}:${toString config.services.gitea.settings.server.HTTP_PORT} ''; - + virtualHosts."http://auth.xinyang.life:80".extraConfig = '' - reverse_proxy ${config.security.acme.certs."auth.xinyang.life".listenHTTP} + reverse_proxy ${config.security.acme.certs."auth.xinyang.life".listenHTTP} ''; virtualHosts."https://auth.xinyang.life".extraConfig = '' reverse_proxy https://127.0.0.1:${toString kanidm_listen_port} { @@ -205,7 +238,7 @@ in } } ''; - virtualHosts."https://ntfy.xinyang.life".extraConfig = '' + virtualHosts."https://ntfy.xinyang.life".extraConfig = '' reverse_proxy unix/${config.services.ntfy-sh.settings.listen-unix} @httpget { protocol http @@ -214,5 +247,13 @@ in } redir @httpget https://{host}{uri} ''; + + virtualHosts."https://grafana.xinyang.life".extraConfig = + let + grafanaSettings = config.services.grafana.settings.server; + in + '' + reverse_proxy http://${grafanaSettings.http_addr}:${toString grafanaSettings.http_port} + ''; }; } diff --git a/machines/weilite/default.nix b/machines/weilite/default.nix index 0f6bf18..0ad8822 100644 --- a/machines/weilite/default.nix +++ b/machines/weilite/default.nix @@ -42,6 +42,10 @@ with lib; }; }; + custom.prometheus = { + enable = true; + }; + systemd.mounts = [ { what = "immich"; where = "/mnt/XinPhotos/immich"; diff --git a/modules/nixos/prometheus/caddy.nix b/modules/nixos/prometheus/caddy.nix index d35049b..96b7f43 100644 --- a/modules/nixos/prometheus/caddy.nix +++ b/modules/nixos/prometheus/caddy.nix @@ -30,13 +30,6 @@ in labels = { severity = "critical"; }; annotations = { summary = "Upstream {{ $labels.unstream }} not healthy"; }; } - { - alert = "HighRequestLatency"; - expr = "histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 5"; - for = "2m"; - labels = { severity = "warning"; }; - annotations = { summary = "High request latency on {{ $labels.instance }}"; description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; }; - } ]; } ]; diff --git a/modules/nixos/prometheus/default.nix b/modules/nixos/prometheus/default.nix index c0f0a70..8c43908 100644 --- a/modules/nixos/prometheus/default.nix +++ b/modules/nixos/prometheus/default.nix @@ -28,6 +28,7 @@ in ./blackbox.nix ./caddy.nix ./gotosocial.nix + ./immich.nix ./ntfy-sh.nix ./restic.nix ]; @@ -46,6 +47,7 @@ in blackbox.enable = mkExporterOption false; caddy.enable = mkExporterOption config.services.caddy.enable; gotosocial.enable = mkExporterOption config.services.gotosocial.enable; + immich.enable = mkExporterOption config.services.immich.enable; ntfy-sh.enable = mkExporterOption config.services.gotosocial.enable; }; grafana = { diff --git a/modules/nixos/prometheus/immich.nix b/modules/nixos/prometheus/immich.nix new file mode 100644 index 0000000..095075d --- /dev/null +++ b/modules/nixos/prometheus/immich.nix @@ -0,0 +1,26 @@ +{ config, lib, ... }: +let + cfg = config.custom.prometheus; + immichEnv = config.services.immich.environment; + metricPort = + if builtins.hasAttr "IMMICH_API_METRICS_PORT" immichEnv + then immichEnv.IMMICH_API_METRICS_PORT + else 8081; +in +{ + config = lib.mkIf (cfg.enable && cfg.exporters.immich.enable) { + services.immich.environment = { + IMMICH_METRICS = "true"; + }; + + services.prometheus.scrapeConfigs = [ + { + job_name = "immich"; + static_configs = [ + { targets = [ "127.0.0.1:${toString metricPort}" ]; } + ]; + } + ]; + }; + +}