prometheus: enable every where

flake.lock

@@ -99,11 +99,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1722203588,
+        "lastModified": 1722462338,
-        "narHash": "sha256-91V5FMSQ4z9bkhTCf0f86Zjw0bh367daSf0mzCIW0vU=",
+        "narHash": "sha256-ss0G8t8RJVDewA3MyqgAlV951cWRK6EtVhVKEZ7J5LU=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "792757f643cedc13f02098d8ed506d82e19ec1da",
+        "rev": "6e090576c4824b16e8759ebca3958c5b09659ee8",
         "type": "github"
       },
       "original": {
@@ -143,11 +143,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1722302960,
+        "lastModified": 1722476581,
-        "narHash": "sha256-byZl18UZCHy3vLhxrXp8THzlzmwNfil93ZQLY30i7/Q=",
+        "narHash": "sha256-dCNcvjaOTu+cPin3VUym9pglsghWYJe5oUpKTuAgiiU=",
         "owner": "nix-community",
         "repo": "nix-vscode-extensions",
-        "rev": "e1a1e6cabd0140ed353e173290e6d92510f5fd66",
+        "rev": "1fe57eaf074d28246ec310486fe3db4ae44d0451",
         "type": "github"
       },
       "original": {
@@ -158,11 +158,11 @@
     },
     "nixos-hardware": {
       "locked": {
-        "lastModified": 1722278305,
+        "lastModified": 1722332872,
-        "narHash": "sha256-xLBAegsn9wbj+pQfbX07kykd5VBV3Ywk3IbObVAAlWA=",
+        "narHash": "sha256-2xLM4sc5QBfi0U/AANJAW21Bj4ZX479MHPMPkB+eKBU=",
         "owner": "NixOS",
         "repo": "nixos-hardware",
-        "rev": "eab049fe178c11395d65a858ba1b56461ba9652d",
+        "rev": "14c333162ba53c02853add87a0000cbd7aa230c2",
         "type": "github"
       },
       "original": {
@@ -174,11 +174,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1722307517,
+        "lastModified": 1722489601,
-        "narHash": "sha256-QTsnr7l9MlOVMASsv6w1luxAKqR32RJceBYQlg5bpkM=",
+        "narHash": "sha256-sB37J92AwEcmzg0GgxdI1TU6M+psUpbo0iYLFJBmsfo=",
         "owner": "xinyangli",
         "repo": "nixpkgs",
-        "rev": "ebd00a4a357b00eb56b5d11f57aeb2b1fca9be34",
+        "rev": "eee3d54e62749dfd0f263e3903ca0ec1ebdbe72b",
         "type": "github"
       },
       "original": {
@@ -190,11 +190,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1722087241,
+        "lastModified": 1722221733,
-        "narHash": "sha256-2ShmEaFi0kJVOEEu5gmlykN5dwjWYWYUJmlRTvZQRpU=",
+        "narHash": "sha256-sga9SrrPb+pQJxG1ttJfMPheZvDOxApFfwXCFO0H9xw=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "8c50662509100d53229d4be607f1a3a31157fa12",
+        "rev": "12bf09802d77264e441f48e25459c10c93eada2e",
         "type": "github"
       },
       "original": {
@@ -222,11 +222,11 @@
     },
     "nur": {
       "locked": {
-        "lastModified": 1722309060,
+        "lastModified": 1722485061,
-        "narHash": "sha256-lJ5auEUvSI0H0GwW5yWLgizvJ2A+N4aL2u2Xqa6JVCc=",
+        "narHash": "sha256-opkrX6noshjk2V3PKBiksA8+M6K7cu3EuiuAWL04pNs=",
         "owner": "nix-community",
         "repo": "NUR",
-        "rev": "e491266f3f0e1fee7709c4d3d68130b5500dcd46",
+        "rev": "3bf06551d5922d420607091f5a3321e712ece307",
         "type": "github"
       },
       "original": {

machines/dolomite/default.nix

@@ -44,7 +44,7 @@ in
     networking.firewall.allowedUDPPorts = [ ] ++ (lib.range 6311 6314);
 
     custom.prometheus = {
-      enable = false;
+      enable = true;
       exporters.blackbox.enable = true;
     };
 
@@ -161,6 +161,10 @@ in
               outbound = "dns-out";
               protocol = "dns";
             }
+            {
+              inbound = "sg0";
+              outbound = "direct";
+            }
             {
               inbound = "sg4";
               outbound = "direct";

machines/massicot/default.nix

@@ -25,6 +25,9 @@
         owner = "prometheus";
         sopsFile = ../secrets.yaml;
       };
+      grafana_oauth_secret = {
+        owner = "grafana";
+      };
     };
   };
 

machines/massicot/kanidm-provision.nix

@@ -18,7 +18,19 @@
       members = [ "xin" ];
     };
     immich-users = {
-      members = [ "xin" "zhuo" ];
+      members = [ "xin" "zhuo" "ycm" ];
+    };
+    grafana-superadmins = {
+      members = [ "xin" ];
+    };
+    grafana-admins = {
+      members = [ "xin" ];
+    };
+    grafana-editors = {
+      members = [ "xin" ];
+    };
+    grafana-users = {
+      members = [ "xin" ];
     };
   };
   persons = {
@@ -31,6 +43,11 @@
       displayName = "Zhuo";
       mailAddresses = [ "13681104320@163.com" ];
     };
+
+    ycm = {
+      displayName = "Chunming";
+      mailAddresses = [ "chunmingyou@gmail.com" ];
+    };
   };
   systems.oauth2 = {
     forgejo = {
@@ -75,5 +92,22 @@
         immich-users = [ "openid" "email" "profile" ];
       };
     };
+    grafana = {
+      displayName = "Grafana";
+      originUrl = "https://grafana.xinyang.life/";
+      scopeMaps = {
+        grafana-users = [ "openid" "email" "profile" "groups" ];
+      };
+      claimMaps = {
+        grafana_role = {
+          joinType = "array";
+          valuesByGroup = {
+            grafana-superadmins = [ "GrafanaAdmin" ];
+            grafana-admins = [ "Admin" ];
+            grafana-editors = [ "Editor" ];
+          };
+        };
+      };
+    };
   };
 }

machines/massicot/secrets.yaml

@@ -1,6 +1,7 @@
 storage_box_mount: ENC[AES256_GCM,data:9lOAL3tkfB0pN4/cuM4SX0xoMrW0UUEzTN8spw3MQ3BWrfsRc3Stsce3puXz1sRf,iv:7Q9wzpBgQ3tqcfy0n/c6Ya84Kg60nhR/e2H0pVntWsY=,tag:9a0xvNBGQpCvhxgmV3hrww==,type:str]
 gts_env: ENC[AES256_GCM,data:CKFKHXCJvTD0HFkVrBWhabcl/cloCT03qcZIc5JymiIAu+o6wef6gsQlkKP81vxC9S3XMYtLgXQ03D7Jetkfg+7nafF1+ogN,iv:/axRqZIatwYL++/KmBIievPPyKRkHGmVpgRe2Eet+fg=,tag:gwxyuePOYiD1vlSyq3yjXA==,type:str]
 hedgedoc_env: ENC[AES256_GCM,data:zwAA+zKSJT0tZyYArCaa1lfL0y8DNHDp/thS11DrVxNvjmk38o0ydsKArfZKzFYye+qNBzz1B4sPCdW4cFgQUNgbM+n9AvoMB8CssdmQ+sALKmozA5aEV23q+khZSGlHocP6WA==,iv:SgZruOS1nanK64Ex1dvgoD1HzbGbNa4DFSBuVoaNgEc=,tag:R+I8m1AloDCXs5PdpEpS0w==,type:str]
+grafana_oauth_secret: ENC[AES256_GCM,data:2dSgxeWXNtlvbrgW9whCVuM6tfzd4lVhynwQTSPbBJndhI8scpJle7LjI1+b14FS9boBsuYO+ym4Pf1I8/jJtKkj6X6I0BmXFBC/SfpCpo+ZGrxacg==,iv:N8iTPqMagKP3hWc7n0bjgYKvaFaw11ITvDn9lUkkAPY=,tag:Cz59fA2Zq3jVvhfxaFuGAA==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -25,8 +26,8 @@ sops:
             dnFBa0lDWWZtS1BHdzBoVzNTaGNkSEEKi/W1n7RT8NpTp00SBMwxsUJAPDhumJ/i
             V2VnaSNwouD3SswTcoBzqQpBP9XrqzjIYGke90ZODFQbMY9WDQ+O0g==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-12-22T08:05:27Z"
+    lastmodified: "2024-07-31T09:24:12Z"
-    mac: ENC[AES256_GCM,data:CiXU49arW+3w4/Lkh4l+6VjopyP7XNCU4AmuwZmnmQ7Vv4RCt84fC6lM6o4HiCc5jB07QY+2WZ5LvWz9zgSt636UpnCMgbG1w2Lxae38fW02RHJv90rn+cyyddB5kSucr5/P5NKBOZut54Cf4zVW9BaqajpQMxe4hEOn+xXpXz8=,iv:beWRlUvb6OUOK+mUXdvpvmM8S7xK0QIkIA2Bk9QA35c=,tag:KrBXqsAdBAhtwygdEHnUqQ==,type:str]
+    mac: ENC[AES256_GCM,data:/TIuK0O0e3Kkb9yjVE4GEPLRRFo1wQEzfcuCcX/hS4eGSgVPu8p52meEzVW7Z9GLiKsmgSW+L5fW4k+kXGcOfKr1BarjfHa0pGcfoW/gb8BV2TFmX9rQk9ioh5m5NT97pv5KgrpPIU+HjUEe5ORebVZh5sW/R3Vh3PCyagINcIs=,iv:mU4P7BUnMjA/hIhX9SUImOuazoccPdnmeNIPGJUXaLw=,tag:EMXAVLgFZk3Mgv2O1rgibg==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
-    version: 3.8.1
+    version: 3.9.0

machines/massicot/services.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, inputs, ... }:
+{ config, pkgs, ... }:
 let
   kanidm_listen_port = 5324;
 in
@@ -31,15 +31,16 @@ in
     exporters.blackbox.enable = true;
   };
 
-  systemd.mounts = map (share: {
+  systemd.mounts = map
-    what = "//u380335-sub1.your-storagebox.de/u380335-sub1/${share}";
+    (share: {
-    where = "/mnt/storage/${share}";
+      what = "//u380335-sub1.your-storagebox.de/u380335-sub1/${share}";
-    type = "cifs";
+      where = "/mnt/storage/${share}";
-    options = "rw,uid=${share},gid=${share},credentials=${config.sops.secrets.storage_box_mount.path},_netdev,fsc";
+      type = "cifs";
-    before = [ "${share}.service" ];
+      options = "rw,uid=${share},gid=${share},credentials=${config.sops.secrets.storage_box_mount.path},_netdev,fsc";
-    after = [ "cachefilesd.service" ];
+      before = [ "${share}.service" ];
-    wantedBy = [ "${share}.service" ];
+      after = [ "cachefilesd.service" ];
-  }) [ "forgejo" "gotosocial" "conduit" "hedgedoc" ];
+      wantedBy = [ "${share}.service" ];
+    }) [ "forgejo" "gotosocial" "conduit" "hedgedoc" ];
 
   services.cachefilesd.enable = true;
 
@@ -53,9 +54,9 @@ in
   security.acme = {
     acceptTerms = true;
     certs."auth.xinyang.life" = {
-        email = "lixinyang411@gmail.com";
+      email = "lixinyang411@gmail.com";
-        listenHTTP = "127.0.0.1:1360";
+      listenHTTP = "127.0.0.1:1360";
-        group = "kanidm";
+      group = "kanidm";
     };
   };
 
@@ -162,6 +163,38 @@ in
     };
   };
 
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        http_addr = "127.0.0.1";
+        http_port = 3003;
+        root_url = "https://grafana.xinyang.life";
+        domain = "grafana.xinyang.life";
+      };
+      "auth.generic_oauth" = {
+        enabled = true;
+        name = "Kanidm";
+        client_id = "grafana";
+        scopes = "openid,profile,email,groups";
+        auth_url = "https://auth.xinyang.life/ui/oauth2";
+        token_url = "https://auth.xinyang.life/oauth2/token";
+        api_url = "https://auth.xinyang.life/oauth2/openid/grafana/userinfo";
+        use_pkce = true;
+        use_refresh_token = true;
+        allow_sign_up = true;
+        login_attribute_path = "preferred_username";
+        groups_attribute_path = "groups";
+        role_attribute_path = "contains(grafana_role[*], 'GrafanaAdmin') && 'GrafanaAdmin' || contains(grafana_role[*], 'Admin') && 'Admin' || contains(grafana_role[*], 'Editor') && 'Editor' || 'Viewer'";
+        allow_assign_grafana_admin = true;
+        auto_login = true;
+      };
+      "auth" = { disable_login_form = true; };
+    };
+  };
+
+  systemd.services.grafana.serviceConfig.EnvironmentFile = config.sops.secrets.grafana_oauth_secret.path;
+
   users.users.git = {
     isSystemUser = true;
     useDefaultShell = true;
@@ -194,7 +227,7 @@ in
     '';
 
     virtualHosts."http://auth.xinyang.life:80".extraConfig = ''
-        reverse_proxy ${config.security.acme.certs."auth.xinyang.life".listenHTTP}
+      reverse_proxy ${config.security.acme.certs."auth.xinyang.life".listenHTTP}
     '';
     virtualHosts."https://auth.xinyang.life".extraConfig = ''
       reverse_proxy https://127.0.0.1:${toString kanidm_listen_port} {
@@ -205,7 +238,7 @@ in
           }
       }
     '';
-    virtualHosts."https://ntfy.xinyang.life".extraConfig =  ''
+    virtualHosts."https://ntfy.xinyang.life".extraConfig = ''
       reverse_proxy unix/${config.services.ntfy-sh.settings.listen-unix}
       @httpget {
         protocol http
@@ -214,5 +247,13 @@ in
       }
       redir @httpget https://{host}{uri}
     '';
+
+    virtualHosts."https://grafana.xinyang.life".extraConfig =
+      let
+        grafanaSettings = config.services.grafana.settings.server;
+      in
+      ''
+        reverse_proxy http://${grafanaSettings.http_addr}:${toString grafanaSettings.http_port}
+      '';
   };
 }

machines/weilite/default.nix

@@ -42,6 +42,10 @@ with lib;
       };
     };
 
+    custom.prometheus = {
+      enable = true;
+    };
+
     systemd.mounts = [
       { what = "immich";
         where = "/mnt/XinPhotos/immich";

modules/nixos/prometheus/caddy.nix

@@ -30,13 +30,6 @@ in
             labels = { severity = "critical"; };
             annotations = { summary = "Upstream {{ $labels.unstream }} not healthy"; };
           }
-          {
-            alert = "HighRequestLatency";
-            expr = "histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 5";
-            for = "2m";
-            labels = { severity = "warning"; };
-            annotations = { summary = "High request latency on {{ $labels.instance }}"; description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; };
-          }
         ];
       }
     ];

modules/nixos/prometheus/default.nix

@@ -28,6 +28,7 @@ in
     ./blackbox.nix
     ./caddy.nix
     ./gotosocial.nix
+    ./immich.nix
     ./ntfy-sh.nix
     ./restic.nix
   ];
@@ -46,6 +47,7 @@ in
         blackbox.enable = mkExporterOption false;
         caddy.enable = mkExporterOption config.services.caddy.enable;
         gotosocial.enable = mkExporterOption config.services.gotosocial.enable;
+        immich.enable = mkExporterOption config.services.immich.enable;
         ntfy-sh.enable = mkExporterOption config.services.gotosocial.enable;
       };
       grafana = {

modules/nixos/prometheus/immich.nix

@@ -0,0 +1,26 @@
+{ config, lib, ... }:
+let
+  cfg = config.custom.prometheus;
+  immichEnv = config.services.immich.environment;
+  metricPort =
+    if builtins.hasAttr "IMMICH_API_METRICS_PORT" immichEnv
+    then immichEnv.IMMICH_API_METRICS_PORT
+    else 8081;
+in
+{
+  config = lib.mkIf (cfg.enable && cfg.exporters.immich.enable) {
+    services.immich.environment = {
+      IMMICH_METRICS = "true";
+    };
+
+    services.prometheus.scrapeConfigs = [
+      {
+        job_name = "immich";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString metricPort}" ]; }
+        ];
+      }
+    ];
+  };
+
+}