hosts/homeserver/default.nix

@@ -29,8 +29,6 @@
           gui.domain = "syncthing.${privateDomain}";
           doBackups = true;
         };
-
-        prometheus.storageRetentionSize = "20GB";
       };
 
       web-services = {
@@ -61,11 +59,6 @@
           enable = true;
           domain = "bookmarks.${privateDomain}";
         };
-
-        grafana = {
-          enable = true;
-          domain = "grafana.${privateDomain}";
-        };
       };
     };
 }

hosts/homeserver/secrets.json

@@ -27,9 +27,6 @@
   "radicale": {
     "htpasswd": "ENC[AES256_GCM,data:SuVrYk3BP3Hds9sKwiuEOgAA5RPBR3f3RW16xtpO6w9aUjZR37jPKPOPluHxBUIulap2p/Oj5VZvyyeDP2MxQw==,iv:sANgLVPRrZEjlC7n/r5zVma/qIDCraLxi88o/sVgayQ=,tag:KbUM7hTRM0Z9iiiQFUfp/w==,type:str]"
   },
-  "grafana": {
-    "admin-password": "ENC[AES256_GCM,data:MNdjh+025bi5wtP77aKSGzcx8NgfY+YppH4qu/o=,iv:Hv8IF0n0A5+Hs6FQ7tXkdFbPN0ArdZD3vmrdovc0/Yg=,tag:8y0NsLQ+9k+mWnHbHzZvGg==,type:str]"
-  },
   "sops": {
     "age": [
       {
@@ -41,8 +38,8 @@
         "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXOVRQU1R4VlBhdEFUdnR6\nZFNuNldaSkxJL3ptOVVscjRBNkQ4dFBmQUVBCmZrQmFMV0hWbTBQcm1FS3JrR0ZC\nbktvT04xczd6VkdCUWk2NnVVZHNFWkUKLS0tIGUwOHJSMHVsNTEyZEU2VWJFNGVy\nMVFDVThrRGQwZEtPeFYzZUVQYi80ZjAKUd/XzyzqMkMowvyeCnQDbOGJDKbuAUQb\nFClQuiH5iSQQrVPw7SHBNgdqbcdtC+hZ4tpPaV/wWtlpcqpr5mBJSA==\n-----END AGE ENCRYPTED FILE-----\n"
       }
     ],
-    "lastmodified": "2026-03-01T13:14:08Z",
-    "mac": "ENC[AES256_GCM,data:GywNeQDso210CV1rEY1LCwUu5ZDdjyP/W+QOJ5GJhLnLiF3BjivS4J9VbOANduhepnRLLHFKW0+zdzPJJl3ltBFpAuoDigEPF5PbXOT6R7oiEW3kkgD5LP79789ijkeGqGf2DUN0J04OFkcTnTy2tAd2O3kSnhi/EbIMvw7zjUY=,iv:TqXETQOQuuQCJ2+tvPeXHY08BGbhQ4fsu+RqX1EiPHY=,tag:qfaYvY/iuu0Ou9AxhJUnRQ==,type:str]",
+    "lastmodified": "2026-02-27T21:25:33Z",
+    "mac": "ENC[AES256_GCM,data:kvP/sCIqs4Ic/IwEn1FwZOc71N6zAuyn7NDQWF78EW9HFfPjqUwN3z5PJOmEc1U74KbJB1gHIFMfteLx547IhGHn45oHlju7gco8jCdFhX9XL40Hp72B2px3MaG8UXp2DmDel9n7wQYCjNK1W6aOcQnpids/5uZu2gb/sV4/IIM=,iv:mlgEoRieeL/qc9rTZ8tYFrTdOkqph8GdAe2zJydjMIU=,tag:7czP2+PRO/RXZaBanW12tg==,type:str]",
     "unencrypted_suffix": "_unencrypted",
     "version": "3.11.0"
   }

hosts/vps-monitor/default.nix

@@ -32,7 +32,23 @@
           enable = true;
           domain = "status.${privateDomain}";
           generateDefaultEndpoints = true;
-          endpoints."dav.${sproutedDomain}".enable = false;
+          endpoints = {
+            "dav.${sproutedDomain}".enable = false;
+            "alerts.${sproutedDomain}" = {
+              path = "/v1/health";
+              extraConditions = [ "[BODY].healthy == true" ];
+            };
+          };
+        };
+
+        ntfy = {
+          enable = true;
+          domain = "alerts.${sproutedDomain}";
+        };
+
+        grafana = {
+          enable = true;
+          domain = "grafana.${privateDomain}";
         };
       };
   };

hosts/vps-monitor/secrets.json

@@ -3,6 +3,9 @@
   "healthchecks": {
     "ping-key": "ENC[AES256_GCM,data:Zq71AU3oym7fC364YZNyRtx4N2G35Q==,iv:ibMBpcrSocLBhtumsSV00+KVN6Pi4SzE7soCkZcU4fY=,tag:Wv/Wr0wRZGXucMHZHgoNtg==,type:str]"
   },
+  "grafana": {
+    "admin-password": "ENC[AES256_GCM,data:2YRh4DT+1w5W/X3ELIe3Uu2EnMIHG4gUhV5ri6E=,iv:owHyuoupNQO09aRBgU2phIwxg22U1rUqKyYbw2193m4=,tag:dopVQwf4Ewf+lsFterfDOA==,type:str]"
+  },
   "nebula": {
     "host-key": "ENC[AES256_GCM,data:usSLqYOvDAAs7z1xo+gccDqgUE78upK+k522ldKcPoFKKBH87Us7gi6+XAOMDQ79U6i8j4l1lAE8kRdqDuvasodESHVSW9gSnnv5E73MVr0d1Snh7tAewVzneac+2R2R8tUzKzwzWM5SyyvJSoKGBg8WmGzdGT8UqC623utlYQ==,iv:NoZ2u8IK4g1Kwb6uZZ1jXJH4eFO9Jj5Phi5hPM4K72o=,tag:9mOv6oSESH+8r2ZC4yUE+w==,type:str]"
   },
@@ -21,8 +24,8 @@
         "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTVmV4dkZEaWVDNHMxUFdr\nYnhuVWpReXBNSEhhZkltQTE4bEpzSlBzL0VJCk15UFlwa0haWTZNaE1DVzVZVFBI\nd0QzcUptYmQ3dmhhdjhFV0xDSWdmMGMKLS0tIFhWamx6SXJleFFSVUFkRmw2VFZy\nOVVhNm9NSE0yRGFMQjNrM1B6cDVxSXMKrhAkDcWqutgSmQI5O+5i8fcwuTh2/XKr\nljK/Vn8EvGr8qEUeHzOBI1b5VtgngJkVJyfM9G/Q0lZvQF7ZZ5YCgQ==\n-----END AGE ENCRYPTED FILE-----\n"
       }
     ],
-    "lastmodified": "2026-03-01T13:13:54Z",
-    "mac": "ENC[AES256_GCM,data:/jWzYWYyHvFkjRN2jphn1ocPF8AsrULMa8W4U4GKPqdawT3VchEbJl0pKQV/bW5ApJkjHkopWmJjyScpkQXEXY2up6arNsN2IqB89c6jodMQihSKvSdG8pfeeZaFXY1HPL0KNUS/zV5y64rUNX5qv/t9wL9qatcIKP1SDw9Khro=,iv:hPoyKPQwbv6sbWCb0MMAWigDBKqnb/OmLcQN3UGg6yk=,tag:VDO8di4fLBh9XCD24UuvuA==,type:str]",
+    "lastmodified": "2026-01-10T23:14:34Z",
+    "mac": "ENC[AES256_GCM,data:RpG1S1iuaHmBc9wT9Tau6dZHcFX9DwQA0kHcAuTZxbjopLfYJnVosvU9S9N+4tAZqwGLpKqVxX3UiUWLEIFwrcB3xM3/SdpKWD4CE/cyMbvwWCIN4RcHveFJ6py7ItXtipzLC0lZeRWDXP+IyREuKggAID50zXuSIPr6BDlnkYA=,iv:gWykL+vhNB3tLHtHlhbNd5iUPKjB5vDpx0nAf5j+F8Y=,tag:MAI8aaSR+g33IIYM4opqCw==,type:str]",
     "unencrypted_suffix": "_unencrypted",
     "version": "3.11.0"
   }

modules/nixos/services/alloy.nix

@@ -57,7 +57,7 @@ in
           |> lib.map (host: "https://${host.config.custom.services.prometheus.domain}/api/v1/write");
       in
       {
-        "alloy/prometheus-endpoint.alloy" = {
+        "alloy/metrics-endpoint.alloy" = {
           enable = cfg.collect.metrics |> anyIsTrue;
           text =
             prometheusEndpoints
@@ -80,8 +80,9 @@ in
               enable_collectors = ["systemd", "processes"]
             }
 
-            discovery.relabel "node_exporter" {
+            discovery.relabel "unix_exporter" {
               targets = prometheus.exporter.unix.default.targets
+
               rule {
                 target_label = "job"
                 replacement = "node"
@@ -89,7 +90,7 @@ in
             }
 
             prometheus.scrape "node_exporter" {
-              targets = discovery.relabel.node_exporter.output
+              targets = discovery.relabel.unix_exporter.output
               forward_to = [prometheus.remote_write.default.receiver]
               scrape_interval = "15s"
             }

modules/nixos/services/prometheus.nix

@@ -19,10 +19,6 @@ in
       type = lib.types.port;
       default = 9090;
     };
-    storageRetentionSize = lib.mkOption {
-      type = lib.types.nonEmptyStr;
-      default = "2GB";
-    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -34,10 +30,7 @@ in
       inherit (cfg) port;
       webExternalUrl = "https://${cfg.domain}";
 
-      extraFlags = [
-        "--web.enable-remote-write-receiver"
-        "--storage.tsdb.retention.size=${cfg.storageRetentionSize}"
-      ];
+      extraFlags = [ "--web.enable-remote-write-receiver" ];
       globalConfig = {
         scrape_interval = "30s";
         external_labels.monitor = "global";

modules/nixos/web-services/gatus.nix

@@ -19,13 +19,10 @@ in
       type = lib.types.port;
       default = 8080;
     };
-    alerts = {
-      enable = lib.mkEnableOption "";
     ntfyUrl = lib.mkOption {
       type = lib.types.nonEmptyStr;
       default = "https://${config.custom.web-services.ntfy.domain}";
     };
-    };
     generateDefaultEndpoints = lib.mkEnableOption "";
     endpoints = lib.mkOption {
       type = lib.types.attrsOf (
@@ -65,7 +62,7 @@ in
                 default = [ ];
               };
               enableAlerts = lib.mkEnableOption "" // {
-                default = cfg.alerts.enable;
+                default = true;
               };
             };
           }
@@ -115,7 +112,7 @@ in
           path = "${dataDir}/data.db";
         };
         connectivity.checker.target = "1.1.1.1:53"; # Cloudflare DNS
-        alerting.ntfy = lib.mkIf cfg.alerts.enable {
+        alerting.ntfy = {
           topic = "uptime";
           url = cfg.ntfyUrl;
           click = "https://${cfg.domain}";

modules/nixos/web-services/grafana.nix

@@ -7,6 +7,13 @@
 }:
 let
   cfg = config.custom.web-services.grafana;
+
+  prometheusDomains =
+    allHosts
+    |> lib.attrValues
+    |> lib.map (host: host.config.custom.services.prometheus)
+    |> lib.filter (prometheus: prometheus.enable)
+    |> lib.map (prometheus: prometheus.domain);
 in
 {
   options.custom.web-services.grafana = {
@@ -19,6 +26,15 @@ in
       type = lib.types.port;
       default = 3000;
     };
+    datasources.prometheus = {
+      enable = lib.mkEnableOption "" // {
+        default = prometheusDomains != [ ];
+      };
+      url = lib.mkOption {
+        type = lib.types.nonEmptyStr;
+        default = "https://metrics.${config.custom.networking.overlay.fqdn}";
+      };
+    };
     dashboards.nodeExporter.enable = lib.mkEnableOption "" // {
       default = true;
     };
@@ -57,24 +73,21 @@ in
 
         datasources.settings = {
           prune = true;
-          datasources =
-            allHosts
-            |> lib.attrValues
-            |> lib.filter (host: host.config.custom.services.prometheus.enable)
-            |> lib.map (host: {
-              name = "Prometheus ${host.config.networking.hostName}";
+          datasources = lib.optional cfg.datasources.prometheus.enable {
+            name = "Prometheus";
             type = "prometheus";
-              url = "https://${host.config.custom.services.prometheus.domain}";
-              isDefault = host.config.networking.hostName == config.networking.hostName;
+            inherit (cfg.datasources.prometheus) url;
+            isDefault = true;
             jsonData = {
               prometheusType = "Prometheus";
               prometheusVersion = "3.7.2";
             };
-            });
           };
         };
       };
 
+    };
+
     # https://grafana.com/grafana/dashboards/1860-node-exporter-full/
     environment.etc."grafana-dashboards/node-exporter-full.json" = {
       enable = cfg.dashboards.nodeExporter.enable;
@@ -85,6 +98,20 @@ in
       };
     };
 
-    custom.services.caddy.virtualHosts.${cfg.domain}.port = cfg.port;
+    custom.services.caddy = {
+      virtualHosts.${cfg.domain}.port = cfg.port;
+
+      virtualHosts."metrics.${config.custom.networking.overlay.fqdn}".extraConfig =
+        let
+          upstreams = prometheusDomains |> lib.map (domain: "https://${domain}") |> lib.concatStringsSep " ";
+        in
+        ''
+          reverse_proxy ${upstreams} {
+            header_up Host {upstream_hostport}
+            lb_policy first
+            health_uri /api/v1/status/buildinfo
+          }
+        '';
+    };
   };
 }