diff --git a/hosts/homeserver/default.nix b/hosts/homeserver/default.nix index 26ed9ca..e6c18bd 100644 --- a/hosts/homeserver/default.nix +++ b/hosts/homeserver/default.nix @@ -29,8 +29,6 @@ gui.domain = "syncthing.${privateDomain}"; doBackups = true; }; - - prometheus.storageRetentionSize = "20GB"; }; web-services = { @@ -61,11 +59,6 @@ enable = true; domain = "bookmarks.${privateDomain}"; }; - - grafana = { - enable = true; - domain = "grafana.${privateDomain}"; - }; }; }; } diff --git a/hosts/homeserver/secrets.json b/hosts/homeserver/secrets.json index 22e54b2..b0c9251 100644 --- a/hosts/homeserver/secrets.json +++ b/hosts/homeserver/secrets.json @@ -27,9 +27,6 @@ "radicale": { "htpasswd": "ENC[AES256_GCM,data:SuVrYk3BP3Hds9sKwiuEOgAA5RPBR3f3RW16xtpO6w9aUjZR37jPKPOPluHxBUIulap2p/Oj5VZvyyeDP2MxQw==,iv:sANgLVPRrZEjlC7n/r5zVma/qIDCraLxi88o/sVgayQ=,tag:KbUM7hTRM0Z9iiiQFUfp/w==,type:str]" }, - "grafana": { - "admin-password": "ENC[AES256_GCM,data:MNdjh+025bi5wtP77aKSGzcx8NgfY+YppH4qu/o=,iv:Hv8IF0n0A5+Hs6FQ7tXkdFbPN0ArdZD3vmrdovc0/Yg=,tag:8y0NsLQ+9k+mWnHbHzZvGg==,type:str]" - }, "sops": { "age": [ { @@ -41,8 +38,8 @@ "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXOVRQU1R4VlBhdEFUdnR6\nZFNuNldaSkxJL3ptOVVscjRBNkQ4dFBmQUVBCmZrQmFMV0hWbTBQcm1FS3JrR0ZC\nbktvT04xczd6VkdCUWk2NnVVZHNFWkUKLS0tIGUwOHJSMHVsNTEyZEU2VWJFNGVy\nMVFDVThrRGQwZEtPeFYzZUVQYi80ZjAKUd/XzyzqMkMowvyeCnQDbOGJDKbuAUQb\nFClQuiH5iSQQrVPw7SHBNgdqbcdtC+hZ4tpPaV/wWtlpcqpr5mBJSA==\n-----END AGE ENCRYPTED FILE-----\n" } ], - "lastmodified": "2026-03-01T13:14:08Z", - "mac": "ENC[AES256_GCM,data:GywNeQDso210CV1rEY1LCwUu5ZDdjyP/W+QOJ5GJhLnLiF3BjivS4J9VbOANduhepnRLLHFKW0+zdzPJJl3ltBFpAuoDigEPF5PbXOT6R7oiEW3kkgD5LP79789ijkeGqGf2DUN0J04OFkcTnTy2tAd2O3kSnhi/EbIMvw7zjUY=,iv:TqXETQOQuuQCJ2+tvPeXHY08BGbhQ4fsu+RqX1EiPHY=,tag:qfaYvY/iuu0Ou9AxhJUnRQ==,type:str]", + "lastmodified": "2026-02-27T21:25:33Z", + "mac": "ENC[AES256_GCM,data:kvP/sCIqs4Ic/IwEn1FwZOc71N6zAuyn7NDQWF78EW9HFfPjqUwN3z5PJOmEc1U74KbJB1gHIFMfteLx547IhGHn45oHlju7gco8jCdFhX9XL40Hp72B2px3MaG8UXp2DmDel9n7wQYCjNK1W6aOcQnpids/5uZu2gb/sV4/IIM=,iv:mlgEoRieeL/qc9rTZ8tYFrTdOkqph8GdAe2zJydjMIU=,tag:7czP2+PRO/RXZaBanW12tg==,type:str]", "unencrypted_suffix": "_unencrypted", "version": "3.11.0" } diff --git a/hosts/vps-monitor/default.nix b/hosts/vps-monitor/default.nix index e45880b..109cdff 100644 --- a/hosts/vps-monitor/default.nix +++ b/hosts/vps-monitor/default.nix @@ -32,7 +32,23 @@ enable = true; domain = "status.${privateDomain}"; generateDefaultEndpoints = true; - endpoints."dav.${sproutedDomain}".enable = false; + endpoints = { + "dav.${sproutedDomain}".enable = false; + "alerts.${sproutedDomain}" = { + path = "/v1/health"; + extraConditions = [ "[BODY].healthy == true" ]; + }; + }; + }; + + ntfy = { + enable = true; + domain = "alerts.${sproutedDomain}"; + }; + + grafana = { + enable = true; + domain = "grafana.${privateDomain}"; }; }; }; diff --git a/hosts/vps-monitor/secrets.json b/hosts/vps-monitor/secrets.json index 7e95241..fa99e10 100644 --- a/hosts/vps-monitor/secrets.json +++ b/hosts/vps-monitor/secrets.json @@ -3,6 +3,9 @@ "healthchecks": { "ping-key": "ENC[AES256_GCM,data:Zq71AU3oym7fC364YZNyRtx4N2G35Q==,iv:ibMBpcrSocLBhtumsSV00+KVN6Pi4SzE7soCkZcU4fY=,tag:Wv/Wr0wRZGXucMHZHgoNtg==,type:str]" }, + "grafana": { + "admin-password": "ENC[AES256_GCM,data:2YRh4DT+1w5W/X3ELIe3Uu2EnMIHG4gUhV5ri6E=,iv:owHyuoupNQO09aRBgU2phIwxg22U1rUqKyYbw2193m4=,tag:dopVQwf4Ewf+lsFterfDOA==,type:str]" + }, "nebula": { "host-key": "ENC[AES256_GCM,data:usSLqYOvDAAs7z1xo+gccDqgUE78upK+k522ldKcPoFKKBH87Us7gi6+XAOMDQ79U6i8j4l1lAE8kRdqDuvasodESHVSW9gSnnv5E73MVr0d1Snh7tAewVzneac+2R2R8tUzKzwzWM5SyyvJSoKGBg8WmGzdGT8UqC623utlYQ==,iv:NoZ2u8IK4g1Kwb6uZZ1jXJH4eFO9Jj5Phi5hPM4K72o=,tag:9mOv6oSESH+8r2ZC4yUE+w==,type:str]" }, @@ -21,8 +24,8 @@ "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTVmV4dkZEaWVDNHMxUFdr\nYnhuVWpReXBNSEhhZkltQTE4bEpzSlBzL0VJCk15UFlwa0haWTZNaE1DVzVZVFBI\nd0QzcUptYmQ3dmhhdjhFV0xDSWdmMGMKLS0tIFhWamx6SXJleFFSVUFkRmw2VFZy\nOVVhNm9NSE0yRGFMQjNrM1B6cDVxSXMKrhAkDcWqutgSmQI5O+5i8fcwuTh2/XKr\nljK/Vn8EvGr8qEUeHzOBI1b5VtgngJkVJyfM9G/Q0lZvQF7ZZ5YCgQ==\n-----END AGE ENCRYPTED FILE-----\n" } ], - "lastmodified": "2026-03-01T13:13:54Z", - "mac": "ENC[AES256_GCM,data:/jWzYWYyHvFkjRN2jphn1ocPF8AsrULMa8W4U4GKPqdawT3VchEbJl0pKQV/bW5ApJkjHkopWmJjyScpkQXEXY2up6arNsN2IqB89c6jodMQihSKvSdG8pfeeZaFXY1HPL0KNUS/zV5y64rUNX5qv/t9wL9qatcIKP1SDw9Khro=,iv:hPoyKPQwbv6sbWCb0MMAWigDBKqnb/OmLcQN3UGg6yk=,tag:VDO8di4fLBh9XCD24UuvuA==,type:str]", + "lastmodified": "2026-01-10T23:14:34Z", + "mac": "ENC[AES256_GCM,data:RpG1S1iuaHmBc9wT9Tau6dZHcFX9DwQA0kHcAuTZxbjopLfYJnVosvU9S9N+4tAZqwGLpKqVxX3UiUWLEIFwrcB3xM3/SdpKWD4CE/cyMbvwWCIN4RcHveFJ6py7ItXtipzLC0lZeRWDXP+IyREuKggAID50zXuSIPr6BDlnkYA=,iv:gWykL+vhNB3tLHtHlhbNd5iUPKjB5vDpx0nAf5j+F8Y=,tag:MAI8aaSR+g33IIYM4opqCw==,type:str]", "unencrypted_suffix": "_unencrypted", "version": "3.11.0" } diff --git a/modules/nixos/services/alloy.nix b/modules/nixos/services/alloy.nix index 5abfd2c..8c270c1 100644 --- a/modules/nixos/services/alloy.nix +++ b/modules/nixos/services/alloy.nix @@ -57,7 +57,7 @@ in |> lib.map (host: "https://${host.config.custom.services.prometheus.domain}/api/v1/write"); in { - "alloy/prometheus-endpoint.alloy" = { + "alloy/metrics-endpoint.alloy" = { enable = cfg.collect.metrics |> anyIsTrue; text = prometheusEndpoints @@ -80,8 +80,9 @@ in enable_collectors = ["systemd", "processes"] } - discovery.relabel "node_exporter" { + discovery.relabel "unix_exporter" { targets = prometheus.exporter.unix.default.targets + rule { target_label = "job" replacement = "node" @@ -89,7 +90,7 @@ in } prometheus.scrape "node_exporter" { - targets = discovery.relabel.node_exporter.output + targets = discovery.relabel.unix_exporter.output forward_to = [prometheus.remote_write.default.receiver] scrape_interval = "15s" } diff --git a/modules/nixos/services/prometheus.nix b/modules/nixos/services/prometheus.nix index 182a70c..0f0fb58 100644 --- a/modules/nixos/services/prometheus.nix +++ b/modules/nixos/services/prometheus.nix @@ -19,10 +19,6 @@ in type = lib.types.port; default = 9090; }; - storageRetentionSize = lib.mkOption { - type = lib.types.nonEmptyStr; - default = "2GB"; - }; }; config = lib.mkIf cfg.enable { @@ -34,10 +30,7 @@ in inherit (cfg) port; webExternalUrl = "https://${cfg.domain}"; - extraFlags = [ - "--web.enable-remote-write-receiver" - "--storage.tsdb.retention.size=${cfg.storageRetentionSize}" - ]; + extraFlags = [ "--web.enable-remote-write-receiver" ]; globalConfig = { scrape_interval = "30s"; external_labels.monitor = "global"; diff --git a/modules/nixos/web-services/gatus.nix b/modules/nixos/web-services/gatus.nix index 1582efe..c47da79 100644 --- a/modules/nixos/web-services/gatus.nix +++ b/modules/nixos/web-services/gatus.nix @@ -19,12 +19,9 @@ in type = lib.types.port; default = 8080; }; - alerts = { - enable = lib.mkEnableOption ""; - ntfyUrl = lib.mkOption { - type = lib.types.nonEmptyStr; - default = "https://${config.custom.web-services.ntfy.domain}"; - }; + ntfyUrl = lib.mkOption { + type = lib.types.nonEmptyStr; + default = "https://${config.custom.web-services.ntfy.domain}"; }; generateDefaultEndpoints = lib.mkEnableOption ""; endpoints = lib.mkOption { @@ -65,7 +62,7 @@ in default = [ ]; }; enableAlerts = lib.mkEnableOption "" // { - default = cfg.alerts.enable; + default = true; }; }; } @@ -115,7 +112,7 @@ in path = "${dataDir}/data.db"; }; connectivity.checker.target = "1.1.1.1:53"; # Cloudflare DNS - alerting.ntfy = lib.mkIf cfg.alerts.enable { + alerting.ntfy = { topic = "uptime"; url = cfg.ntfyUrl; click = "https://${cfg.domain}"; diff --git a/modules/nixos/web-services/grafana.nix b/modules/nixos/web-services/grafana.nix index 96913ec..959336a 100644 --- a/modules/nixos/web-services/grafana.nix +++ b/modules/nixos/web-services/grafana.nix @@ -7,6 +7,13 @@ }: let cfg = config.custom.web-services.grafana; + + prometheusDomains = + allHosts + |> lib.attrValues + |> lib.map (host: host.config.custom.services.prometheus) + |> lib.filter (prometheus: prometheus.enable) + |> lib.map (prometheus: prometheus.domain); in { options.custom.web-services.grafana = { @@ -19,6 +26,15 @@ in type = lib.types.port; default = 3000; }; + datasources.prometheus = { + enable = lib.mkEnableOption "" // { + default = prometheusDomains != [ ]; + }; + url = lib.mkOption { + type = lib.types.nonEmptyStr; + default = "https://metrics.${config.custom.networking.overlay.fqdn}"; + }; + }; dashboards.nodeExporter.enable = lib.mkEnableOption "" // { default = true; }; @@ -57,22 +73,19 @@ in datasources.settings = { prune = true; - datasources = - allHosts - |> lib.attrValues - |> lib.filter (host: host.config.custom.services.prometheus.enable) - |> lib.map (host: { - name = "Prometheus ${host.config.networking.hostName}"; - type = "prometheus"; - url = "https://${host.config.custom.services.prometheus.domain}"; - isDefault = host.config.networking.hostName == config.networking.hostName; - jsonData = { - prometheusType = "Prometheus"; - prometheusVersion = "3.7.2"; - }; - }); + datasources = lib.optional cfg.datasources.prometheus.enable { + name = "Prometheus"; + type = "prometheus"; + inherit (cfg.datasources.prometheus) url; + isDefault = true; + jsonData = { + prometheusType = "Prometheus"; + prometheusVersion = "3.7.2"; + }; + }; }; }; + }; # https://grafana.com/grafana/dashboards/1860-node-exporter-full/ @@ -85,6 +98,20 @@ in }; }; - custom.services.caddy.virtualHosts.${cfg.domain}.port = cfg.port; + custom.services.caddy = { + virtualHosts.${cfg.domain}.port = cfg.port; + + virtualHosts."metrics.${config.custom.networking.overlay.fqdn}".extraConfig = + let + upstreams = prometheusDomains |> lib.map (domain: "https://${domain}") |> lib.concatStringsSep " "; + in + '' + reverse_proxy ${upstreams} { + header_up Host {upstream_hostport} + lb_policy first + health_uri /api/v1/status/buildinfo + } + ''; + }; }; }