From 0e643d568bc7d6a8e01048fcebacdba308e12c72 Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Sun, 1 Mar 2026 13:58:58 +0100 Subject: [PATCH 1/6] alloy: Improve naming --- modules/nixos/services/alloy.nix | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/nixos/services/alloy.nix b/modules/nixos/services/alloy.nix index 8c270c1..5abfd2c 100644 --- a/modules/nixos/services/alloy.nix +++ b/modules/nixos/services/alloy.nix @@ -57,7 +57,7 @@ in |> lib.map (host: "https://${host.config.custom.services.prometheus.domain}/api/v1/write"); in { - "alloy/metrics-endpoint.alloy" = { + "alloy/prometheus-endpoint.alloy" = { enable = cfg.collect.metrics |> anyIsTrue; text = prometheusEndpoints @@ -80,9 +80,8 @@ in enable_collectors = ["systemd", "processes"] } - discovery.relabel "unix_exporter" { + discovery.relabel "node_exporter" { targets = prometheus.exporter.unix.default.targets - rule { target_label = "job" replacement = "node" @@ -90,7 +89,7 @@ in } prometheus.scrape "node_exporter" { - targets = discovery.relabel.unix_exporter.output + targets = discovery.relabel.node_exporter.output forward_to = [prometheus.remote_write.default.receiver] scrape_interval = "15s" } From 45293668414ee433519dffeee646ff84d4750c5a Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Sun, 1 Mar 2026 14:00:52 +0100 Subject: [PATCH 2/6] grafana: Remove prometheus load balancer and offer all instance as possible datasources --- modules/nixos/web-services/grafana.nix | 57 +++++++------------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/modules/nixos/web-services/grafana.nix b/modules/nixos/web-services/grafana.nix index 959336a..96913ec 100644 --- a/modules/nixos/web-services/grafana.nix +++ b/modules/nixos/web-services/grafana.nix @@ -7,13 +7,6 @@ }: let cfg = config.custom.web-services.grafana; - - prometheusDomains = - allHosts - |> lib.attrValues - |> lib.map (host: host.config.custom.services.prometheus) - |> lib.filter (prometheus: prometheus.enable) - |> lib.map (prometheus: prometheus.domain); in { options.custom.web-services.grafana = { @@ -26,15 +19,6 @@ in type = lib.types.port; default = 3000; }; - datasources.prometheus = { - enable = lib.mkEnableOption "" // { - default = prometheusDomains != [ ]; - }; - url = lib.mkOption { - type = lib.types.nonEmptyStr; - default = "https://metrics.${config.custom.networking.overlay.fqdn}"; - }; - }; dashboards.nodeExporter.enable = lib.mkEnableOption "" // { default = true; }; @@ -73,19 +57,22 @@ in datasources.settings = { prune = true; - datasources = lib.optional cfg.datasources.prometheus.enable { - name = "Prometheus"; - type = "prometheus"; - inherit (cfg.datasources.prometheus) url; - isDefault = true; - jsonData = { - prometheusType = "Prometheus"; - prometheusVersion = "3.7.2"; - }; - }; + datasources = + allHosts + |> lib.attrValues + |> lib.filter (host: host.config.custom.services.prometheus.enable) + |> lib.map (host: { + name = "Prometheus ${host.config.networking.hostName}"; + type = "prometheus"; + url = "https://${host.config.custom.services.prometheus.domain}"; + isDefault = host.config.networking.hostName == config.networking.hostName; + jsonData = { + prometheusType = "Prometheus"; + prometheusVersion = "3.7.2"; + }; + }); }; }; - }; # https://grafana.com/grafana/dashboards/1860-node-exporter-full/ @@ -98,20 +85,6 @@ in }; }; - custom.services.caddy = { - virtualHosts.${cfg.domain}.port = cfg.port; - - virtualHosts."metrics.${config.custom.networking.overlay.fqdn}".extraConfig = - let - upstreams = prometheusDomains |> lib.map (domain: "https://${domain}") |> lib.concatStringsSep " "; - in - '' - reverse_proxy ${upstreams} { - header_up Host {upstream_hostport} - lb_policy first - health_uri /api/v1/status/buildinfo - } - ''; - }; + custom.services.caddy.virtualHosts.${cfg.domain}.port = cfg.port; }; } From 55d5da375af95372f7706b20e164584cae334b3c Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Sun, 1 Mar 2026 14:07:18 +0100 Subject: [PATCH 3/6] prometheus: Add option `storageRetentionSize` --- modules/nixos/services/prometheus.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/nixos/services/prometheus.nix b/modules/nixos/services/prometheus.nix index 0f0fb58..182a70c 100644 --- a/modules/nixos/services/prometheus.nix +++ b/modules/nixos/services/prometheus.nix @@ -19,6 +19,10 @@ in type = lib.types.port; default = 9090; }; + storageRetentionSize = lib.mkOption { + type = lib.types.nonEmptyStr; + default = "2GB"; + }; }; config = lib.mkIf cfg.enable { @@ -30,7 +34,10 @@ in inherit (cfg) port; webExternalUrl = "https://${cfg.domain}"; - extraFlags = [ "--web.enable-remote-write-receiver" ]; + extraFlags = [ + "--web.enable-remote-write-receiver" + "--storage.tsdb.retention.size=${cfg.storageRetentionSize}" + ]; globalConfig = { scrape_interval = "30s"; external_labels.monitor = "global"; From 18e89c466540383f78a4ce83ff43d767ece41116 Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Sun, 1 Mar 2026 14:08:59 +0100 Subject: [PATCH 4/6] Host grafana on homeserver instead of vps-monitor --- hosts/homeserver/default.nix | 6 ++++++ hosts/vps-monitor/default.nix | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/hosts/homeserver/default.nix b/hosts/homeserver/default.nix index e6c18bd..bd870dc 100644 --- a/hosts/homeserver/default.nix +++ b/hosts/homeserver/default.nix @@ -59,6 +59,12 @@ enable = true; domain = "bookmarks.${privateDomain}"; }; + + grafana = { + enable = true; + domain = "grafana.${privateDomain}"; + storageRetentionSize = "20GB"; + }; }; }; } diff --git a/hosts/vps-monitor/default.nix b/hosts/vps-monitor/default.nix index 109cdff..d45cc23 100644 --- a/hosts/vps-monitor/default.nix +++ b/hosts/vps-monitor/default.nix @@ -45,11 +45,6 @@ enable = true; domain = "alerts.${sproutedDomain}"; }; - - grafana = { - enable = true; - domain = "grafana.${privateDomain}"; - }; }; }; } From 21e99f135382e4966f09d2c5d908ff9bd7990565 Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Sun, 1 Mar 2026 14:14:50 +0100 Subject: [PATCH 5/6] homeserver: Fix misplaced option and missing secret --- hosts/homeserver/default.nix | 3 ++- hosts/homeserver/secrets.json | 7 +++++-- hosts/vps-monitor/secrets.json | 7 ++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/hosts/homeserver/default.nix b/hosts/homeserver/default.nix index bd870dc..26ed9ca 100644 --- a/hosts/homeserver/default.nix +++ b/hosts/homeserver/default.nix @@ -29,6 +29,8 @@ gui.domain = "syncthing.${privateDomain}"; doBackups = true; }; + + prometheus.storageRetentionSize = "20GB"; }; web-services = { @@ -63,7 +65,6 @@ grafana = { enable = true; domain = "grafana.${privateDomain}"; - storageRetentionSize = "20GB"; }; }; }; diff --git a/hosts/homeserver/secrets.json b/hosts/homeserver/secrets.json index b0c9251..22e54b2 100644 --- a/hosts/homeserver/secrets.json +++ b/hosts/homeserver/secrets.json @@ -27,6 +27,9 @@ "radicale": { "htpasswd": "ENC[AES256_GCM,data:SuVrYk3BP3Hds9sKwiuEOgAA5RPBR3f3RW16xtpO6w9aUjZR37jPKPOPluHxBUIulap2p/Oj5VZvyyeDP2MxQw==,iv:sANgLVPRrZEjlC7n/r5zVma/qIDCraLxi88o/sVgayQ=,tag:KbUM7hTRM0Z9iiiQFUfp/w==,type:str]" }, + "grafana": { + "admin-password": "ENC[AES256_GCM,data:MNdjh+025bi5wtP77aKSGzcx8NgfY+YppH4qu/o=,iv:Hv8IF0n0A5+Hs6FQ7tXkdFbPN0ArdZD3vmrdovc0/Yg=,tag:8y0NsLQ+9k+mWnHbHzZvGg==,type:str]" + }, "sops": { "age": [ { @@ -38,8 +41,8 @@ "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXOVRQU1R4VlBhdEFUdnR6\nZFNuNldaSkxJL3ptOVVscjRBNkQ4dFBmQUVBCmZrQmFMV0hWbTBQcm1FS3JrR0ZC\nbktvT04xczd6VkdCUWk2NnVVZHNFWkUKLS0tIGUwOHJSMHVsNTEyZEU2VWJFNGVy\nMVFDVThrRGQwZEtPeFYzZUVQYi80ZjAKUd/XzyzqMkMowvyeCnQDbOGJDKbuAUQb\nFClQuiH5iSQQrVPw7SHBNgdqbcdtC+hZ4tpPaV/wWtlpcqpr5mBJSA==\n-----END AGE ENCRYPTED FILE-----\n" } ], - "lastmodified": "2026-02-27T21:25:33Z", - "mac": "ENC[AES256_GCM,data:kvP/sCIqs4Ic/IwEn1FwZOc71N6zAuyn7NDQWF78EW9HFfPjqUwN3z5PJOmEc1U74KbJB1gHIFMfteLx547IhGHn45oHlju7gco8jCdFhX9XL40Hp72B2px3MaG8UXp2DmDel9n7wQYCjNK1W6aOcQnpids/5uZu2gb/sV4/IIM=,iv:mlgEoRieeL/qc9rTZ8tYFrTdOkqph8GdAe2zJydjMIU=,tag:7czP2+PRO/RXZaBanW12tg==,type:str]", + "lastmodified": "2026-03-01T13:14:08Z", + "mac": "ENC[AES256_GCM,data:GywNeQDso210CV1rEY1LCwUu5ZDdjyP/W+QOJ5GJhLnLiF3BjivS4J9VbOANduhepnRLLHFKW0+zdzPJJl3ltBFpAuoDigEPF5PbXOT6R7oiEW3kkgD5LP79789ijkeGqGf2DUN0J04OFkcTnTy2tAd2O3kSnhi/EbIMvw7zjUY=,iv:TqXETQOQuuQCJ2+tvPeXHY08BGbhQ4fsu+RqX1EiPHY=,tag:qfaYvY/iuu0Ou9AxhJUnRQ==,type:str]", "unencrypted_suffix": "_unencrypted", "version": "3.11.0" } diff --git a/hosts/vps-monitor/secrets.json b/hosts/vps-monitor/secrets.json index fa99e10..7e95241 100644 --- a/hosts/vps-monitor/secrets.json +++ b/hosts/vps-monitor/secrets.json @@ -3,9 +3,6 @@ "healthchecks": { "ping-key": "ENC[AES256_GCM,data:Zq71AU3oym7fC364YZNyRtx4N2G35Q==,iv:ibMBpcrSocLBhtumsSV00+KVN6Pi4SzE7soCkZcU4fY=,tag:Wv/Wr0wRZGXucMHZHgoNtg==,type:str]" }, - "grafana": { - "admin-password": "ENC[AES256_GCM,data:2YRh4DT+1w5W/X3ELIe3Uu2EnMIHG4gUhV5ri6E=,iv:owHyuoupNQO09aRBgU2phIwxg22U1rUqKyYbw2193m4=,tag:dopVQwf4Ewf+lsFterfDOA==,type:str]" - }, "nebula": { "host-key": "ENC[AES256_GCM,data:usSLqYOvDAAs7z1xo+gccDqgUE78upK+k522ldKcPoFKKBH87Us7gi6+XAOMDQ79U6i8j4l1lAE8kRdqDuvasodESHVSW9gSnnv5E73MVr0d1Snh7tAewVzneac+2R2R8tUzKzwzWM5SyyvJSoKGBg8WmGzdGT8UqC623utlYQ==,iv:NoZ2u8IK4g1Kwb6uZZ1jXJH4eFO9Jj5Phi5hPM4K72o=,tag:9mOv6oSESH+8r2ZC4yUE+w==,type:str]" }, @@ -24,8 +21,8 @@ "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTVmV4dkZEaWVDNHMxUFdr\nYnhuVWpReXBNSEhhZkltQTE4bEpzSlBzL0VJCk15UFlwa0haWTZNaE1DVzVZVFBI\nd0QzcUptYmQ3dmhhdjhFV0xDSWdmMGMKLS0tIFhWamx6SXJleFFSVUFkRmw2VFZy\nOVVhNm9NSE0yRGFMQjNrM1B6cDVxSXMKrhAkDcWqutgSmQI5O+5i8fcwuTh2/XKr\nljK/Vn8EvGr8qEUeHzOBI1b5VtgngJkVJyfM9G/Q0lZvQF7ZZ5YCgQ==\n-----END AGE ENCRYPTED FILE-----\n" } ], - "lastmodified": "2026-01-10T23:14:34Z", - "mac": "ENC[AES256_GCM,data:RpG1S1iuaHmBc9wT9Tau6dZHcFX9DwQA0kHcAuTZxbjopLfYJnVosvU9S9N+4tAZqwGLpKqVxX3UiUWLEIFwrcB3xM3/SdpKWD4CE/cyMbvwWCIN4RcHveFJ6py7ItXtipzLC0lZeRWDXP+IyREuKggAID50zXuSIPr6BDlnkYA=,iv:gWykL+vhNB3tLHtHlhbNd5iUPKjB5vDpx0nAf5j+F8Y=,tag:MAI8aaSR+g33IIYM4opqCw==,type:str]", + "lastmodified": "2026-03-01T13:13:54Z", + "mac": "ENC[AES256_GCM,data:/jWzYWYyHvFkjRN2jphn1ocPF8AsrULMa8W4U4GKPqdawT3VchEbJl0pKQV/bW5ApJkjHkopWmJjyScpkQXEXY2up6arNsN2IqB89c6jodMQihSKvSdG8pfeeZaFXY1HPL0KNUS/zV5y64rUNX5qv/t9wL9qatcIKP1SDw9Khro=,iv:hPoyKPQwbv6sbWCb0MMAWigDBKqnb/OmLcQN3UGg6yk=,tag:VDO8di4fLBh9XCD24UuvuA==,type:str]", "unencrypted_suffix": "_unencrypted", "version": "3.11.0" } From b80fc49dee26adf786c693a212fae7a65fbe6ade Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Sun, 1 Mar 2026 14:33:03 +0100 Subject: [PATCH 6/6] vps-monitor: Disable gatus notifications --- hosts/vps-monitor/default.nix | 13 +------------ modules/nixos/web-services/gatus.nix | 13 ++++++++----- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/hosts/vps-monitor/default.nix b/hosts/vps-monitor/default.nix index d45cc23..e45880b 100644 --- a/hosts/vps-monitor/default.nix +++ b/hosts/vps-monitor/default.nix @@ -32,18 +32,7 @@ enable = true; domain = "status.${privateDomain}"; generateDefaultEndpoints = true; - endpoints = { - "dav.${sproutedDomain}".enable = false; - "alerts.${sproutedDomain}" = { - path = "/v1/health"; - extraConditions = [ "[BODY].healthy == true" ]; - }; - }; - }; - - ntfy = { - enable = true; - domain = "alerts.${sproutedDomain}"; + endpoints."dav.${sproutedDomain}".enable = false; }; }; }; diff --git a/modules/nixos/web-services/gatus.nix b/modules/nixos/web-services/gatus.nix index c47da79..1582efe 100644 --- a/modules/nixos/web-services/gatus.nix +++ b/modules/nixos/web-services/gatus.nix @@ -19,9 +19,12 @@ in type = lib.types.port; default = 8080; }; - ntfyUrl = lib.mkOption { - type = lib.types.nonEmptyStr; - default = "https://${config.custom.web-services.ntfy.domain}"; + alerts = { + enable = lib.mkEnableOption ""; + ntfyUrl = lib.mkOption { + type = lib.types.nonEmptyStr; + default = "https://${config.custom.web-services.ntfy.domain}"; + }; }; generateDefaultEndpoints = lib.mkEnableOption ""; endpoints = lib.mkOption { @@ -62,7 +65,7 @@ in default = [ ]; }; enableAlerts = lib.mkEnableOption "" // { - default = true; + default = cfg.alerts.enable; }; }; } @@ -112,7 +115,7 @@ in path = "${dataDir}/data.db"; }; connectivity.checker.target = "1.1.1.1:53"; # Cloudflare DNS - alerting.ntfy = { + alerting.ntfy = lib.mkIf cfg.alerts.enable { topic = "uptime"; url = cfg.ntfyUrl; click = "https://${cfg.domain}";