hosts/vps-monitor/default.nix

@@ -50,6 +50,11 @@
           enable = true;
           domain = "grafana.${privateDomain}";
         };
+
+        victoriametrics = {
+          enable = true;
+          domain = "metrics.${privateDomain}";
+        };
       };
   };
 }

modules/nixos/services/alertmanager.nix

@@ -72,10 +72,7 @@ in
           http.addr = "localhost:${toString cfg.ntfyBridgePort}";
           ntfy = {
             baseurl = "https://ntfy.sh";
-            notification = {
-              topic = "splitleaf";
-              priority = "default";
-            };
+            notification.topic = "splitleaf";
           };
         };
       };

modules/nixos/services/alloy.nix

@@ -1,9 +1,4 @@
-{
-  config,
-  lib,
-  allHosts,
-  ...
-}:
+{ config, lib, ... }:
 let
   cfg = config.custom.services.alloy;
 in
@@ -18,6 +13,10 @@ in
       type = lib.types.port;
       default = 12345;
     };
+    metricsEndpoint = lib.mkOption {
+      type = lib.types.nonEmptyStr;
+      default = "https://metrics.${config.custom.networking.overlay.domain}/prometheus/api/v1/write";
+    };
     collect.metrics = {
       system = lib.mkEnableOption "" // {
         default = true;
@@ -48,30 +47,19 @@ in
 
     environment.etc =
       let
-        anyIsTrue = attrs: attrs |> lib.attrValues |> lib.any lib.id;
-
-        prometheusEndpoints =
-          allHosts
-          |> lib.attrValues
-          |> lib.filter (host: host.config.custom.services.prometheus.enable)
-          |> lib.map (host: "https://${host.config.custom.services.prometheus.domain}/api/v1/write");
+        isTrue = x: x;
+        anyIsTrue = attrs: attrs |> lib.attrValues |> lib.any isTrue;
       in
       {
         "alloy/metrics-endpoint.alloy" = {
           enable = cfg.collect.metrics |> anyIsTrue;
-          text =
-            prometheusEndpoints
-            |> lib.map (url: ''
+          text = ''
+            prometheus.remote_write "default" {
               endpoint {
-                url = "${url}"
+                url = "${cfg.metricsEndpoint}"
               }
-            '')
-            |> lib.concatLines
-            |> (endpoints: ''
-              prometheus.remote_write "default" {
-                ${endpoints}
-              }
-            '');
+            }
+          '';
         };
         "alloy/system-metrics.alloy" = {
           enable = cfg.collect.metrics.system;
@@ -97,7 +85,7 @@ in
                 instance    = constants.hostname,
               }]
               forward_to      = [prometheus.remote_write.default.receiver]
-              scrape_interval = "30s"
+              scrape_interval = "15s"
             }
           '';
         };

modules/nixos/services/prometheus.nix

@@ -30,7 +30,6 @@ in
       inherit (cfg) port;
       webExternalUrl = "https://${cfg.domain}";
 
-      extraFlags = [ "--web.enable-remote-write-receiver" ];
       globalConfig = {
         scrape_interval = "30s";
         external_labels.monitor = "global";

modules/nixos/web-services/grafana.nix

@@ -2,18 +2,10 @@
   config,
   pkgs,
   lib,
-  allHosts,
   ...
 }:
 let
   cfg = config.custom.web-services.grafana;
-
-  prometheusDomains =
-    allHosts
-    |> lib.attrValues
-    |> lib.map (host: host.config.custom.services.prometheus)
-    |> lib.filter (prometheus: prometheus.enable)
-    |> lib.map (prometheus: prometheus.domain);
 in
 {
   options.custom.web-services.grafana = {
@@ -26,17 +18,33 @@ in
       type = lib.types.port;
       default = 3000;
     };
-    datasources.prometheus = {
-      enable = lib.mkEnableOption "" // {
-        default = prometheusDomains != [ ];
+    datasources = {
+      prometheus = {
+        enable = lib.mkEnableOption "" // {
+          default = config.custom.web-services.victoriametrics.enable;
+        };
+        url = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          default = "https://${config.custom.web-services.victoriametrics.domain}";
+        };
       };
-      url = lib.mkOption {
-        type = lib.types.nonEmptyStr;
-        default = "https://metrics.${config.custom.networking.overlay.fqdn}";
+      victoriametrics = {
+        enable = lib.mkEnableOption "" // {
+          default = config.custom.web-services.victoriametrics.enable;
+        };
+        url = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          default = "https://${config.custom.web-services.victoriametrics.domain}";
+        };
       };
     };
-    dashboards.nodeExporter.enable = lib.mkEnableOption "" // {
-      default = true;
+    dashboards = {
+      nodeExporter.enable = lib.mkEnableOption "" // {
+        default = true;
+      };
+      victoriametrics.enable = lib.mkEnableOption "" // {
+        default = config.custom.web-services.victoriametrics.enable;
+      };
     };
   };
 
@@ -73,45 +81,65 @@ in
 
         datasources.settings = {
           prune = true;
-          datasources = lib.optional cfg.datasources.prometheus.enable {
-            name = "Prometheus";
-            type = "prometheus";
-            inherit (cfg.datasources.prometheus) url;
-            isDefault = true;
-            jsonData = {
-              prometheusType = "Prometheus";
-              prometheusVersion = "3.7.2";
-            };
-          };
+          datasources = [
+            (lib.mkIf cfg.datasources.prometheus.enable {
+              name = "Prometheus";
+              type = "prometheus";
+              inherit (cfg.datasources.prometheus) url;
+              isDefault = true;
+              jsonData = {
+                prometheusType = "Prometheus";
+                prometheusVersion = "2.50.0";
+              };
+            })
+            (lib.mkIf cfg.datasources.victoriametrics.enable {
+              name = "VictoriaMetrics";
+              type = "victoriametrics-metrics-datasource";
+              inherit (cfg.datasources.victoriametrics) url;
+              isDefault = false;
+            })
+          ];
         };
       };
 
+      declarativePlugins =
+        let
+          plugins = pkgs.grafanaPlugins;
+        in
+        [
+          (lib.optional cfg.datasources.victoriametrics.enable plugins.victoriametrics-metrics-datasource)
+        ]
+        |> lib.concatLists;
     };
 
-    # https://grafana.com/grafana/dashboards/1860-node-exporter-full/
-    environment.etc."grafana-dashboards/node-exporter-full.json" = {
-      enable = cfg.dashboards.nodeExporter.enable;
-      source = pkgs.fetchurl {
-        name = "node-exporter-full.json";
-        url = "https://grafana.com/api/dashboards/1860/revisions/41/download";
-        hash = "sha256-EywgxEayjwNIGDvSmA/S56Ld49qrTSbIYFpeEXBJlTs=";
+    environment.etc = {
+      # https://grafana.com/grafana/dashboards/1860-node-exporter-full/
+      "grafana-dashboards/node-exporter-full.json" = {
+        enable = cfg.dashboards.nodeExporter.enable;
+        source = pkgs.fetchurl {
+          name = "node-exporter-full.json";
+          url = "https://grafana.com/api/dashboards/1860/revisions/41/download";
+          hash = "sha256-EywgxEayjwNIGDvSmA/S56Ld49qrTSbIYFpeEXBJlTs=";
+        };
+      };
+      # https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/
+      "grafana-dashboards/victoriametrics-single-node-patched.json" = {
+        enable = cfg.dashboards.victoriametrics.enable;
+        source =
+          pkgs.fetchurl {
+            name = "victoriametrics-single-node.json";
+            url = "https://grafana.com/api/dashboards/10229/revisions/41/download";
+            hash = "sha256-mwtah8A2w81WZjf5bUXoTJfS1R9UX+tua2PiDrBKJCQ=";
+          }
+          |> (
+            src:
+            pkgs.runCommand "victoriametrics-single-node-patched.json" { buildInputs = [ pkgs.gnused ]; } ''
+              sed 's/victoriametrics-logs-//g' ${src} > $out
+            ''
+          );
       };
     };
 
-    custom.services.caddy = {
-      virtualHosts.${cfg.domain}.port = cfg.port;
-
-      virtualHosts."metrics.${config.custom.networking.overlay.fqdn}".extraConfig =
-        let
-          upstreams = prometheusDomains |> lib.map (domain: "https://${domain}") |> lib.concatStringsSep " ";
-        in
-        ''
-          reverse_proxy ${upstreams} {
-            header_up Host {upstream_hostport}
-            lb_policy first
-            health_uri /api/v1/status/buildinfo
-          }
-        '';
-    };
+    custom.services.caddy.virtualHosts.${cfg.domain}.port = cfg.port;
   };
 }

modules/nixos/web-services/victoriametrics.nix

@@ -0,0 +1,49 @@
+{ config, lib, ... }:
+let
+  cfg = config.custom.web-services.victoriametrics;
+in
+{
+  options.custom.web-services.victoriametrics = {
+    enable = lib.mkEnableOption "";
+    domain = lib.mkOption {
+      type = lib.types.nonEmptyStr;
+      default = "";
+    };
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 8428;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users = {
+      users.victoriametrics = {
+        isSystemUser = true;
+        group = config.users.groups.victoriametrics.name;
+      };
+      groups.victoriametrics = { };
+    };
+
+    systemd.services.victoriametrics.serviceConfig = {
+      DynamicUser = lib.mkForce false;
+      User = config.users.users.victoriametrics.name;
+      Group = config.users.groups.victoriametrics.name;
+    };
+
+    services.victoriametrics = {
+      enable = true;
+      listenAddress = "localhost:${toString cfg.port}";
+      extraOptions = [
+        "-selfScrapeInterval=15s"
+        "-selfScrapeJob=victoriametrics"
+        "-selfScrapeInstance=${config.networking.hostName}"
+      ];
+    };
+
+    custom = {
+      services.caddy.virtualHosts.${cfg.domain}.port = cfg.port;
+
+      persistence.directories = [ "/var/lib/${config.services.victoriametrics.stateDir}" ];
+    };
+  };
+}