victoriametrics: Remove module

hosts/vps-monitor/default.nix

@@ -50,11 +50,6 @@
           enable = true;
           domain = "grafana.${privateDomain}";
         };
-
-        victoriametrics = {
-          enable = true;
-          domain = "metrics.${privateDomain}";
-        };
       };
   };
 }

modules/nixos/services/alertmanager.nix

@@ -72,7 +72,10 @@ in
           http.addr = "localhost:${toString cfg.ntfyBridgePort}";
           ntfy = {
             baseurl = "https://ntfy.sh";
-            notification.topic = "splitleaf";
+            notification = {
+              topic = "splitleaf";
+              priority = "default";
+            };
           };
         };
       };

modules/nixos/services/alloy.nix

@@ -1,4 +1,9 @@
-{ config, lib, ... }:
+{
+  config,
+  lib,
+  allHosts,
+  ...
+}:
 let
   cfg = config.custom.services.alloy;
 in
@@ -13,10 +18,6 @@ in
       type = lib.types.port;
       default = 12345;
     };
-    metricsEndpoint = lib.mkOption {
-      type = lib.types.nonEmptyStr;
-      default = "https://metrics.${config.custom.networking.overlay.domain}/prometheus/api/v1/write";
-    };
     collect.metrics = {
       system = lib.mkEnableOption "" // {
         default = true;
@@ -47,19 +48,30 @@ in
 
     environment.etc =
       let
-        isTrue = x: x;
+        anyIsTrue = attrs: attrs |> lib.attrValues |> lib.any lib.id;
-        anyIsTrue = attrs: attrs |> lib.attrValues |> lib.any isTrue;
+
+        prometheusEndpoints =
+          allHosts
+          |> lib.attrValues
+          |> lib.filter (host: host.config.custom.services.prometheus.enable)
+          |> lib.map (host: "https://${host.config.custom.services.prometheus.domain}/api/v1/write");
       in
       {
         "alloy/metrics-endpoint.alloy" = {
           enable = cfg.collect.metrics |> anyIsTrue;
-          text = ''
+          text =
-            prometheus.remote_write "default" {
+            prometheusEndpoints
+            |> lib.map (url: ''
               endpoint {
-                url = "${cfg.metricsEndpoint}"
+                url = "${url}"
               }
+            '')
+            |> lib.concatLines
+            |> (endpoints: ''
+              prometheus.remote_write "default" {
+                ${endpoints}
               }
-          '';
+            '');
         };
         "alloy/system-metrics.alloy" = {
           enable = cfg.collect.metrics.system;
@@ -85,7 +97,7 @@ in
                 instance    = constants.hostname,
               }]
               forward_to      = [prometheus.remote_write.default.receiver]
-              scrape_interval = "15s"
+              scrape_interval = "30s"
             }
           '';
         };

modules/nixos/services/prometheus.nix

@@ -30,6 +30,7 @@ in
       inherit (cfg) port;
       webExternalUrl = "https://${cfg.domain}";
 
+      extraFlags = [ "--web.enable-remote-write-receiver" ];
       globalConfig = {
         scrape_interval = "30s";
         external_labels.monitor = "global";

modules/nixos/web-services/grafana.nix

@@ -2,10 +2,18 @@
   config,
   pkgs,
   lib,
+  allHosts,
   ...
 }:
 let
   cfg = config.custom.web-services.grafana;
+
+  prometheusDomains =
+    allHosts
+    |> lib.attrValues
+    |> lib.map (host: host.config.custom.services.prometheus)
+    |> lib.filter (prometheus: prometheus.enable)
+    |> lib.map (prometheus: prometheus.domain);
 in
 {
   options.custom.web-services.grafana = {
@@ -18,34 +26,18 @@ in
       type = lib.types.port;
       default = 3000;
     };
-    datasources = {
+    datasources.prometheus = {
-      prometheus = {
       enable = lib.mkEnableOption "" // {
-          default = config.custom.web-services.victoriametrics.enable;
+        default = prometheusDomains != [ ];
       };
       url = lib.mkOption {
         type = lib.types.nonEmptyStr;
-          default = "https://${config.custom.web-services.victoriametrics.domain}";
+        default = "https://metrics.${config.custom.networking.overlay.fqdn}";
       };
     };
-      victoriametrics = {
+    dashboards.nodeExporter.enable = lib.mkEnableOption "" // {
-        enable = lib.mkEnableOption "" // {
-          default = config.custom.web-services.victoriametrics.enable;
-        };
-        url = lib.mkOption {
-          type = lib.types.nonEmptyStr;
-          default = "https://${config.custom.web-services.victoriametrics.domain}";
-        };
-      };
-    };
-    dashboards = {
-      nodeExporter.enable = lib.mkEnableOption "" // {
       default = true;
     };
-      victoriametrics.enable = lib.mkEnableOption "" // {
-        default = config.custom.web-services.victoriametrics.enable;
-      };
-    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -81,40 +73,23 @@ in
 
         datasources.settings = {
           prune = true;
-          datasources = [
+          datasources = lib.optional cfg.datasources.prometheus.enable {
-            (lib.mkIf cfg.datasources.prometheus.enable {
             name = "Prometheus";
             type = "prometheus";
             inherit (cfg.datasources.prometheus) url;
             isDefault = true;
             jsonData = {
               prometheusType = "Prometheus";
-                prometheusVersion = "2.50.0";
+              prometheusVersion = "3.7.2";
+            };
           };
-            })
-            (lib.mkIf cfg.datasources.victoriametrics.enable {
-              name = "VictoriaMetrics";
-              type = "victoriametrics-metrics-datasource";
-              inherit (cfg.datasources.victoriametrics) url;
-              isDefault = false;
-            })
-          ];
         };
       };
 
-      declarativePlugins =
-        let
-          plugins = pkgs.grafanaPlugins;
-        in
-        [
-          (lib.optional cfg.datasources.victoriametrics.enable plugins.victoriametrics-metrics-datasource)
-        ]
-        |> lib.concatLists;
     };
 
-    environment.etc = {
     # https://grafana.com/grafana/dashboards/1860-node-exporter-full/
-      "grafana-dashboards/node-exporter-full.json" = {
+    environment.etc."grafana-dashboards/node-exporter-full.json" = {
       enable = cfg.dashboards.nodeExporter.enable;
       source = pkgs.fetchurl {
         name = "node-exporter-full.json";
@@ -122,24 +97,21 @@ in
         hash = "sha256-EywgxEayjwNIGDvSmA/S56Ld49qrTSbIYFpeEXBJlTs=";
       };
     };
-      # https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/
-      "grafana-dashboards/victoriametrics-single-node-patched.json" = {
-        enable = cfg.dashboards.victoriametrics.enable;
-        source =
-          pkgs.fetchurl {
-            name = "victoriametrics-single-node.json";
-            url = "https://grafana.com/api/dashboards/10229/revisions/41/download";
-            hash = "sha256-mwtah8A2w81WZjf5bUXoTJfS1R9UX+tua2PiDrBKJCQ=";
-          }
-          |> (
-            src:
-            pkgs.runCommand "victoriametrics-single-node-patched.json" { buildInputs = [ pkgs.gnused ]; } ''
-              sed 's/victoriametrics-logs-//g' ${src} > $out
-            ''
-          );
-      };
-    };
 
-    custom.services.caddy.virtualHosts.${cfg.domain}.port = cfg.port;
+    custom.services.caddy = {
+      virtualHosts.${cfg.domain}.port = cfg.port;
+
+      virtualHosts."metrics.${config.custom.networking.overlay.fqdn}".extraConfig =
+        let
+          upstreams = prometheusDomains |> lib.map (domain: "https://${domain}") |> lib.concatStringsSep " ";
+        in
+        ''
+          reverse_proxy ${upstreams} {
+            header_up Host {upstream_hostport}
+            lb_policy first
+            health_uri /api/v1/status/buildinfo
+          }
+        '';
+    };
   };
 }

modules/nixos/web-services/victoriametrics.nix

@@ -1,49 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.web-services.victoriametrics;
-in
-{
-  options.custom.web-services.victoriametrics = {
-    enable = lib.mkEnableOption "";
-    domain = lib.mkOption {
-      type = lib.types.nonEmptyStr;
-      default = "";
-    };
-    port = lib.mkOption {
-      type = lib.types.port;
-      default = 8428;
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    users = {
-      users.victoriametrics = {
-        isSystemUser = true;
-        group = config.users.groups.victoriametrics.name;
-      };
-      groups.victoriametrics = { };
-    };
-
-    systemd.services.victoriametrics.serviceConfig = {
-      DynamicUser = lib.mkForce false;
-      User = config.users.users.victoriametrics.name;
-      Group = config.users.groups.victoriametrics.name;
-    };
-
-    services.victoriametrics = {
-      enable = true;
-      listenAddress = "localhost:${toString cfg.port}";
-      extraOptions = [
-        "-selfScrapeInterval=15s"
-        "-selfScrapeJob=victoriametrics"
-        "-selfScrapeInstance=${config.networking.hostName}"
-      ];
-    };
-
-    custom = {
-      services.caddy.virtualHosts.${cfg.domain}.port = cfg.port;
-
-      persistence.directories = [ "/var/lib/${config.services.victoriametrics.stateDir}" ];
-    };
-  };
-}