grafana: Ingest metrics from the prometheus instances

modules/nixos/web-services/grafana.nix

@@ -2,10 +2,18 @@
   config,
   pkgs,
   lib,
+  allHosts,
   ...
 }:
 let
   cfg = config.custom.web-services.grafana;
+
+  prometheusDomains =
+    allHosts
+    |> lib.attrValues
+    |> lib.map (host: host.config.custom.services.prometheus)
+    |> lib.filter (prometheus: prometheus.enable)
+    |> lib.map (prometheus: prometheus.domain);
 in
 {
   options.custom.web-services.grafana = {
@@ -18,34 +26,18 @@ in
       type = lib.types.port;
       default = 3000;
     };
-    datasources = {
+    datasources.prometheus = {
-      prometheus = {
       enable = lib.mkEnableOption "" // {
-          default = config.custom.web-services.victoriametrics.enable;
+        default = prometheusDomains != [ ];
       };
       url = lib.mkOption {
         type = lib.types.nonEmptyStr;
-          default = "https://${config.custom.web-services.victoriametrics.domain}";
+        default = "https://metrics.${config.custom.networking.overlay.fqdn}";
       };
     };
-      victoriametrics = {
+    dashboards.nodeExporter.enable = lib.mkEnableOption "" // {
-        enable = lib.mkEnableOption "" // {
-          default = config.custom.web-services.victoriametrics.enable;
-        };
-        url = lib.mkOption {
-          type = lib.types.nonEmptyStr;
-          default = "https://${config.custom.web-services.victoriametrics.domain}";
-        };
-      };
-    };
-    dashboards = {
-      nodeExporter.enable = lib.mkEnableOption "" // {
       default = true;
     };
-      victoriametrics.enable = lib.mkEnableOption "" // {
-        default = config.custom.web-services.victoriametrics.enable;
-      };
-    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -81,40 +73,23 @@ in
 
         datasources.settings = {
           prune = true;
-          datasources = [
+          datasources = lib.optional cfg.datasources.prometheus.enable {
-            (lib.mkIf cfg.datasources.prometheus.enable {
             name = "Prometheus";
             type = "prometheus";
             inherit (cfg.datasources.prometheus) url;
             isDefault = true;
             jsonData = {
               prometheusType = "Prometheus";
-                prometheusVersion = "2.50.0";
+              prometheusVersion = "3.7.2";
+            };
           };
-            })
-            (lib.mkIf cfg.datasources.victoriametrics.enable {
-              name = "VictoriaMetrics";
-              type = "victoriametrics-metrics-datasource";
-              inherit (cfg.datasources.victoriametrics) url;
-              isDefault = false;
-            })
-          ];
         };
       };
 
-      declarativePlugins =
-        let
-          plugins = pkgs.grafanaPlugins;
-        in
-        [
-          (lib.optional cfg.datasources.victoriametrics.enable plugins.victoriametrics-metrics-datasource)
-        ]
-        |> lib.concatLists;
     };
 
-    environment.etc = {
     # https://grafana.com/grafana/dashboards/1860-node-exporter-full/
-      "grafana-dashboards/node-exporter-full.json" = {
+    environment.etc."grafana-dashboards/node-exporter-full.json" = {
       enable = cfg.dashboards.nodeExporter.enable;
       source = pkgs.fetchurl {
         name = "node-exporter-full.json";
@@ -122,24 +97,21 @@ in
         hash = "sha256-EywgxEayjwNIGDvSmA/S56Ld49qrTSbIYFpeEXBJlTs=";
       };
     };
-      # https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/
-      "grafana-dashboards/victoriametrics-single-node-patched.json" = {
-        enable = cfg.dashboards.victoriametrics.enable;
-        source =
-          pkgs.fetchurl {
-            name = "victoriametrics-single-node.json";
-            url = "https://grafana.com/api/dashboards/10229/revisions/41/download";
-            hash = "sha256-mwtah8A2w81WZjf5bUXoTJfS1R9UX+tua2PiDrBKJCQ=";
-          }
-          |> (
-            src:
-            pkgs.runCommand "victoriametrics-single-node-patched.json" { buildInputs = [ pkgs.gnused ]; } ''
-              sed 's/victoriametrics-logs-//g' ${src} > $out
-            ''
-          );
-      };
-    };
 
-    custom.services.caddy.virtualHosts.${cfg.domain}.port = cfg.port;
+    custom.services.caddy = {
+      virtualHosts.${cfg.domain}.port = cfg.port;
+
+      virtualHosts."metrics.${config.custom.networking.overlay.fqdn}".extraConfig =
+        let
+          upstreams = prometheusDomains |> lib.map (domain: "https://${domain}") |> lib.concatStringsSep " ";
+        in
+        ''
+          reverse_proxy ${upstreams} {
+            header_up Host {upstream_hostport}
+            lb_policy first
+            health_uri /api/v1/status/buildinfo
+          }
+        '';
+    };
   };
 }