diff --git a/hosts/vps-monitor/default.nix b/hosts/vps-monitor/default.nix index 1cc36c3..109cdff 100644 --- a/hosts/vps-monitor/default.nix +++ b/hosts/vps-monitor/default.nix @@ -50,11 +50,6 @@ enable = true; domain = "grafana.${privateDomain}"; }; - - victoriametrics = { - enable = true; - domain = "metrics.${privateDomain}"; - }; }; }; } diff --git a/modules/nixos/services/alertmanager.nix b/modules/nixos/services/alertmanager.nix index 6e4f103..295e2e7 100644 --- a/modules/nixos/services/alertmanager.nix +++ b/modules/nixos/services/alertmanager.nix @@ -72,7 +72,10 @@ in http.addr = "localhost:${toString cfg.ntfyBridgePort}"; ntfy = { baseurl = "https://ntfy.sh"; - notification.topic = "splitleaf"; + notification = { + topic = "splitleaf"; + priority = "default"; + }; }; }; }; diff --git a/modules/nixos/services/alloy.nix b/modules/nixos/services/alloy.nix index 3bed041..5330db7 100644 --- a/modules/nixos/services/alloy.nix +++ b/modules/nixos/services/alloy.nix @@ -1,4 +1,9 @@ -{ config, lib, ... }: +{ + config, + lib, + allHosts, + ... +}: let cfg = config.custom.services.alloy; in @@ -13,10 +18,6 @@ in type = lib.types.port; default = 12345; }; - metricsEndpoint = lib.mkOption { - type = lib.types.nonEmptyStr; - default = "https://metrics.${config.custom.networking.overlay.domain}/prometheus/api/v1/write"; - }; collect.metrics = { system = lib.mkEnableOption "" // { default = true; @@ -47,19 +48,30 @@ in environment.etc = let - isTrue = x: x; - anyIsTrue = attrs: attrs |> lib.attrValues |> lib.any isTrue; + anyIsTrue = attrs: attrs |> lib.attrValues |> lib.any lib.id; + + prometheusEndpoints = + allHosts + |> lib.attrValues + |> lib.filter (host: host.config.custom.services.prometheus.enable) + |> lib.map (host: "https://${host.config.custom.services.prometheus.domain}/api/v1/write"); in { "alloy/metrics-endpoint.alloy" = { enable = cfg.collect.metrics |> anyIsTrue; - text = '' - prometheus.remote_write "default" { + text = + prometheusEndpoints + |> lib.map (url: '' endpoint { - url = "${cfg.metricsEndpoint}" + url = "${url}" } - } - ''; + '') + |> lib.concatLines + |> (endpoints: '' + prometheus.remote_write "default" { + ${endpoints} + } + ''); }; "alloy/system-metrics.alloy" = { enable = cfg.collect.metrics.system; @@ -85,7 +97,7 @@ in instance = constants.hostname, }] forward_to = [prometheus.remote_write.default.receiver] - scrape_interval = "15s" + scrape_interval = "30s" } ''; }; diff --git a/modules/nixos/services/prometheus.nix b/modules/nixos/services/prometheus.nix index 7e2cc16..0f0fb58 100644 --- a/modules/nixos/services/prometheus.nix +++ b/modules/nixos/services/prometheus.nix @@ -30,6 +30,7 @@ in inherit (cfg) port; webExternalUrl = "https://${cfg.domain}"; + extraFlags = [ "--web.enable-remote-write-receiver" ]; globalConfig = { scrape_interval = "30s"; external_labels.monitor = "global"; diff --git a/modules/nixos/web-services/grafana.nix b/modules/nixos/web-services/grafana.nix index 60f4134..959336a 100644 --- a/modules/nixos/web-services/grafana.nix +++ b/modules/nixos/web-services/grafana.nix @@ -2,10 +2,18 @@ config, pkgs, lib, + allHosts, ... }: let cfg = config.custom.web-services.grafana; + + prometheusDomains = + allHosts + |> lib.attrValues + |> lib.map (host: host.config.custom.services.prometheus) + |> lib.filter (prometheus: prometheus.enable) + |> lib.map (prometheus: prometheus.domain); in { options.custom.web-services.grafana = { @@ -18,33 +26,17 @@ in type = lib.types.port; default = 3000; }; - datasources = { - prometheus = { - enable = lib.mkEnableOption "" // { - default = config.custom.web-services.victoriametrics.enable; - }; - url = lib.mkOption { - type = lib.types.nonEmptyStr; - default = "https://${config.custom.web-services.victoriametrics.domain}"; - }; + datasources.prometheus = { + enable = lib.mkEnableOption "" // { + default = prometheusDomains != [ ]; }; - victoriametrics = { - enable = lib.mkEnableOption "" // { - default = config.custom.web-services.victoriametrics.enable; - }; - url = lib.mkOption { - type = lib.types.nonEmptyStr; - default = "https://${config.custom.web-services.victoriametrics.domain}"; - }; + url = lib.mkOption { + type = lib.types.nonEmptyStr; + default = "https://metrics.${config.custom.networking.overlay.fqdn}"; }; }; - dashboards = { - nodeExporter.enable = lib.mkEnableOption "" // { - default = true; - }; - victoriametrics.enable = lib.mkEnableOption "" // { - default = config.custom.web-services.victoriametrics.enable; - }; + dashboards.nodeExporter.enable = lib.mkEnableOption "" // { + default = true; }; }; @@ -81,65 +73,45 @@ in datasources.settings = { prune = true; - datasources = [ - (lib.mkIf cfg.datasources.prometheus.enable { - name = "Prometheus"; - type = "prometheus"; - inherit (cfg.datasources.prometheus) url; - isDefault = true; - jsonData = { - prometheusType = "Prometheus"; - prometheusVersion = "2.50.0"; - }; - }) - (lib.mkIf cfg.datasources.victoriametrics.enable { - name = "VictoriaMetrics"; - type = "victoriametrics-metrics-datasource"; - inherit (cfg.datasources.victoriametrics) url; - isDefault = false; - }) - ]; + datasources = lib.optional cfg.datasources.prometheus.enable { + name = "Prometheus"; + type = "prometheus"; + inherit (cfg.datasources.prometheus) url; + isDefault = true; + jsonData = { + prometheusType = "Prometheus"; + prometheusVersion = "3.7.2"; + }; + }; }; }; - declarativePlugins = + }; + + # https://grafana.com/grafana/dashboards/1860-node-exporter-full/ + environment.etc."grafana-dashboards/node-exporter-full.json" = { + enable = cfg.dashboards.nodeExporter.enable; + source = pkgs.fetchurl { + name = "node-exporter-full.json"; + url = "https://grafana.com/api/dashboards/1860/revisions/41/download"; + hash = "sha256-EywgxEayjwNIGDvSmA/S56Ld49qrTSbIYFpeEXBJlTs="; + }; + }; + + custom.services.caddy = { + virtualHosts.${cfg.domain}.port = cfg.port; + + virtualHosts."metrics.${config.custom.networking.overlay.fqdn}".extraConfig = let - plugins = pkgs.grafanaPlugins; + upstreams = prometheusDomains |> lib.map (domain: "https://${domain}") |> lib.concatStringsSep " "; in - [ - (lib.optional cfg.datasources.victoriametrics.enable plugins.victoriametrics-metrics-datasource) - ] - |> lib.concatLists; - }; - - environment.etc = { - # https://grafana.com/grafana/dashboards/1860-node-exporter-full/ - "grafana-dashboards/node-exporter-full.json" = { - enable = cfg.dashboards.nodeExporter.enable; - source = pkgs.fetchurl { - name = "node-exporter-full.json"; - url = "https://grafana.com/api/dashboards/1860/revisions/41/download"; - hash = "sha256-EywgxEayjwNIGDvSmA/S56Ld49qrTSbIYFpeEXBJlTs="; - }; - }; - # https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/ - "grafana-dashboards/victoriametrics-single-node-patched.json" = { - enable = cfg.dashboards.victoriametrics.enable; - source = - pkgs.fetchurl { - name = "victoriametrics-single-node.json"; - url = "https://grafana.com/api/dashboards/10229/revisions/41/download"; - hash = "sha256-mwtah8A2w81WZjf5bUXoTJfS1R9UX+tua2PiDrBKJCQ="; + '' + reverse_proxy ${upstreams} { + header_up Host {upstream_hostport} + lb_policy first + health_uri /api/v1/status/buildinfo } - |> ( - src: - pkgs.runCommand "victoriametrics-single-node-patched.json" { buildInputs = [ pkgs.gnused ]; } '' - sed 's/victoriametrics-logs-//g' ${src} > $out - '' - ); - }; + ''; }; - - custom.services.caddy.virtualHosts.${cfg.domain}.port = cfg.port; }; } diff --git a/modules/nixos/web-services/victoriametrics.nix b/modules/nixos/web-services/victoriametrics.nix deleted file mode 100644 index 0aae07b..0000000 --- a/modules/nixos/web-services/victoriametrics.nix +++ /dev/null @@ -1,49 +0,0 @@ -{ config, lib, ... }: -let - cfg = config.custom.web-services.victoriametrics; -in -{ - options.custom.web-services.victoriametrics = { - enable = lib.mkEnableOption ""; - domain = lib.mkOption { - type = lib.types.nonEmptyStr; - default = ""; - }; - port = lib.mkOption { - type = lib.types.port; - default = 8428; - }; - }; - - config = lib.mkIf cfg.enable { - users = { - users.victoriametrics = { - isSystemUser = true; - group = config.users.groups.victoriametrics.name; - }; - groups.victoriametrics = { }; - }; - - systemd.services.victoriametrics.serviceConfig = { - DynamicUser = lib.mkForce false; - User = config.users.users.victoriametrics.name; - Group = config.users.groups.victoriametrics.name; - }; - - services.victoriametrics = { - enable = true; - listenAddress = "localhost:${toString cfg.port}"; - extraOptions = [ - "-selfScrapeInterval=15s" - "-selfScrapeJob=victoriametrics" - "-selfScrapeInstance=${config.networking.hostName}" - ]; - }; - - custom = { - services.caddy.virtualHosts.${cfg.domain}.port = cfg.port; - - persistence.directories = [ "/var/lib/${config.services.victoriametrics.stateDir}" ]; - }; - }; -}