diff --git a/hosts/vps-monitor/default.nix b/hosts/vps-monitor/default.nix index 109cdff..1cc36c3 100644 --- a/hosts/vps-monitor/default.nix +++ b/hosts/vps-monitor/default.nix @@ -50,6 +50,11 @@ enable = true; domain = "grafana.${privateDomain}"; }; + + victoriametrics = { + enable = true; + domain = "metrics.${privateDomain}"; + }; }; }; } diff --git a/modules/nixos/services/alertmanager.nix b/modules/nixos/services/alertmanager.nix index 295e2e7..6e4f103 100644 --- a/modules/nixos/services/alertmanager.nix +++ b/modules/nixos/services/alertmanager.nix @@ -72,10 +72,7 @@ in http.addr = "localhost:${toString cfg.ntfyBridgePort}"; ntfy = { baseurl = "https://ntfy.sh"; - notification = { - topic = "splitleaf"; - priority = "default"; - }; + notification.topic = "splitleaf"; }; }; }; diff --git a/modules/nixos/services/alloy.nix b/modules/nixos/services/alloy.nix index 5330db7..3bed041 100644 --- a/modules/nixos/services/alloy.nix +++ b/modules/nixos/services/alloy.nix @@ -1,9 +1,4 @@ -{ - config, - lib, - allHosts, - ... -}: +{ config, lib, ... }: let cfg = config.custom.services.alloy; in @@ -18,6 +13,10 @@ in type = lib.types.port; default = 12345; }; + metricsEndpoint = lib.mkOption { + type = lib.types.nonEmptyStr; + default = "https://metrics.${config.custom.networking.overlay.domain}/prometheus/api/v1/write"; + }; collect.metrics = { system = lib.mkEnableOption "" // { default = true; @@ -48,30 +47,19 @@ in environment.etc = let - anyIsTrue = attrs: attrs |> lib.attrValues |> lib.any lib.id; - - prometheusEndpoints = - allHosts - |> lib.attrValues - |> lib.filter (host: host.config.custom.services.prometheus.enable) - |> lib.map (host: "https://${host.config.custom.services.prometheus.domain}/api/v1/write"); + isTrue = x: x; + anyIsTrue = attrs: attrs |> lib.attrValues |> lib.any isTrue; in { "alloy/metrics-endpoint.alloy" = { enable = cfg.collect.metrics |> anyIsTrue; - text = - prometheusEndpoints - |> lib.map (url: '' + text = '' + prometheus.remote_write "default" { endpoint { - url = "${url}" + url = "${cfg.metricsEndpoint}" } - '') - |> lib.concatLines - |> (endpoints: '' - prometheus.remote_write "default" { - ${endpoints} - } - ''); + } + ''; }; "alloy/system-metrics.alloy" = { enable = cfg.collect.metrics.system; @@ -97,7 +85,7 @@ in instance = constants.hostname, }] forward_to = [prometheus.remote_write.default.receiver] - scrape_interval = "30s" + scrape_interval = "15s" } ''; }; diff --git a/modules/nixos/services/prometheus.nix b/modules/nixos/services/prometheus.nix index 0f0fb58..7e2cc16 100644 --- a/modules/nixos/services/prometheus.nix +++ b/modules/nixos/services/prometheus.nix @@ -30,7 +30,6 @@ in inherit (cfg) port; webExternalUrl = "https://${cfg.domain}"; - extraFlags = [ "--web.enable-remote-write-receiver" ]; globalConfig = { scrape_interval = "30s"; external_labels.monitor = "global"; diff --git a/modules/nixos/web-services/grafana.nix b/modules/nixos/web-services/grafana.nix index 959336a..60f4134 100644 --- a/modules/nixos/web-services/grafana.nix +++ b/modules/nixos/web-services/grafana.nix @@ -2,18 +2,10 @@ config, pkgs, lib, - allHosts, ... }: let cfg = config.custom.web-services.grafana; - - prometheusDomains = - allHosts - |> lib.attrValues - |> lib.map (host: host.config.custom.services.prometheus) - |> lib.filter (prometheus: prometheus.enable) - |> lib.map (prometheus: prometheus.domain); in { options.custom.web-services.grafana = { @@ -26,17 +18,33 @@ in type = lib.types.port; default = 3000; }; - datasources.prometheus = { - enable = lib.mkEnableOption "" // { - default = prometheusDomains != [ ]; + datasources = { + prometheus = { + enable = lib.mkEnableOption "" // { + default = config.custom.web-services.victoriametrics.enable; + }; + url = lib.mkOption { + type = lib.types.nonEmptyStr; + default = "https://${config.custom.web-services.victoriametrics.domain}"; + }; }; - url = lib.mkOption { - type = lib.types.nonEmptyStr; - default = "https://metrics.${config.custom.networking.overlay.fqdn}"; + victoriametrics = { + enable = lib.mkEnableOption "" // { + default = config.custom.web-services.victoriametrics.enable; + }; + url = lib.mkOption { + type = lib.types.nonEmptyStr; + default = "https://${config.custom.web-services.victoriametrics.domain}"; + }; }; }; - dashboards.nodeExporter.enable = lib.mkEnableOption "" // { - default = true; + dashboards = { + nodeExporter.enable = lib.mkEnableOption "" // { + default = true; + }; + victoriametrics.enable = lib.mkEnableOption "" // { + default = config.custom.web-services.victoriametrics.enable; + }; }; }; @@ -73,45 +81,65 @@ in datasources.settings = { prune = true; - datasources = lib.optional cfg.datasources.prometheus.enable { - name = "Prometheus"; - type = "prometheus"; - inherit (cfg.datasources.prometheus) url; - isDefault = true; - jsonData = { - prometheusType = "Prometheus"; - prometheusVersion = "3.7.2"; - }; - }; + datasources = [ + (lib.mkIf cfg.datasources.prometheus.enable { + name = "Prometheus"; + type = "prometheus"; + inherit (cfg.datasources.prometheus) url; + isDefault = true; + jsonData = { + prometheusType = "Prometheus"; + prometheusVersion = "2.50.0"; + }; + }) + (lib.mkIf cfg.datasources.victoriametrics.enable { + name = "VictoriaMetrics"; + type = "victoriametrics-metrics-datasource"; + inherit (cfg.datasources.victoriametrics) url; + isDefault = false; + }) + ]; }; }; + declarativePlugins = + let + plugins = pkgs.grafanaPlugins; + in + [ + (lib.optional cfg.datasources.victoriametrics.enable plugins.victoriametrics-metrics-datasource) + ] + |> lib.concatLists; }; - # https://grafana.com/grafana/dashboards/1860-node-exporter-full/ - environment.etc."grafana-dashboards/node-exporter-full.json" = { - enable = cfg.dashboards.nodeExporter.enable; - source = pkgs.fetchurl { - name = "node-exporter-full.json"; - url = "https://grafana.com/api/dashboards/1860/revisions/41/download"; - hash = "sha256-EywgxEayjwNIGDvSmA/S56Ld49qrTSbIYFpeEXBJlTs="; + environment.etc = { + # https://grafana.com/grafana/dashboards/1860-node-exporter-full/ + "grafana-dashboards/node-exporter-full.json" = { + enable = cfg.dashboards.nodeExporter.enable; + source = pkgs.fetchurl { + name = "node-exporter-full.json"; + url = "https://grafana.com/api/dashboards/1860/revisions/41/download"; + hash = "sha256-EywgxEayjwNIGDvSmA/S56Ld49qrTSbIYFpeEXBJlTs="; + }; + }; + # https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/ + "grafana-dashboards/victoriametrics-single-node-patched.json" = { + enable = cfg.dashboards.victoriametrics.enable; + source = + pkgs.fetchurl { + name = "victoriametrics-single-node.json"; + url = "https://grafana.com/api/dashboards/10229/revisions/41/download"; + hash = "sha256-mwtah8A2w81WZjf5bUXoTJfS1R9UX+tua2PiDrBKJCQ="; + } + |> ( + src: + pkgs.runCommand "victoriametrics-single-node-patched.json" { buildInputs = [ pkgs.gnused ]; } '' + sed 's/victoriametrics-logs-//g' ${src} > $out + '' + ); }; }; - custom.services.caddy = { - virtualHosts.${cfg.domain}.port = cfg.port; - - virtualHosts."metrics.${config.custom.networking.overlay.fqdn}".extraConfig = - let - upstreams = prometheusDomains |> lib.map (domain: "https://${domain}") |> lib.concatStringsSep " "; - in - '' - reverse_proxy ${upstreams} { - header_up Host {upstream_hostport} - lb_policy first - health_uri /api/v1/status/buildinfo - } - ''; - }; + custom.services.caddy.virtualHosts.${cfg.domain}.port = cfg.port; }; } diff --git a/modules/nixos/web-services/victoriametrics.nix b/modules/nixos/web-services/victoriametrics.nix new file mode 100644 index 0000000..0aae07b --- /dev/null +++ b/modules/nixos/web-services/victoriametrics.nix @@ -0,0 +1,49 @@ +{ config, lib, ... }: +let + cfg = config.custom.web-services.victoriametrics; +in +{ + options.custom.web-services.victoriametrics = { + enable = lib.mkEnableOption ""; + domain = lib.mkOption { + type = lib.types.nonEmptyStr; + default = ""; + }; + port = lib.mkOption { + type = lib.types.port; + default = 8428; + }; + }; + + config = lib.mkIf cfg.enable { + users = { + users.victoriametrics = { + isSystemUser = true; + group = config.users.groups.victoriametrics.name; + }; + groups.victoriametrics = { }; + }; + + systemd.services.victoriametrics.serviceConfig = { + DynamicUser = lib.mkForce false; + User = config.users.users.victoriametrics.name; + Group = config.users.groups.victoriametrics.name; + }; + + services.victoriametrics = { + enable = true; + listenAddress = "localhost:${toString cfg.port}"; + extraOptions = [ + "-selfScrapeInterval=15s" + "-selfScrapeJob=victoriametrics" + "-selfScrapeInstance=${config.networking.hostName}" + ]; + }; + + custom = { + services.caddy.virtualHosts.${cfg.domain}.port = cfg.port; + + persistence.directories = [ "/var/lib/${config.services.victoriametrics.stateDir}" ]; + }; + }; +}