SebastianStork/nixos-config
1
0
Fork 0
mirror of https://github.com/SebastianStork/nixos-config.git synced 2026-03-22 20:09:07 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

prometheus: Enhance alert rules

Browse source
This commit is contained in:
SebastianStork 2026-03-10 20:15:39 +01:00
parent 8cf724fb97
commit 67f8f1689a
Signed by: SebastianStork
SSH key fingerprint: SHA256:tRrGdjYOwgHxpSc/wTOZQZEjxcb15P0tyXRsbAfd+2Q
1 changed files with 62 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

54
modules/nixos/services/prometheus.nix
View file

@ -58,25 +58,25 @@ in
scrapeConfigs = [ scrapeConfigs = [
{ {
job_name = "prometheus"; job_name = "prometheus";
static_configs = lib.singleton { static_configs =
targets =
allHosts allHosts
|> lib.attrValues |> lib.attrValues
|> lib.map (host: host.config.custom.services.prometheus) |> lib.filter (host: host.config.custom.services.prometheus.enable)
|> lib.filter (prometheus: prometheus.enable) |> lib.map (host: {
|> lib.map (prometheus: prometheus.domain); targets = lib.singleton host.config.custom.services.prometheus.domain;
}; labels.instance = host.config.networking.hostName;
});
} }
{ {
job_name = "alertmanager"; job_name = "alertmanager";
static_configs = lib.singleton { static_configs =
targets =
allHosts allHosts
|> lib.attrValues |> lib.attrValues
|> lib.map (host: host.config.custom.services.alertmanager) |> lib.filter (host: host.config.custom.services.alertmanager.enable)
|> lib.filter (alertmanager: alertmanager.enable) |> lib.map (host: {
|> lib.map (alertmanager: alertmanager.domain); targets = lib.singleton host.config.custom.services.alertmanager.domain;
}; labels.instance = host.config.networking.hostName;
});
} }
]; ];
@ -84,15 +84,31 @@ in
{ {
groups = lib.singleton { groups = lib.singleton {
name = "Rules"; name = "Rules";
rules = [ rules =
{ (
allHosts
|> lib.attrValues
|> lib.filter (host: host.config.custom.services.alloy.enable)
|> lib.filter (host: host.config.custom.networking.overlay.role == "server")
|> lib.map (host: host.config.networking.hostName)
|> lib.map (hostName: {
alert = "InstanceDown"; alert = "InstanceDown";
expr = "up == 0"; expr = ''absent_over_time(up{instance="${hostName}", job="node"}[2m])'';
for = "2m";
labels.severity = "critical"; labels.severity = "critical";
annotations = { annotations = {
summary = "{{ $labels.instance }} is DOWN"; summary = "${hostName} is DOWN";
description = "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes."; description = "${hostName} has not reported any metrics for more than 2 minutes.";
};
})
)
++ [
{
alert = "ServiceDown";
expr = ''up{job=~"prometheus|alertmanager"} == 0'';
for = "2m";
annotations = {
summary = "{{ $labels.job }} on {{ $labels.instance }} is DOWN";
description = "{{ $labels.job }} on {{ $labels.instance }} has been down for more than 2 minutes.";
}; };
} }
{ {
@ -116,7 +132,7 @@ in
}; };
} }
|> lib.strings.toJSON |> lib.strings.toJSON
|> pkgs.writeText "prometheus-instance-down-rule" |> pkgs.writeText "prometheus-rules"
|> toString |> toString
|> lib.singleton; |> lib.singleton;
}; };