modules/nixos/services/alertmanager.nix

@@ -63,14 +63,6 @@ in
             name = "ntfy";
             webhook_configs = lib.singleton { url = "http://localhost:${toString cfg.ntfyBridgePort}/hook"; };
           };
-          inhibit_rules = lib.singleton {
-            source_matchers = [
-              ''alertname="InstanceDown"''
-              ''job="node"''
-            ];
-            target_matchers = lib.singleton ''alertname!="InstanceDown"'';
-            equal = [ "instance" ];
-          };
         };
       };
 

modules/nixos/services/nameservers/public.nix

@@ -46,12 +46,7 @@ let
     in
     inputs.dns.lib.toString zone {
       SOA = {
-        nameServer =
+        nameServer = "${netCfg.hostName}.${zone}.";
-          nsRecords
-          |> lib.map (record: record.name)
-          |> lib.naturalSort
-          |> lib.head
-          |> (hostName: "${hostName}.${zone}.");
         adminEmail = "hostmaster@sstork.dev";
         serial = 1;
       };

modules/nixos/services/prometheus.nix

@@ -58,25 +58,25 @@ in
       scrapeConfigs = [
         {
           job_name = "prometheus";
-          static_configs =
+          static_configs = lib.singleton {
+            targets =
               allHosts
               |> lib.attrValues
-            |> lib.filter (host: host.config.custom.services.prometheus.enable)
+              |> lib.map (host: host.config.custom.services.prometheus)
-            |> lib.map (host: {
+              |> lib.filter (prometheus: prometheus.enable)
-              targets = lib.singleton host.config.custom.services.prometheus.domain;
+              |> lib.map (prometheus: prometheus.domain);
-              labels.instance = host.config.networking.hostName;
+          };
-            });
         }
         {
           job_name = "alertmanager";
-          static_configs =
+          static_configs = lib.singleton {
+            targets =
               allHosts
               |> lib.attrValues
-            |> lib.filter (host: host.config.custom.services.alertmanager.enable)
+              |> lib.map (host: host.config.custom.services.alertmanager)
-            |> lib.map (host: {
+              |> lib.filter (alertmanager: alertmanager.enable)
-              targets = lib.singleton host.config.custom.services.alertmanager.domain;
+              |> lib.map (alertmanager: alertmanager.domain);
-              labels.instance = host.config.networking.hostName;
+          };
-            });
         }
       ];
 
@@ -84,31 +84,15 @@ in
         {
           groups = lib.singleton {
             name = "Rules";
-            rules =
+            rules = [
-              (
+              {
-                allHosts
-                |> lib.attrValues
-                |> lib.filter (host: host.config.custom.services.alloy.enable)
-                |> lib.filter (host: host.config.custom.networking.overlay.role == "server")
-                |> lib.map (host: host.config.networking.hostName)
-                |> lib.map (hostName: {
                 alert = "InstanceDown";
-                  expr = ''absent_over_time(up{instance="${hostName}", job="node"}[2m])'';
+                expr = "up == 0";
+                for = "2m";
                 labels.severity = "critical";
                 annotations = {
-                    summary = "${hostName} is DOWN";
+                  summary = "{{ $labels.instance }} is DOWN";
-                    description = "${hostName} has not reported any metrics for more than 2 minutes.";
+                  description = "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.";
-                  };
-                })
-              )
-              ++ [
-                {
-                  alert = "ServiceDown";
-                  expr = ''up{job=~"prometheus|alertmanager"} == 0'';
-                  for = "2m";
-                  annotations = {
-                    summary = "{{ $labels.job }} on {{ $labels.instance }} is DOWN";
-                    description = "{{ $labels.job }} on {{ $labels.instance }} has been down for more than 2 minutes.";
                 };
               }
               {
@@ -132,7 +116,7 @@ in
           };
         }
         |> lib.strings.toJSON
-        |> pkgs.writeText "prometheus-rules"
+        |> pkgs.writeText "prometheus-instance-down-rule"
         |> toString
         |> lib.singleton;
     };