From 36f24bdd30b179d17423c38d2794deaa5a506f2e Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Tue, 3 Mar 2026 16:47:32 +0100 Subject: [PATCH 1/6] prometheus, alertmanager: Improve alerts --- modules/nixos/services/alertmanager.nix | 14 ++++++++++++++ modules/nixos/services/prometheus.nix | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/nixos/services/alertmanager.nix b/modules/nixos/services/alertmanager.nix index 295e2e7..45bbf84 100644 --- a/modules/nixos/services/alertmanager.nix +++ b/modules/nixos/services/alertmanager.nix @@ -75,6 +75,20 @@ in notification = { topic = "splitleaf"; priority = "default"; + templates = { + title = ''{{ index .Annotations "summary" }}''; + description = ""; + }; + tags = [ + { + condition = ''status == "resolved"''; + tag = "white_check_mark"; + } + { + condition = ''status == "firing"''; + tag = "rotating_light"; + } + ]; }; }; }; diff --git a/modules/nixos/services/prometheus.nix b/modules/nixos/services/prometheus.nix index 182a70c..7c1b17c 100644 --- a/modules/nixos/services/prometheus.nix +++ b/modules/nixos/services/prometheus.nix @@ -89,7 +89,7 @@ in expr = "up == 0"; for = "2m"; labels.severity = "critical"; - annotations.summary = "Instance {{ $labels.instance }} down"; + annotations.summary = "{{ $labels.instance }} is DOWN"; }; }; } From 7607620fbb6933bc87af0249d8cfa5a7ad187c29 Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Tue, 3 Mar 2026 16:49:28 +0100 Subject: [PATCH 2/6] workflows/ci: Add success/failure ntfy notifications --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cb43c4..a64000e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,3 +105,31 @@ jobs: - uses: actions/checkout@v5 - name: Push to deploy branch run: git push origin HEAD:deploy --force + notify-success: + if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') + needs: [build-package, build-check, build-server, build-workstation, deploy] + runs-on: ubuntu-latest + steps: + - name: Send ntfy success notification + run: | + curl -s \ + -H "Title: CI succeeded" \ + -H "Priority: low" \ + -H "Tags: white_check_mark" \ + -H "Actions: view, Open workflow run, ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ + -d "Commit ${{ github.sha }} succeeded." \ + https://ntfy.sh/splitleaf + notify-failure: + if: always() && contains(needs.*.result, 'failure') + needs: [build-package, build-check, build-server, build-workstation, deploy] + runs-on: ubuntu-latest + steps: + - name: Send ntfy failure notification + run: | + curl -s \ + -H "Title: CI failed" \ + -H "Priority: default" \ + -H "Tags: rotating_light" \ + -H "Actions: view, Open workflow run, ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ + -d "Commit ${{ github.sha }} failed." \ + https://ntfy.sh/splitleaf From c910f76c023991cd7ece2c8189c0d3d18314dfe3 Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Tue, 3 Mar 2026 16:56:22 +0100 Subject: [PATCH 3/6] workflows/ci: Use markdown for notification body --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a64000e..f0a0491 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,8 @@ jobs: -H "Priority: low" \ -H "Tags: white_check_mark" \ -H "Actions: view, Open workflow run, ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ - -d "Commit ${{ github.sha }} succeeded." \ + -H "Markdown: yes" \ + -d "Commit `${{ github.sha }}` succeeded." \ https://ntfy.sh/splitleaf notify-failure: if: always() && contains(needs.*.result, 'failure') @@ -131,5 +132,6 @@ jobs: -H "Priority: default" \ -H "Tags: rotating_light" \ -H "Actions: view, Open workflow run, ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ - -d "Commit ${{ github.sha }} failed." \ + -H "Markdown: yes" \ + -d "Commit `${{ github.sha }}` failed." \ https://ntfy.sh/splitleaf From 5ff1cb2851a09978bd3aba3c5636ac645954d838 Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Tue, 3 Mar 2026 17:02:42 +0100 Subject: [PATCH 4/6] workflows/ci: Notify directly after the deploy job --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0a0491..d59c425 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,32 +106,32 @@ jobs: - name: Push to deploy branch run: git push origin HEAD:deploy --force notify-success: - if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') - needs: [build-package, build-check, build-server, build-workstation, deploy] + if: always() && needs.deploy.result == 'success' + needs: [deploy] runs-on: ubuntu-latest steps: - name: Send ntfy success notification run: | curl -s \ - -H "Title: CI succeeded" \ + -H "Title: CI: Deploy succeeded" \ -H "Priority: low" \ -H "Tags: white_check_mark" \ -H "Actions: view, Open workflow run, ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ -H "Markdown: yes" \ - -d "Commit `${{ github.sha }}` succeeded." \ + -d "Commit \`${GITHUB_SHA::7}\` deployed successfully." \ https://ntfy.sh/splitleaf notify-failure: - if: always() && contains(needs.*.result, 'failure') - needs: [build-package, build-check, build-server, build-workstation, deploy] + if: always() && needs.deploy.result != 'success' + needs: [deploy] runs-on: ubuntu-latest steps: - name: Send ntfy failure notification run: | curl -s \ - -H "Title: CI failed" \ + -H "Title: CI: Deploy failed" \ -H "Priority: default" \ -H "Tags: rotating_light" \ -H "Actions: view, Open workflow run, ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ -H "Markdown: yes" \ - -d "Commit `${{ github.sha }}` failed." \ + -d "Commit \`${GITHUB_SHA::7}\` failed to deploy." \ https://ntfy.sh/splitleaf From 2cd07d85810f48a62c2c2f7ed31bfd7f7ec226bb Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Tue, 3 Mar 2026 17:09:11 +0100 Subject: [PATCH 5/6] workflows/ci: Combine the two notify jobs into one --- .github/workflows/ci.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d59c425..376bf53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,12 +105,13 @@ jobs: - uses: actions/checkout@v5 - name: Push to deploy branch run: git push origin HEAD:deploy --force - notify-success: - if: always() && needs.deploy.result == 'success' + notify: + if: always() needs: [deploy] runs-on: ubuntu-latest steps: - - name: Send ntfy success notification + - name: Notify success + if: needs.deploy.result == 'success' run: | curl -s \ -H "Title: CI: Deploy succeeded" \ @@ -120,12 +121,8 @@ jobs: -H "Markdown: yes" \ -d "Commit \`${GITHUB_SHA::7}\` deployed successfully." \ https://ntfy.sh/splitleaf - notify-failure: - if: always() && needs.deploy.result != 'success' - needs: [deploy] - runs-on: ubuntu-latest - steps: - - name: Send ntfy failure notification + - name: Notify failure + if: needs.deploy.result != 'success' run: | curl -s \ -H "Title: CI: Deploy failed" \ From 8855886547499c54c92e6ffd7baf5ad018f52a71 Mon Sep 17 00:00:00 2001 From: SebastianStork Date: Tue, 3 Mar 2026 17:23:19 +0100 Subject: [PATCH 6/6] prometheus, alertmanager: Add description to alert --- modules/nixos/services/alertmanager.nix | 5 +---- modules/nixos/services/prometheus.nix | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/nixos/services/alertmanager.nix b/modules/nixos/services/alertmanager.nix index 45bbf84..4a2dcd8 100644 --- a/modules/nixos/services/alertmanager.nix +++ b/modules/nixos/services/alertmanager.nix @@ -75,10 +75,7 @@ in notification = { topic = "splitleaf"; priority = "default"; - templates = { - title = ''{{ index .Annotations "summary" }}''; - description = ""; - }; + templates.title = ''{{ index .Annotations "summary" }}''; tags = [ { condition = ''status == "resolved"''; diff --git a/modules/nixos/services/prometheus.nix b/modules/nixos/services/prometheus.nix index 7c1b17c..f9984b5 100644 --- a/modules/nixos/services/prometheus.nix +++ b/modules/nixos/services/prometheus.nix @@ -89,7 +89,10 @@ in expr = "up == 0"; for = "2m"; labels.severity = "critical"; - annotations.summary = "{{ $labels.instance }} is DOWN"; + annotations = { + summary = "{{ $labels.instance }} is DOWN"; + description = "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes."; + }; }; }; }