ci: Actually run rollback when deployment fails

.github/workflows/ci.yml

@@ -73,22 +73,72 @@ jobs:
           useDaemon: false
       - name: Build server
         run: nix build .#nixosConfigurations.${{ matrix.server }}.config.system.build.toplevel --print-build-logs
-  deploy:
+  trigger-deploy:
     needs: [build-check, build-server]
     runs-on: ubuntu-latest
+    outputs:
+      previous-sha: ${{ steps.previous-sha.outputs.sha }}
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v5
+      - name: Get previous deploy SHA
+        id: previous-sha
+        run: |
+          sha=$(git ls-remote origin deploy | cut -f1)
+          printf "sha=%s" "$sha" >> "$GITHUB_OUTPUT"
+      - name: Push to deploy branch
+        run: git push --force origin HEAD:refs/heads/deploy
+  await-deploy:
+    name: ${{ matrix.server }}
+    needs: [parse-flake, trigger-deploy]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        server: ${{ fromJson(needs.parse-flake.outputs.servers) }}
+    steps:
+      - name: Wait for deployment
+        timeout-minutes: 5
+        run: |
+          repo_url="https://github.com/${{ github.repository }}.git"
+          expected_sha="${{ github.sha }}"
+          branch="deployed/${{ matrix.server }}"
+
+          while true; do
+            deployed_sha=$(git ls-remote "$repo_url" "$branch" | cut -f1)
+
+            if [[ "$deployed_sha" == "$expected_sha" ]]; then
+              echo "✅ ${{ matrix.server }} deployed $expected_sha"
+              exit 0
+            else
+              echo "⏳ ${{ matrix.server }}: deployed ${deployed_sha::7}, waiting for ${expected_sha::7}..."
+            fi
+
+            sleep 5
+          done
+  rollback-deploy:
+    if: always() && needs.await-deploy.result == 'failure'
+    needs: [trigger-deploy, await-deploy]
+    runs-on: ubuntu-latest
     permissions:
       contents: write
     steps:
       - uses: actions/checkout@v5
-      - name: Push to deploy branch
+        with:
-        run: git push origin HEAD:deploy --force
+          fetch-depth: 0
+      - name: Rollback deploy branch
+        run: |
+          previous_sha="${{ needs.trigger-deploy.outputs.previous-sha }}"
+          echo "Rolling back deploy branch to $previous_sha"
+          git push --force origin "$previous_sha:refs/heads/deploy"
   notify:
     if: always()
-    needs: [deploy]
+    needs: [await-deploy]
     runs-on: ubuntu-latest
     steps:
       - name: Notify success
-        if: needs.deploy.result == 'success'
+        if: needs.await-deploy.result == 'success'
         run: |
           curl -s \
             -H "Title: CI: Deploy succeeded" \
@@ -99,7 +149,7 @@ jobs:
             -d "Commit \`${GITHUB_SHA::7}\` deployed successfully." \
             https://ntfy.sh/splitleaf
       - name: Notify failure
-        if: needs.deploy.result == 'failure'
+        if: needs.await-deploy.result == 'failure'
         run: |
           curl -s \
             -H "Title: CI: Deploy failed" \

hosts/srv-core/secrets.json

@@ -30,6 +30,9 @@
   "grafana": {
     "admin-password": "ENC[AES256_GCM,data:MNdjh+025bi5wtP77aKSGzcx8NgfY+YppH4qu/o=,iv:Hv8IF0n0A5+Hs6FQ7tXkdFbPN0ArdZD3vmrdovc0/Yg=,tag:8y0NsLQ+9k+mWnHbHzZvGg==,type:str]"
   },
+  "git": {
+    "push-token": "ENC[AES256_GCM,data:VEpbgf7EPI4rWV6ywmCdTz1J1nBK3vefg58pxd6uBaJgVS7kbhKAyWizw/WRUaW3YplHvS05I/dErUKlklZBZ3QgEKItnpSM/J8fjEJnb48U7bPtRFjqTStOmgUG,iv:X+wv6csKraZDEgD3GdhbvR7KnBx6uftDS1e4rX7fAZQ=,tag:WmGU2/N/3CXFxSGbeYFwhA==,type:str]"
+  },
   "sops": {
     "age": [
       {
@@ -41,9 +44,9 @@
         "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXOVRQU1R4VlBhdEFUdnR6\nZFNuNldaSkxJL3ptOVVscjRBNkQ4dFBmQUVBCmZrQmFMV0hWbTBQcm1FS3JrR0ZC\nbktvT04xczd6VkdCUWk2NnVVZHNFWkUKLS0tIGUwOHJSMHVsNTEyZEU2VWJFNGVy\nMVFDVThrRGQwZEtPeFYzZUVQYi80ZjAKUd/XzyzqMkMowvyeCnQDbOGJDKbuAUQb\nFClQuiH5iSQQrVPw7SHBNgdqbcdtC+hZ4tpPaV/wWtlpcqpr5mBJSA==\n-----END AGE ENCRYPTED FILE-----\n"
       }
     ],
-    "lastmodified": "2026-03-01T13:14:08Z",
+    "lastmodified": "2026-03-06T17:32:42Z",
-    "mac": "ENC[AES256_GCM,data:GywNeQDso210CV1rEY1LCwUu5ZDdjyP/W+QOJ5GJhLnLiF3BjivS4J9VbOANduhepnRLLHFKW0+zdzPJJl3ltBFpAuoDigEPF5PbXOT6R7oiEW3kkgD5LP79789ijkeGqGf2DUN0J04OFkcTnTy2tAd2O3kSnhi/EbIMvw7zjUY=,iv:TqXETQOQuuQCJ2+tvPeXHY08BGbhQ4fsu+RqX1EiPHY=,tag:qfaYvY/iuu0Ou9AxhJUnRQ==,type:str]",
+    "mac": "ENC[AES256_GCM,data:racxzbcs0cZ0dmkN5/Vo4SyjUaTyu2G/e60l2jN8A2SiJOLkPCwhlpJ347J01P/4RF/Pf6nnq5w2t/HLja7EAZtUw1kn+qCwI/79Eu8EBMdTvUfGmjNCB5CU+0oih4idUbSqP6UVgTT3GFQ4BaqksKQUC+e7gvsCzX55Z0vHsbU=,iv:gCW5W33sE+HH6fcvCqkwQBGMeQpiNgx388vvFas8YyY=,tag:GWwOS04FrbKrgq6YE77QTA==,type:str]",
     "unencrypted_suffix": "_unencrypted",
-    "version": "3.11.0"
+    "version": "3.12.1"
   }
 }

hosts/vps-ns/secrets.json

@@ -7,6 +7,9 @@
     "api-key": "ENC[AES256_GCM,data:oqnAPVfLU8CG64+TsRijZ/2Wzy11bt3PvoEqbpWZbcXIE2aM0oZtUUtCxt0DiWp8Uyta6AO40V8+EGkzeqL4O6VLxU4=,iv:KaEwSmoG5zYxsWjUxwqbfe77Iiv03IAnFaIjQ5YoYkc=,tag:ilzXSYElARjnWkOcBKZBdA==,type:str]",
     "secret-api-key": "ENC[AES256_GCM,data:dGOAsu3kPJmDwhddZGgrY8KrDJeS7PiEPjEVh/h8BgSygRKLInow/7PIaHcy8gIlsGFvU2CYORY7Vmf3QCxYbRTkdIQ=,iv:otDnIv0B1h1H6usJqSNVqv9UUcmx9r5Cn18Q6DFwBME=,tag:O2O3V33TmalVuL6y4V9ufw==,type:str]"
   },
+  "git": {
+    "push-token": "ENC[AES256_GCM,data:czn22goPG/e8I7/BVVtmcRNQDXT7ELr7BNlNCPoFdGbxQwBqeVSocB+rDPRW26nZgm2r10Q1WkzxqXRlRz8qtgXzPT9C7DYFoE+cS0nrHIqq/hVFWWBFhUZJ41qG,iv:OiHnk023rJuYLx3UygPdzGlXnZRTj9WCfkQd3IauGDU=,tag:SJv01tjeOKzAfSgxI62F4w==,type:str]"
+  },
   "sops": {
     "age": [
       {
@@ -18,9 +21,9 @@
         "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTVmV4dkZEaWVDNHMxUFdr\nYnhuVWpReXBNSEhhZkltQTE4bEpzSlBzL0VJCk15UFlwa0haWTZNaE1DVzVZVFBI\nd0QzcUptYmQ3dmhhdjhFV0xDSWdmMGMKLS0tIFhWamx6SXJleFFSVUFkRmw2VFZy\nOVVhNm9NSE0yRGFMQjNrM1B6cDVxSXMKrhAkDcWqutgSmQI5O+5i8fcwuTh2/XKr\nljK/Vn8EvGr8qEUeHzOBI1b5VtgngJkVJyfM9G/Q0lZvQF7ZZ5YCgQ==\n-----END AGE ENCRYPTED FILE-----\n"
       }
     ],
-    "lastmodified": "2026-03-01T13:43:41Z",
+    "lastmodified": "2026-03-06T17:33:30Z",
-    "mac": "ENC[AES256_GCM,data:9EQHy+GkONH/opiCrf9pafzZwRJ2xKwEPHO2YDDJD/1jw8IGvxg2GBt16QzP0PMczzASiLawOBbm2NsUmdz1bCkhABanj2+NQyIzX1xu5gqVKSmNb4BRUPY9M98atFl+2fzaeksC7t/BIizbGnehjRYQ5wFCsxNpCTw4G/Afzyk=,iv:dPIcQD6SPl821dUoezzW6cq7RlWGuJYbRCP4gcUvdpw=,tag:rBBJaxhWQ+fWbF3CMdpI+g==,type:str]",
+    "mac": "ENC[AES256_GCM,data:T/DxRmJ2tFobfSLhrN/PetxRv+66oLvjJuEqhcjT8Te88Pov+S+OqJ3tJnoHwhlQQLz3O+lnwEKD+HJTZYdl5SYsZwawGwrw5VJVnXIoeh7UqRNF6VqZNWY3ucJSJAh8NnnqflSRxjpyuYTOt5e25vLjsNIDyzwlGDS62+IbHZs=,iv:ANUah06mpDpx5/6xptlgW2zS/ZLKVo7zbjZnOVSVnuE=,tag:IsG5cxXUSUDRms4aSTd29A==,type:str]",
     "unencrypted_suffix": "_unencrypted",
-    "version": "3.11.0"
+    "version": "3.12.1"
   }
 }

hosts/vps-www/secrets.json

@@ -20,6 +20,9 @@
     "api-key": "ENC[AES256_GCM,data:KT1vUUnm0d2T9H1YI5SE6xgTVVdmbLFJeL/+otlw0mheH+9svnC19YP9szB/kBLAPLes+2qunpqM02B9tJxGgD30SNc=,iv:wVIkhrCQ7t7lkR0q0OO/XWZbgZYuja53XgUjfy1fTNA=,tag:MJ5tjbIofqcKXDuzD0b1aQ==,type:str]",
     "secret-api-key": "ENC[AES256_GCM,data:lmU41AkCVb15sFuMqTZ7qZ978D5BLEYYidsb0yPPjiPPUmnfqBNM5lMVo4k4gt/PcMLcRVv5I6DDqE44zOK7YPf6Fck=,iv:sS7PfE2/CjUekIXGlvEA/N6R5JE7BvgfeXJO5dzv6/8=,tag:rp8FY3hEOyoafmwTdROJ7Q==,type:str]"
   },
+  "git": {
+    "push-token": "ENC[AES256_GCM,data:bmhIbvUQknj0cQ4HEfbrRgjAC29OJ6d3KWAmu11ArPuXyarxNvm8RI7roD8rgZ94Ta5LS6WNqPhmTEz5KyWDaV0N5aXQ2fqWWqoTOUJe8BNGG557TvVjVENqynBN,iv:HSQJdHckzn48/byk4WbPKppfRv8WNrNzlMbdbGZdJzw=,tag:O3SLtx1EMqsF/sBhZHfZLw==,type:str]"
+  },
   "sops": {
     "age": [
       {
@@ -31,9 +34,9 @@
         "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvVnphWlNaTUU0QytIdnEv\nQnNzelN4MUMreXZrME5KZStFMDg2V1VFd21rCkk2Uy9ITEF3SjlRVEdMMXlPdHhG\nam1PNnp3emtnMnczeFFSSStJaHF6TkkKLS0tIEJKbFRzbmNqMjk5NXVHZnhlWWZ3\nYng5L1F5YUJGOTg3TTJCK281SG9Id3MKsmH2yj19ig2g+KzBGLD9dWkdvr6TLdSd\nuuDC+frhj7wWrEomOOjIoYtWHXkUtTSAnCEZhrhfyupYhEvlFfWRlw==\n-----END AGE ENCRYPTED FILE-----\n"
       }
     ],
-    "lastmodified": "2026-02-11T17:18:44Z",
+    "lastmodified": "2026-03-06T17:33:14Z",
-    "mac": "ENC[AES256_GCM,data:CVnAf/P0Uj1kzU148d9LodrM7vNSmQ1F7LZQC5WEeAhPl+w1FgizNdzyQo+ZnuDBHkAflwJlB1kSS7Ou5/8mC9pBY1Kc13f4f+vpXNbZEyoUXaVqoESPSFtCdLrV+Fj54xwZEAUPzwvujuf9ud3SIs76vDF8LjTHxJFk3JPX4Ys=,iv:vPmUUwYnhB/jFCN4Fz2Bwh6cbb9MZ8DzVeY5m4Fno1U=,tag:YRozMlFU7DvjBRG/K1qmyA==,type:str]",
+    "mac": "ENC[AES256_GCM,data:FKOin7fw3EH+isB8niH43mImfn5HrAtUAdaHnW0QWcL4kQ4RhLGjY2rPr3X/3x0MqtphIK9hVh7K9OcwBznUicJ4kxfmV/+VjGdoQtyOsbkqPfJ2KY9FZvP2m7qf01PGE+S25vlh6h9d1gEAR2SdRJukvkIT2FmRfGGmPFVJWdo=,iv:3PSPP6coFZpd6IYnMttn0HCdtFZvmRR/kjjbhgs8XWM=,tag:2zgUzX5llbb6fEcH1YKtyw==,type:str]",
     "unencrypted_suffix": "_unencrypted",
-    "version": "3.11.0"
+    "version": "3.12.1"
   }
 }

modules/nixos/services/comin.nix

@@ -1,11 +1,32 @@
 {
   config,
   inputs,
+  pkgs,
   lib,
   ...
 }:
 let
   cfg = config.custom.services.comin;
+
+  postDeploymentScript =
+    pkgs.writeShellApplication {
+      name = "comin-post-deployment";
+      runtimeInputs = [ pkgs.git ];
+      text = ''
+        if [[ "$COMIN_STATUS" != "done" ]]; then
+          echo "Deployment not successful (status: $COMIN_STATUS), skipping branch update"
+          exit 0
+        fi
+
+        token=$(cat "''${CREDENTIALS_DIRECTORY}/git-push-token")
+        repo_url="https://x-access-token:$token@github.com/SebastianStork/nixos-config.git"
+
+        git push --force "$repo_url" "$COMIN_GIT_SHA:refs/heads/deployed/$COMIN_HOSTNAME"
+
+        echo "Updated deployed/$COMIN_HOSTNAME to $COMIN_GIT_SHA"
+      '';
+    }
+    |> lib.getExe;
 in
 {
   imports = [ inputs.comin.nixosModules.comin ];
@@ -19,6 +40,11 @@ in
   };
 
   config = lib.mkIf cfg.enable {
+    sops.secrets."git/push-token" = { };
+    systemd.services.comin.serviceConfig.LoadCredential = "git-push-token:${
+      config.sops.secrets."git/push-token".path
+    }";
+
     services.comin = {
       enable = true;
       remotes = lib.singleton {
@@ -30,6 +56,7 @@ in
         listen_address = "127.0.0.1";
         port = cfg.metricsPort;
       };
+      postDeploymentCommand = postDeploymentScript;
     };
   };
 }