From 66ec7a51e3845bf6eb0702dd3b9f1df546bcef12 Mon Sep 17 00:00:00 2001
From: Ege Demir <demegire@gmail.com>
Date: Tue, 13 Oct 2026 12:55:04 +0000
Subject: [PATCH 1/2] Deploy every merge to staging, then production; require a
 staging run before merging

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/pull_request_template.md     |   7 ++
 .github/workflows/deploy-staging.yml |  28 +++++--
 .github/workflows/deploy.yml         | 114 +++++++++++++++++++++++++++
 .github/workflows/staging-check.yml  |  47 +++++++++++
 AGENTS.md                            |   5 +-
 CONTRIBUTING.md                      |   5 +-
 docs/repository-policy.md            |  30 ++++++-
 docs/staging.md                      |  67 ++++++++++------
 infra/activate_release.sh            |  66 +++++++++++++++-
 infra/deploy.sh                      |  33 +++++---
 10 files changed, 352 insertions(+), 50 deletions(-)
 create mode 100644 .github/workflows/deploy.yml
 create mode 100644 .github/workflows/staging-check.yml

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index e7eb0889..31d31ae3 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -6,6 +6,13 @@
 
 <!-- List checks run, results and any skips. For UI changes, add a synthetic-data screenshot. -->
 
+## Tested on staging
+
+<!-- Changes to the running service: deploy this branch to staging (push it to the staging
+     branch, or run deploy-staging with its ref), use the changed behavior there, and say
+     what you did and what happened. Name the staging run ID when a workflow ran. Merging
+     deploys to production. Docs, tests or evals only: delete this section. -->
+
 ## Notes
 
 <!-- Mention limitations or setup changes. For integrations, link official API/auth docs,
diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index 150628a9..00af0868 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -1,20 +1,22 @@
 name: deploy-staging
 
-# Staging runs every merge to main, and any branch on request. Production deploys stay manual.
+# Try a branch on staging before merging it: push it to the staging branch
+# (git push -f origin my-branch:staging) or run this workflow with its ref.
+# Merges to main reach staging (and then production) through the deploy workflow.
 on:
   push:
-    branches: [main]
+    branches: [staging]
   workflow_dispatch:
     inputs:
       ref:
-        description: Branch, tag or SHA to deploy to staging
+        description: Branch, tag, SHA or refs/pull/<n>/head to deploy to staging
         required: true
-        default: main
 
 permissions:
   contents: read
 
-# One release at a time; a newer request waits instead of interrupting a deploy midway.
+# One branch deploy at a time; a newer request waits instead of interrupting a deploy.
+# Deploys from every workflow also wait for each other on the host itself.
 concurrency:
   group: deploy-staging
   cancel-in-progress: false
@@ -22,7 +24,7 @@ concurrency:
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 45
     # The staging environment is the only identity Google Cloud trusts for this deployer.
     environment:
       name: staging
@@ -30,6 +32,7 @@ jobs:
     permissions:
       contents: read
       id-token: write
+      statuses: write
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -46,3 +49,16 @@ jobs:
           TIN_LITE_GCP_PROJECT: ${{ vars.GCP_PROJECT }}
           TIN_LITE_STATIC_IP: ${{ vars.STAGING_STATIC_IP }}
         run: bash infra/deploy.sh
+      # The required "staging" check on main: this exact commit ran on staging. Testing it
+      # there, and saying what was tested in the pull request, is up to its author.
+      - name: Mark the commit as deployed to staging
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          ACTOR: ${{ github.actor }}
+        run: |
+          sha="$(git rev-parse HEAD)"
+          gh api --method POST "repos/${REPO}/statuses/${sha}" \
+            -f state=success -f context=staging \
+            -f description="Deployed to staging by ${ACTOR}; test it there before merging" \
+            -f target_url=https://app.staging.tin.computer
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
new file mode 100644
index 00000000..ae47e6b3
--- /dev/null
+++ b/.github/workflows/deploy.yml
@@ -0,0 +1,114 @@
+name: deploy
+
+# A merge to main reaches production with no manual step: once ci passes on main, the same
+# commit deploys to staging, and only if staging starts cleanly does it deploy to production.
+# Running this workflow by hand redeploys any commit on main (for example, to roll back).
+on:
+  workflow_run:
+    workflows: [ci]
+    types: [completed]
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      sha:
+        description: Full SHA of a commit on main to deploy to staging and production
+        required: true
+
+permissions:
+  contents: read
+
+# Newer merges include older ones: a waiting run is replaced by the newest, never interrupted.
+concurrency:
+  group: deploy-main
+  cancel-in-progress: false
+
+jobs:
+  release:
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      (github.event.workflow_run.conclusion == 'success' &&
+       github.event.workflow_run.event == 'push')
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      sha: ${{ steps.pick.outputs.sha }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: main
+          fetch-depth: 0
+          persist-credentials: false
+      - id: pick
+        env:
+          DISPATCHED: ${{ inputs.sha }}
+          FROM_CI: ${{ github.event.workflow_run.head_sha }}
+        run: |
+          sha="${DISPATCHED:-${FROM_CI}}"
+          if [[ ! "${sha}" =~ ^[0-9a-f]{40}$ ]]; then
+            echo "a full commit SHA is required" >&2
+            exit 1
+          fi
+          if ! git merge-base --is-ancestor "${sha}" origin/main; then
+            echo "${sha} is not on main" >&2
+            exit 1
+          fi
+          echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
+
+  staging:
+    needs: release
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    environment:
+      name: staging
+      url: https://app.staging.tin.computer
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ needs.release.outputs.sha }}
+          persist-credentials: false
+      - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
+        with:
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.GCP_DEPLOY_SERVICE_ACCOUNT }}
+      - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
+      - name: Deploy to staging
+        env:
+          TIN_LITE_DEPLOY_TARGET: staging
+          TIN_LITE_GCP_PROJECT: ${{ vars.GCP_PROJECT }}
+          TIN_LITE_STATIC_IP: ${{ vars.STAGING_STATIC_IP }}
+        run: bash infra/deploy.sh
+
+  production:
+    needs: [release, staging]
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    # Only main may use this environment; Google trusts only it for the production deployer.
+    environment:
+      name: production
+      url: https://app.tin.computer
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ needs.release.outputs.sha }}
+          persist-credentials: false
+      - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
+        with:
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.GCP_DEPLOY_SERVICE_ACCOUNT }}
+      - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
+      - name: Deploy to production
+        if: vars.PROD_AUTO_DEPLOY != 'false' || github.event_name == 'workflow_dispatch'
+        env:
+          TIN_LITE_DEPLOY_TARGET: prod
+          TIN_LITE_GCP_PROJECT: ${{ vars.GCP_PROJECT }}
+          TIN_LITE_STATIC_IP: ${{ vars.PRODUCTION_STATIC_IP }}
+        run: bash infra/deploy.sh
+      - name: Report a paused production deploy
+        if: vars.PROD_AUTO_DEPLOY == 'false' && github.event_name != 'workflow_dispatch'
+        run: echo "::warning::PROD_AUTO_DEPLOY is false; production was not deployed"
diff --git a/.github/workflows/staging-check.yml b/.github/workflows/staging-check.yml
new file mode 100644
index 00000000..1dadaa79
--- /dev/null
+++ b/.github/workflows/staging-check.yml
@@ -0,0 +1,47 @@
+name: staging-check
+
+# Sets the required "staging" status on a pull request's head commit. A change to the
+# running service stays pending until that exact commit is deployed to staging
+# (deploy-staging marks it). Changes that cannot affect the service pass here.
+#
+# pull_request_target so fork pull requests can receive a status. This job never checks out
+# or runs pull request code; it only reads the list of changed paths.
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+permissions:
+  contents: read
+
+jobs:
+  staging:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      pull-requests: read
+      statuses: write
+    steps:
+      - env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          NUMBER: ${{ github.event.pull_request.number }}
+          SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          set -euo pipefail
+          [[ "${NUMBER}" =~ ^[0-9]+$ && "${SHA}" =~ ^[0-9a-f]{40}$ ]]
+          # Paths that never reach the running service.
+          exempt='^(docs/|tests/|workflow_evals/|\.github/(pull_request_template\.md|review-replies\.md|CODEOWNERS)$|[^/]+\.md$|LICENSE$)'
+          service="$(gh api --paginate "repos/${REPO}/pulls/${NUMBER}/files" --jq '.[].filename' |
+            grep -vE "${exempt}" || true)"
+          if [[ -z "${service}" ]]; then
+            gh api --method POST "repos/${REPO}/statuses/${SHA}" -f state=success \
+              -f context=staging -f description="No change to the running service"
+            exit 0
+          fi
+          current="$(gh api "repos/${REPO}/commits/${SHA}/status" \
+            --jq '[.statuses[] | select(.context == "staging")][0].state // ""')"
+          if [[ "${current}" != "success" ]]; then
+            gh api --method POST "repos/${REPO}/statuses/${SHA}" -f state=pending \
+              -f context=staging \
+              -f description="Deploy this commit to staging and test it (deploy-staging)"
+          fi
diff --git a/AGENTS.md b/AGENTS.md
index 86ab4025..cb01318f 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -17,7 +17,10 @@ first. Workflow pull requests that don't meet it are closed automatically.
   frontend framework, provider fallback or plugin marketplace as incidental work.
 - Preserve unrelated edits and use an isolated worktree when another agent is working.
   Do not deploy, change provider configuration, rewrite history or publish repositories
-  without explicit authority. A merged PR is not a production deployment.
+  without explicit authority. Merging to main deploys to production: once ci passes, the
+  deploy workflow ships that commit to staging and then production, so a merge needs the
+  same authority as a deploy. Run a service change on staging before merging it; see
+  [staging](docs/staging.md).
 - Do not copy runtime dependencies from unrelated repositories. Keep the import-linter
   boundaries and the explicit registry; no wildcard workflow registration.
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4c346be9..04fbf71d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -36,8 +36,9 @@ Do not use it to skip review of external contributions. See the
 
 PR tests run on GitHub-hosted runners with a read-only repository token and no production
 credentials. A maintainer may need to approve a first-time contributor's CI run; that only
-allows the tests to run, not the PR to merge or deploy. Live provider tests and deployment
-remain separate maintainer operations.
+allows the tests to run, not the PR to merge or deploy. A merge to main deploys to
+production, so maintainers run a service change on [staging](docs/staging.md) before
+merging it. Live provider tests remain a separate maintainer operation.
 
 ## Contributing an integration
 
diff --git a/docs/repository-policy.md b/docs/repository-policy.md
index ee20be19..54ea6096 100644
--- a/docs/repository-policy.md
+++ b/docs/repository-policy.md
@@ -1,6 +1,6 @@
 # Repository contribution policy
 
-The simple path is: focused PR, passing `verify` checks, one Tin review, merge.
+The simple path is: focused PR, passing `verify` checks, tested on staging, one Tin review, merge.
 Bug fixes, docs and integration contributions do not need an approved issue first.
 Discuss architecture changes before starting a large implementation.
 
@@ -24,8 +24,24 @@ Any future authorized release-history rewrite needs a deliberate temporary rules
 and a verified private backup. Repository admins can edit settings; branch protection
 is not a defense against an administrator intentionally changing the policy.
 
-These settings are GitHub repository rules, not deployment automation. A merge does not
-deploy the application. Existing operator-controlled deployments are unchanged.
+## Merging deploys
+
+A merge to `main` deploys the application. When `ci` passes on `main`, the `deploy`
+workflow ships that commit to staging and, if staging starts cleanly, the same commit to
+production. There is no separate release step, so test before merging:
+
+- The required `staging` status passes on its own for changes that cannot reach the
+  running service (docs, tests, evals, Markdown). Anything else stays pending until that
+  exact commit is deployed to staging by `deploy-staging`. Every new push resets it.
+- Deploying marks the commit; it does not prove the change works. The PR's
+  **Tested on staging** section says what was exercised there and what happened, and the
+  reviewer judges it.
+- The maintainer bypass covers this check too. Skipping staging for a service change is
+  the same as deploying it untested.
+- Set the repository variable `PROD_AUTO_DEPLOY=false` to stop automatic production
+  deploys; merges then stop at staging. Running `deploy` by hand with a commit on `main`
+  redeploys it, which is also how to roll back. Nothing rolls back on its own: a release
+  may have changed the catalog for good, so a person chooses forward or back.
 
 ## CI for contributions
 
@@ -37,6 +53,9 @@ deployment. Superseded PR checks are cancelled; main-branch checks run independe
 Do not execute contributor code with `pull_request_target`, production environments,
 privileged self-hosted runners or a secret-bearing follow-up workflow. Live-provider
 acceptance stays a separate maintainer-run check against authorized test resources.
+`staging-check` uses `pull_request_target` only to read a PR's changed paths and set its
+status; it never checks out the code. Deploying an outside PR to staging runs it next to
+real provider credentials, so a maintainer reads the diff first.
 
 When public, use **Require approval for first-time contributors** for fork workflow
 runs. Approving CI means allowing code to run in this test environment, not approving
@@ -46,7 +65,10 @@ See [GitHub's fork approval guidance](https://docs.github.com/en/actions/how-tos
 
 ## Repository settings
 
-- Require the verify check, one code-owner review and resolved review threads on main.
+- Require the verify and staging checks, one code-owner review and resolved review threads
+  on main.
+- Restrict the `production` environment to `main`; Google Cloud trusts only that
+  environment for the production deployer, and only `staging` for the staging deployer.
 - Limit the review/check bypass to the named Tin maintainer team.
 - Independently block force-pushes and deletion of main with no bypass actors.
 - Keep the default Actions token read-only and prevent Actions from approving PRs.
diff --git a/docs/staging.md b/docs/staging.md
index fba43ba8..4f180d89 100644
--- a/docs/staging.md
+++ b/docs/staging.md
@@ -5,29 +5,50 @@ change end to end, as production would, before it reaches `app.tin.computer`. Lo
 development does not reproduce production's providers, callbacks or release process;
 staging does, with its own state.
 
-A deploy to staging is not a production deployment. Production stays a separate,
-explicitly authorized step.
-
-## Deploying
-
-- Every push to `main` deploys to staging (`.github/workflows/deploy-staging.yml`).
-- Any branch, tag or SHA deploys on request: **Actions → deploy-staging → Run workflow**,
-  or `gh workflow run deploy-staging -f ref=<branch>`. One deploy runs at a time; a newer
-  request waits. A branch deploy stays until the next one, so say in the team channel
-  when you take staging for a branch.
-- From a workstation: `TIN_LITE_DEPLOY_TARGET=staging TIN_LITE_GCP_PROJECT=<project>
-  bash infra/deploy.sh`.
-
-A staging deploy installs the host scripts, unpacks the release, runs `tin-lite migrate`
-from the new release and only then switches `current` and restarts. Its database is
-long-lived and never reset, so it applies every migration and catalog change in the same
-order production will. A release that would crash production's boot (a changed executor,
-a reused built-in ID) crashes staging's first.
-
-The workflow authenticates to Google Cloud with workload identity. Google trusts only
-jobs from this repository in the `staging` GitHub environment, and the deployer can log
-in to the staging host only. The environment's variables name the provider, service
-account, project and static IP.
+Merging to `main` deploys to production, so staging is where a change is tested first.
+
+## Testing a change before merging
+
+1. Deploy the branch: `git push -f origin my-branch:staging`, or **Actions →
+   deploy-staging → Run workflow** with its ref (`refs/pull/<n>/head` for a fork; read the
+   diff first, since staging holds real provider credentials).
+2. Use the changed behavior on `app.staging.tin.computer` with a staging test project.
+3. Write what you did and saw under **Tested on staging** in the pull request.
+
+Deploying marks that exact commit with the `staging` status that `main` requires. A new
+push resets it. Changes that cannot reach the running service (docs, tests, evals,
+Markdown) pass without a deploy. Staging has one slot: whoever deployed last owns it until
+the next deploy, and a merge to `main` also takes it, so say in the team channel when you
+are testing there.
+
+## From merge to production
+
+When `ci` passes on `main`, the `deploy` workflow deploys that commit to staging and, if it
+starts cleanly, the same commit to production. Every deploy, from CI or a workstation
+(`TIN_LITE_DEPLOY_TARGET=staging|prod TIN_LITE_GCP_PROJECT=<project> bash infra/deploy.sh`),
+goes through `infra/activate_release.sh` on the host:
+
+- It takes a host lock, so deploys by people and CI wait for each other.
+- It stops if inode use is above 90%.
+- It reinstalls host setup (Caddy, forward proxy, units) only when those scripts changed,
+  because that restarts the proxy in-flight sandboxes use.
+- It unpacks the release with the host's `/etc/tin-lite/release.env`, runs
+  `tin-lite migrate`, switches `current` and restarts, which drains the Temporal worker first.
+- It waits for the new process to log "Application startup complete" and fails if it
+  restarts instead. `/healthz` alone is not proof: the old process answers during the drain.
+- It keeps the newest 20 releases.
+
+A failed deploy stops and reports; nothing rolls back on its own. Staging's database is
+long-lived and never reset, so it applies every migration and catalog change in the order
+production will, and a release that would crash production's boot (a changed executor, a
+reused built-in ID) crashes staging's first and never reaches production.
+
+Set the repository variable `PROD_AUTO_DEPLOY=false` to stop at staging. Run **deploy** by
+hand with a commit on `main` to redeploy it or roll back.
+
+Both deployers authenticate with workload identity. Google trusts only this repository's
+`staging` environment for the staging deployer and its `production` environment (restricted
+to `main`) for the production deployer; each can log in to its own host only.
 
 ## What is separate and what is shared
 
diff --git a/infra/activate_release.sh b/infra/activate_release.sh
index 828e1cb5..b031e496 100755
--- a/infra/activate_release.sh
+++ b/infra/activate_release.sh
@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
+# Activate an uploaded release on this host. deploy.sh uploads everything into a private
+# per-deploy directory and runs this script as root; everything below happens under one host
+# lock, so deploys by people and by CI never interleave.
 set -euo pipefail
 
 if [[ "${EUID}" -ne 0 ]]; then
@@ -22,8 +25,36 @@ cleanup() {
 }
 trap cleanup EXIT
 
+exec 9>/run/lock/tin-lite-deploy.lock
+if ! flock -w 1800 9; then
+  echo "another deploy holds the host lock" >&2
+  exit 75
+fi
+
+# The root disk has run out of inodes before (hundreds of releases); stop before it does.
+inode_use="$(df --output=ipcent /opt/tin-lite | tail -n 1 | tr -dc '0-9')"
+if (( inode_use >= 90 )); then
+  echo "inode use on /opt/tin-lite is ${inode_use}%; prune releases before deploying" >&2
+  exit 74
+fi
+
+# Host setup (Caddy, forward proxy, units) runs only when its scripts changed: it restarts
+# the proxy that in-flight sandboxes use. TIN_LITE_FORCE_HOST_INSTALL=1 forces it.
+if [[ -f "${upload_dir}/install_switchboard.sh" ]]; then
+  infra_hash="$(cd "${upload_dir}" && cat install_switchboard.sh install_forward_proxy.sh \
+    proxy_policy.py proxy_grants.py | sha256sum | cut -d' ' -f1)"
+  if [[ "${TIN_LITE_FORCE_HOST_INSTALL:-0}" == "1" ]] ||
+    [[ "$(cat /etc/tin-lite/host-install.sha256 2>/dev/null)" != "${infra_hash}" ]]; then
+    echo "host setup changed; installing"
+    bash "${upload_dir}/install_switchboard.sh"
+    printf '%s\n' "${infra_hash}" > /etc/tin-lite/host-install.sha256
+  fi
+fi
+
 release_sha="$1"
 release_dir="/opt/tin-lite/releases/${release_sha}"
+previous="$(readlink /opt/tin-lite/current || true)"
+echo "current release: ${previous:-none}"
 install -d -m 750 -o tinlite -g tinlite "${release_dir}"
 tar -xzf "${upload_dir}/tin-lite-release.tar.gz" -C "${release_dir}"
 if [[ -f "${upload_dir}/tin-lite-github-app.pem" ]]; then
@@ -33,7 +64,7 @@ if [[ -f "${upload_dir}/tin-lite-github-app.pem" ]]; then
     's#^TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH=.*#TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH=/etc/tin-lite/github-app.pem#' \
     "${upload_dir}/tin-lite.env"
 fi
-# An uploaded .env wins; otherwise the release env kept on the host (staging) is used.
+# An uploaded .env wins; otherwise the release env kept on the host is used.
 release_env="${upload_dir}/tin-lite.env"
 if [[ ! -f "${release_env}" ]]; then
   release_env=/etc/tin-lite/release.env
@@ -58,6 +89,39 @@ fi
 
 ln -sfn "${release_dir}" /opt/tin-lite/current
 systemctl enable tin-lite-switchboard
+restarts_before="$(systemctl show -p NRestarts --value tin-lite-switchboard)"
+restarted_at="$(date '+%Y-%m-%d %H:%M:%S')"
+# A restart drains the Temporal worker first (up to the unit's stop timeout).
 systemctl restart tin-lite-switchboard
 
 rm -f "${upload_dir}/tin-lite-release.tar.gz" "${upload_dir}/tin-lite.env"
+
+# /healthz stays 200 from the old process while it drains, so judge the new one by its own
+# startup line and the restart counter. A failure stops here and is reported; nothing is
+# rolled back automatically, because a release may have changed the catalog for good.
+deadline=$((SECONDS + ${TIN_LITE_STARTUP_TIMEOUT:-420}))
+until journalctl -u tin-lite-switchboard --since "${restarted_at}" --no-pager -q |
+  grep -q "Application startup complete"; do
+  if [[ "$(systemctl show -p NRestarts --value tin-lite-switchboard)" != "${restarts_before}" ]]; then
+    echo "the new release is restarting; it did not start cleanly (previous: ${previous})" >&2
+    journalctl -u tin-lite-switchboard --since "${restarted_at}" --no-pager -q | tail -n 40 >&2
+    exit 71
+  fi
+  if (( SECONDS > deadline )); then
+    echo "the new release did not report startup in time (previous: ${previous})" >&2
+    journalctl -u tin-lite-switchboard --since "${restarted_at}" --no-pager -q | tail -n 40 >&2
+    exit 72
+  fi
+  sleep 5
+done
+echo "release ${release_sha} started"
+
+# Keep the newest 20 releases, plus whatever current and the previous release point at.
+keep_current="$(readlink -f /opt/tin-lite/current)"
+keep_previous="$(readlink -f "${previous:-/nonexistent}" 2>/dev/null || true)"
+ls -1dt /opt/tin-lite/releases/*/ 2>/dev/null | tail -n +21 | while read -r old; do
+  old="${old%/}"
+  if [[ "${old}" != "${keep_current}" && "${old}" != "${keep_previous}" ]]; then
+    rm -rf --one-file-system "${old}"
+  fi
+done
diff --git a/infra/deploy.sh b/infra/deploy.sh
index 58fdf1c3..08d454e4 100755
--- a/infra/deploy.sh
+++ b/infra/deploy.sh
@@ -3,26 +3,20 @@ set -euo pipefail
 
 : "${TIN_LITE_GCP_PROJECT:?Set TIN_LITE_GCP_PROJECT to your deployment project}"
 
-# prod: the release env is uploaded from a local .env. staging: it stays on the host in
-# /etc/tin-lite/release.env, and migrations run before the release goes live.
+# Both targets keep their release env on the host (/etc/tin-lite/release.env) and run
+# migrations before the release goes live. TIN_LITE_UPLOAD_ENV=1 uploads a local .env
+# instead, for a host that has not been given one yet.
 target="${TIN_LITE_DEPLOY_TARGET:-prod}"
 case "${target}" in
   prod)
     instance=tin-lite-switchboard
     address=tin-lite-switchboard-ip
     default_host=app.tin.computer
-    upload_env=1
-    run_migrations="${TIN_LITE_RUN_MIGRATIONS:-0}"
-    health_retries=20
     ;;
   staging)
     instance=tin-lite-staging
     address=tin-lite-staging-ip
     default_host=app.staging.tin.computer
-    upload_env=0
-    run_migrations=1
-    # A first boot publishes the whole catalog and a restart drains running workflows.
-    health_retries=100
     ;;
   *)
     echo "TIN_LITE_DEPLOY_TARGET must be prod or staging" >&2
@@ -31,6 +25,14 @@ case "${target}" in
 esac
 zone=us-central1-a
 region=us-central1
+upload_env="${TIN_LITE_UPLOAD_ENV:-0}"
+run_migrations="${TIN_LITE_RUN_MIGRATIONS:-1}"
+for flag in "${upload_env}" "${run_migrations}" "${TIN_LITE_FORCE_HOST_INSTALL:-0}"; do
+  if [[ "${flag}" != 0 && "${flag}" != 1 ]]; then
+    echo "TIN_LITE_UPLOAD_ENV, TIN_LITE_RUN_MIGRATIONS and TIN_LITE_FORCE_HOST_INSTALL are 0 or 1" >&2
+    exit 64
+  fi
+done
 
 root="$(git rev-parse --show-toplevel)"
 cd "${root}"
@@ -121,15 +123,20 @@ move_env=""
 if [[ "${upload_env}" == "1" ]]; then
   move_env="mv ${upload_dir}/.env ${upload_dir}/tin-lite.env && "
 fi
+force_install="${TIN_LITE_FORCE_HOST_INSTALL:-0}"
+# activate_release.sh takes the host lock, installs host setup only when it changed,
+# migrates, switches, and waits for the new process to report startup.
 gcloud compute ssh "${instance}" \
   --project "${TIN_LITE_GCP_PROJECT}" --zone "${zone}" --quiet \
-  --command="sudo env TIN_LITE_STATIC_IP=${static_ip} TIN_LITE_PRODUCT_HOST=${product_host} bash ${upload_dir}/install_switchboard.sh && \
-${move_env}sudo env TIN_LITE_UPLOAD_DIR=${upload_dir} TIN_LITE_RUN_MIGRATIONS=${run_migrations} bash ${upload_dir}/activate_release.sh ${release_sha}; \
+  --command="${move_env}sudo env TIN_LITE_STATIC_IP=${static_ip} TIN_LITE_PRODUCT_HOST=${product_host} \
+TIN_LITE_UPLOAD_DIR=${upload_dir} TIN_LITE_RUN_MIGRATIONS=${run_migrations} \
+TIN_LITE_FORCE_HOST_INSTALL=${force_install} bash ${upload_dir}/activate_release.sh ${release_sha}; \
 status=\$?; sudo rm -rf ${upload_dir}; exit \$status"
 
+# The new process has reported startup; these confirm it serves through Caddy.
 public_host="tin-lite-switchboard.${static_ip//./-}.sslip.io"
-curl --http1.1 --fail --silent --show-error --retry "${health_retries}" --retry-all-errors --retry-delay 3 \
+curl --http1.1 --fail --silent --show-error --retry 20 --retry-all-errors --retry-delay 3 \
   "https://${public_host}/healthz"
 
-curl --http1.1 --fail --silent --show-error --retry "${health_retries}" --retry-all-errors --retry-delay 3 \
+curl --http1.1 --fail --silent --show-error --retry 20 --retry-all-errors --retry-delay 3 \
   "https://${product_host}/healthz"

From 49dd4561225b39b0da925bbd5928172bc50af3c1 Mon Sep 17 00:00:00 2001
From: Ege Demir <demegire@gmail.com>
Date: Tue, 13 Oct 2026 13:00:19 +0000
Subject: [PATCH 2/2] Release a branch with main's deploy scripts, mark it from
 a separate job, lock under /run

A branch deployed to staging no longer supplies the scripts that run as root on the
host, and its code never shares a job with a token that can write commit statuses.
The host lock moves out of the world-writable /run/lock.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/deploy-staging.yml | 37 ++++++++++++++++++++++------
 infra/activate_release.sh            |  2 +-
 infra/deploy.sh                      | 21 ++++++++++------
 3 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index 00af0868..30c8ab0b 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -29,15 +29,26 @@ jobs:
     environment:
       name: staging
       url: https://app.staging.tin.computer
+    # No write token here: the branch's code is packaged in this job.
     permissions:
       contents: read
       id-token: write
-      statuses: write
+    outputs:
+      sha: ${{ steps.release.outputs.sha }}
     steps:
+      # The scripts that run as root on the host come from main, never from the branch.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: main
+          path: trusted
+          persist-credentials: false
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ inputs.ref || github.sha }}
+          path: release
           persist-credentials: false
+      - id: release
+        run: echo "sha=$(git -C release rev-parse HEAD)" >> "${GITHUB_OUTPUT}"
       - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
         with:
           workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
@@ -48,17 +59,27 @@ jobs:
           TIN_LITE_DEPLOY_TARGET: staging
           TIN_LITE_GCP_PROJECT: ${{ vars.GCP_PROJECT }}
           TIN_LITE_STATIC_IP: ${{ vars.STAGING_STATIC_IP }}
-        run: bash infra/deploy.sh
-      # The required "staging" check on main: this exact commit ran on staging. Testing it
-      # there, and saying what was tested in the pull request, is up to its author.
-      - name: Mark the commit as deployed to staging
-        env:
+          TIN_LITE_RELEASE_SOURCE: ${{ github.workspace }}/release
+        run: bash trusted/infra/deploy.sh
+
+  # The required "staging" check on main: this exact commit ran on staging. Testing it there,
+  # and saying what was tested in the pull request, is up to its author. This job runs no
+  # branch code, so the status token never reaches it.
+  mark:
+    needs: deploy
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      statuses: write
+    steps:
+      - env:
           GH_TOKEN: ${{ github.token }}
           REPO: ${{ github.repository }}
           ACTOR: ${{ github.actor }}
+          SHA: ${{ needs.deploy.outputs.sha }}
         run: |
-          sha="$(git rev-parse HEAD)"
-          gh api --method POST "repos/${REPO}/statuses/${sha}" \
+          [[ "${SHA}" =~ ^[0-9a-f]{40}$ ]]
+          gh api --method POST "repos/${REPO}/statuses/${SHA}" \
             -f state=success -f context=staging \
             -f description="Deployed to staging by ${ACTOR}; test it there before merging" \
             -f target_url=https://app.staging.tin.computer
diff --git a/infra/activate_release.sh b/infra/activate_release.sh
index b031e496..e95d189f 100755
--- a/infra/activate_release.sh
+++ b/infra/activate_release.sh
@@ -25,7 +25,7 @@ cleanup() {
 }
 trap cleanup EXIT
 
-exec 9>/run/lock/tin-lite-deploy.lock
+exec 9>/run/tin-lite-deploy.lock
 if ! flock -w 1800 9; then
   echo "another deploy holds the host lock" >&2
   exit 75
diff --git a/infra/deploy.sh b/infra/deploy.sh
index 08d454e4..772ad37e 100755
--- a/infra/deploy.sh
+++ b/infra/deploy.sh
@@ -34,12 +34,18 @@ for flag in "${upload_env}" "${run_migrations}" "${TIN_LITE_FORCE_HOST_INSTALL:-
   fi
 done
 
-root="$(git rev-parse --show-toplevel)"
+# The host scripts always come from this checkout. TIN_LITE_RELEASE_SOURCE can name a
+# second checkout to release instead, so a branch under test never supplies the scripts that
+# run as root on the host (deploy-staging releases a branch with main's scripts).
+root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
 cd "${root}"
-if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then
-  echo "tracked worktree changes must be committed before deployment" >&2
-  exit 65
-fi
+release_source="$(cd -- "${TIN_LITE_RELEASE_SOURCE:-${root}}" && pwd)"
+for checkout in "${root}" "${release_source}"; do
+  if [[ -n "$(git -C "${checkout}" status --porcelain --untracked-files=no)" ]]; then
+    echo "tracked worktree changes must be committed before deployment (${checkout})" >&2
+    exit 65
+  fi
+done
 github_private_key_path=""
 if [[ "${upload_env}" == "1" ]]; then
   if [[ ! -f .env ]]; then
@@ -63,10 +69,11 @@ if [[ -n "${github_private_key_path}" ]]; then
   fi
 fi
 
-release_sha="$(git rev-parse HEAD)"
+release_sha="$(git -C "${release_source}" rev-parse HEAD)"
 temporary_dir="$(mktemp -d)"
 trap 'rm -rf "${temporary_dir}"' EXIT
-git archive --format=tar.gz --output="${temporary_dir}/tin-lite-release.tar.gz" HEAD
+git -C "${release_source}" archive --format=tar.gz \
+  --output="${temporary_dir}/tin-lite-release.tar.gz" HEAD
 
 # Each deploy uploads into its own private directory, so deploys by different people (or
 # by CI) never collide on files another account left in /tmp. A first connection may print