test: backwards compatibility suite and refactor of the test infra

.github/CODEOWNERS

@@ -0,0 +1,9 @@
+# CODEOWNERS
+
+## TODO(settings): create @datazip-inc/olake-admins and @datazip-inc/state-version-owners with WRITE access
+## TODO(settings): the counts these paths need live in .github/rulesets/state-version-approval.json
+
+/.github/CODEOWNERS @datazip-inc/olake-admins
+
+/constants/state-versions.json @datazip-inc/state-version-owners
+/tests/testutils/compatibility/compatibility_rules.json @datazip-inc/state-version-owners

.github/actions/commit-image/action.yml

@@ -0,0 +1,65 @@
+name: Commit driver images
+description: >
+  Makes the driver images for an arbitrary commit available locally, caching them in a container
+  registry so the first run that needs a commit builds them and every run after pulls. Tagged with
+  the abbreviated sha, which is what the test harness resolves a commit id to and the ref it looks
+  up -- it then finds the image present and skips its own build.
+
+inputs:
+  drivers:
+    description: Space separated drivers to build images for.
+    required: true
+  sha:
+    description: Commit to build. Empty is a no-op, for events that have no commit to compare against.
+    required: false
+    default: ''
+  cache-repo:
+    description: Registry repository the cached images live under, e.g. ghcr.io/owner/repo.
+    required: true
+  registry:
+    description: Registry to authenticate against.
+    required: false
+    default: ghcr.io
+  username:
+    description: Registry user.
+    required: true
+  token:
+    description: Registry token. A read-only one still pulls; publishing is skipped.
+    required: true
+  max-parallel:
+    description: >
+      How many images to build at once. Each is a full Go compile, and the job that calls this
+      usually has its own builds running beside it.
+    required: false
+    default: '3'
+  path:
+    required: false
+    default: tests/.commit-image-src
+
+runs:
+  using: composite
+  steps:
+    - name: Check out the commit
+      if: inputs.sha != ''
+      uses: actions/checkout@v7
+      with:
+        ref: ${{ inputs.sha }}
+        path: ${{ inputs.path }}
+
+    - name: Log in to the cache registry
+      if: inputs.sha != ''
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ inputs.registry }}
+        username: ${{ inputs.username }}
+        password: ${{ inputs.token }}
+
+    - name: Pull or build the commit's images
+      shell: bash
+      env:
+        DRIVERS: ${{ inputs.drivers }}
+        SHA: ${{ inputs.sha }}
+        SRC: ${{ inputs.path }}
+        CACHE_REPO: ${{ inputs.cache-repo }}
+        MAX_PARALLEL: ${{ inputs.max-parallel }}
+      run: ${{ github.action_path }}/build-images-from-commit.sh

.github/actions/commit-image/build-images-from-commit.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ -z "${SHA:-}" ]; then
+  echo "::notice::no commit to prepare images for; skipping"
+  exit 0
+fi
+
+TAG=$(git rev-parse --short "$SHA")
+
+pull_driver_image() {
+  if docker pull -q "$CACHE_REPO/source-$1:$TAG" >/dev/null 2>&1; then
+    docker tag "$CACHE_REPO/source-$1:$TAG" "olakego/source-$1:$TAG"
+    echo "restored olakego/source-$1:$TAG from the cache"
+  fi
+}
+export -f pull_driver_image
+export TAG CACHE_REPO
+printf '%s\n' $DRIVERS | xargs -P 0 -I{} bash -c 'pull_driver_image {}'
+
+# Presence decides what is missing, rather than each pull reporting back: the pulls ran in their own
+# shells, and an image either landed locally or it did not.
+missing=()
+for driver in $DRIVERS; do
+  if ! docker image inspect "olakego/source-$driver:$TAG" >/dev/null 2>&1; then
+    echo "no cached image for $driver at $TAG; it will be built"
+    missing+=("$driver")
+  fi
+done
+
+if [ ${#missing[@]} -eq 0 ]; then
+  echo "every image for $TAG came from $CACHE_REPO"
+  exit 0
+fi
+
+echo "building the iceberg jar for $TAG..."
+started=$SECONDS
+if ! jar_log=$(make -C "$SRC" iceberg.jar 2>&1); then
+  echo "$jar_log"
+  echo "::error::failed to build the iceberg jar at $TAG"
+  exit 1
+fi
+echo "built the iceberg jar in $((SECONDS - started))s"
+
+# TODO: we can use make command for build once this PR merges as local builds gets tagged as olake/source... instead of olakego/source...
+build_driver_image() {
+  local log="$WORK/$1.log"
+  if ! docker buildx build --progress=plain --cache-from type=gha,scope=olake-base \
+      --load --build-arg DRIVER_NAME="$1" -t "olakego/source-$1:$TAG" "$SRC" > "$log" 2>&1; then
+    echo "::group::build $1 -- FAILED"
+    cat "$log"
+    echo "::endgroup::"
+    return 1
+  fi
+
+  # Non-fatal: a fork's token is read-only, and failing to publish only costs the next run the
+  # build this one just did.
+  docker tag "olakego/source-$1:$TAG" "$CACHE_REPO/source-$1:$TAG"
+  docker push "$CACHE_REPO/source-$1:$TAG" >> "$log" 2>&1 \
+    || echo "::notice::could not publish source-$1:$TAG (read-only token?)"
+
+  echo "::group::build $1 -- built and published"
+  cat "$log"
+  echo "::endgroup::"
+}
+export -f build_driver_image
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+export TAG SRC CACHE_REPO WORK
+
+echo "building ${missing[*]}"
+printf '%s\n' "${missing[@]}" | xargs -P "${MAX_PARALLEL:-3}" -I{} bash -euc 'build_driver_image {}'

.github/actions/detect-drivers/action.yml

@@ -14,6 +14,10 @@ outputs:
   drivers:
     description: JSON array of drivers to run; empty when a change touches no tested driver.
     value: ${{ steps.resolve.outputs.drivers }}
+  driver-labels:
+    description: >
+      JSON object mapping each driver to its display name
+    value: ${{ steps.resolve.outputs.driver-labels }}
 
 runs:
   using: composite

.github/actions/detect-drivers/detect-drivers.sh

@@ -26,3 +26,10 @@ fi
 drivers=$(printf '%s\n' $selected | sort -u | jq -Rc '[., inputs] | map(select(. != ""))')
 echo "drivers=$drivers" >> "$GITHUB_OUTPUT"
 echo "Affected drivers: $drivers"
+
+title_case() {
+  jq -c 'map({key: ., value: ((.[0:1] | ascii_upcase) + .[1:])}) | from_entries'
+}
+
+labels=$(printf '%s' "$drivers" | title_case)
+echo "driver-labels=$labels" >> "$GITHUB_OUTPUT"

.github/actions/driver-image/action.yml

@@ -0,0 +1,36 @@
+name: Driver image artifact
+description: >
+  Exports the driver image this run built and uploads it as a run artifact
+
+inputs:
+  driver:
+    description: Driver whose image to export.
+    required: true
+  tag:
+    description: >
+      Tag to export. The suites build and run the current code as `local`; a released baseline or a
+      commit-tagged build can be exported by naming its tag instead.
+    required: false
+    default: local
+  retention-days:
+    description: How long the artifact lives. A day is already far longer than the run that reads it.
+    required: false
+    default: '1'
+
+runs:
+  using: composite
+  steps:
+    - name: Export the image
+      shell: bash
+      env:
+        IMAGE: olakego/source-${{ inputs.driver }}:${{ inputs.tag }}
+        TAR: ${{ runner.temp }}/source-${{ inputs.driver }}.tar
+      run: docker save -o "$TAR" "$IMAGE"
+
+    - name: Upload the image
+      uses: actions/upload-artifact@v7
+      with:
+        name: driver-image-${{ inputs.driver }}
+        path: ${{ runner.temp }}/source-${{ inputs.driver }}.tar
+        retention-days: ${{ inputs.retention-days }}
+        overwrite: true

.github/rulesets/state-version-approval.json

@@ -0,0 +1,49 @@
+{
+  "name": "State version approval",
+  "target": "branch",
+  "enforcement": "active",
+  "conditions": {
+    "ref_name": {
+      "include": [
+        "refs/heads/master",
+        "refs/heads/staging"
+      ],
+      "exclude": []
+    }
+  },
+  "rules": [
+    {
+      "type": "pull_request",
+      "parameters": {
+        "required_approving_review_count": 1,
+        "dismiss_stale_reviews_on_push": true,
+        "require_code_owner_review": true,
+        "require_last_push_approval": true,
+        "required_review_thread_resolution": true,
+        "required_reviewers": [
+          {
+            "file_patterns": [
+              "constants/state-versions.json",
+              "tests/testutils/compatibility/compatibility_rules.json"
+            ],
+            "minimum_approvals": 2,
+            "reviewer": {
+              "id": 0,
+              "type": "Team"
+            }
+          },
+          {
+            "file_patterns": [
+              ".github/CODEOWNERS"
+            ],
+            "minimum_approvals": 2,
+            "reviewer": {
+              "id": 0,
+              "type": "Team"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

.github/scripts/restore-driver-images.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+TAG=${TAG:-local}
+ARTIFACT_PREFIX=${ARTIFACT_PREFIX:-driver-image}
+
+restore_single_image() {
+  local dir="$RUNNER_TEMP/images/$1"
+  gh run download "$GITHUB_RUN_ID" -n "$ARTIFACT_PREFIX-$1" -D "$dir" >/dev/null 2>&1 || return 0
+  docker load -i "$dir/source-$1.tar"
+}
+export -f restore_single_image
+export GH_TOKEN GITHUB_RUN_ID RUNNER_TEMP ARTIFACT_PREFIX
+printf '%s\n' $DRIVERS | xargs -P "${MAX_PARALLEL:-4}" -I{} bash -c 'restore_single_image {}'
+
+missing=()
+for driver in $DRIVERS; do
+  docker image inspect "olakego/source-$driver:$TAG" >/dev/null 2>&1 || missing+=("$driver")
+done
+
+if [ ${#missing[@]} -eq 0 ]; then
+  echo "every driver image came from this run's integration build"
+  exit 0
+fi
+
+echo "::notice::no artifact for ${missing[*]}; building them here"
+make -j --output-sync=target docker.all.build DRIVERS="${missing[*]}" IMAGE_TAG="$TAG" \
+  DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --load"