feat: sequence-store publisher e2e suite

.github/configs/kurtosis-sequencer-e2e.yml

@@ -0,0 +1,36 @@
+polygon_pos_package:
+  network_params:
+    # 4s blocks: the pre-Rio bootstrap (blocks 1-127, publisher fully gated)
+    # forks under host load at faster cadences on small runners. Rio stays at
+    # the package default of block 128, so the fork lands ~8.5 minutes in.
+    el_block_interval_seconds: 4
+
+  # In-enclave sequence store (requires kurtosis-pos with the sequencer
+  # launcher, main as of 0cdede0). The image defaults to seqstore:local,
+  # which CI loads by pulling ghcr.io/0xpolygon/sequence-store and
+  # retagging (or building the private repo from source).
+  sequence_store_params:
+    image: seqstore:local
+    redpanda_count: 1
+    gateway_count: 1
+
+  participants:
+    # 4 publishing validators: 3/4 stake tolerates one dead bor while
+    # heimdall's downtime span rotation stays quorate.
+    - kind: validator
+      cl_type: heimdall-v2
+      cl_image: heimdall-v2:local
+      el_type: bor
+      el_image: bor:local
+      el_bor_use_sequence_store: true
+      count: 4
+
+    # 1 RPC node on the same image; it follows the gateway stream for
+    # preconf receipts once the consumer phase lands.
+    - kind: rpc
+      cl_type: heimdall-v2
+      cl_image: heimdall-v2:local
+      el_type: bor
+      el_image: bor:local
+      el_bor_use_sequence_store: true
+      count: 1

.github/workflows/kurtosis-sequencer-e2e.yml

@@ -0,0 +1,188 @@
+name: Kurtosis Sequencer E2E Tests
+
+# Exercises the sequence-store publisher end to end: publishers live after
+# Rio, stall-free production through a store outage, and the recovery
+# backfill floored at milestone finality.
+#
+# Unlike the sibling workflows this one is path-scoped: it builds bor from
+# the sequencing feature branch (not develop), so it should gate only
+# changes to its own suite rather than every PR in this repo.
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - '.github/workflows/kurtosis-sequencer-e2e.yml'
+      - '.github/configs/kurtosis-sequencer-e2e.yml'
+      - 'configs/kurtosis-sequencer-e2e.yml'
+      - 'tests/sequencer_tests/**'
+  pull_request:
+    branches:
+      - 'main'
+    types: [opened, synchronize]
+    paths:
+      - '.github/workflows/kurtosis-sequencer-e2e.yml'
+      - '.github/configs/kurtosis-sequencer-e2e.yml'
+      - 'configs/kurtosis-sequencer-e2e.yml'
+      - 'tests/sequencer_tests/**'
+  workflow_dispatch:
+
+concurrency:
+  group: kurtosis-sequencer-e2e-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  ENCLAVE_NAME: kurtosis-sequencer-e2e
+  # The sequence-store package versions are not publicly listable; pin a
+  # released tag here once one is published.
+  SEQUENCE_STORE_TAG: latest
+
+jobs:
+  build-bor:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout bor
+        uses: actions/checkout@v5
+        with:
+          repository: 0xPolygon/bor
+          # The sequencer publisher lives on the sequencing branch
+          # (PR #2355); switch this to develop once it merges there.
+          ref: sequencing
+
+      - name: Build bor docker image
+        run: docker build -t bor:local --file Dockerfile .
+
+      - name: Save bor docker image
+        run: docker save bor:local | gzip > bor-image.tar.gz
+
+      - name: Upload bor docker image
+        uses: actions/upload-artifact@v4
+        with:
+          name: bor-image
+          path: bor-image.tar.gz
+          retention-days: 1
+
+  build-heimdall-v2:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout heimdall-v2
+        uses: actions/checkout@v5
+        with:
+          repository: 0xPolygon/heimdall-v2
+          ref: develop
+
+      - name: Build heimdall-v2 docker image
+        run: docker build -t heimdall-v2:local --file Dockerfile .
+
+      - name: Save heimdall-v2 docker image
+        run: docker save heimdall-v2:local | gzip > heimdall-v2-image.tar.gz
+
+      - name: Upload heimdall-v2 docker image
+        uses: actions/upload-artifact@v4
+        with:
+          name: heimdall-v2-image
+          path: heimdall-v2-image.tar.gz
+          retention-days: 1
+
+  e2e-tests:
+    needs:
+      - build-bor
+      - build-heimdall-v2
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    # packages: read lets the default GITHUB_TOKEN pull the sequence-store
+    # ghcr package once this repo is granted access on it.
+    permissions:
+      contents: read
+      actions: write
+      id-token: write
+      packages: read
+    steps:
+      # Set up kurtosis. The sequence-store launcher landed after v1.3.4;
+      # move this pin to the next kurtosis-pos release tag once cut.
+      - name: Checkout kurtosis-pos
+        uses: actions/checkout@v5
+        with:
+          repository: 0xPolygon/kurtosis-pos
+          ref: 0cdede09c54608294c877f3b5081971ca95d389c
+
+      - name: Pre kurtosis run
+        uses: ./.github/actions/kurtosis/setup
+        with:
+          main_branch: main
+
+      - name: Checkout pos-workflows
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ github.sha }}
+          path: pos-workflows
+
+      - name: Copy kurtosis config
+        run: cp ./pos-workflows/.github/configs/kurtosis-sequencer-e2e.yml ./kurtosis-sequencer-e2e.yml
+
+      # Load images
+      - name: Download bor docker image
+        uses: actions/download-artifact@v4
+        with:
+          name: bor-image
+
+      - name: Download heimdall-v2 docker image
+        uses: actions/download-artifact@v4
+        with:
+          name: heimdall-v2-image
+
+      - name: Load bor docker image
+        run: |
+          gunzip -c bor-image.tar.gz | docker load
+          echo "Loaded bor docker image:"
+          docker images bor:local --format "{{.ID}} {{.CreatedAt}}"
+
+      - name: Load heimdall-v2 docker image
+        run: |
+          gunzip -c heimdall-v2-image.tar.gz | docker load
+          echo "Loaded heimdall-v2 docker image:"
+          docker images heimdall-v2:local --format "{{.ID}} {{.CreatedAt}}"
+
+      # The sequence store is a private repo whose container lives at
+      # ghcr.io/0xpolygon/sequence-store. One-time setup for THIS repo,
+      # either:
+      #   - grant 0xPolygon/pos-workflows read access on that package
+      #     (package settings -> Manage Actions access -> add repo, role
+      #     Read) so the default GITHUB_TOKEN works — the grant made for
+      #     the bor repo does not carry over; or
+      #   - provision a fine-grained PAT with read:packages as the
+      #     SEQUENCE_STORE_GHCR_TOKEN secret.
+      - name: Pull sequence-store image
+        env:
+          GHCR_TOKEN: ${{ secrets.SEQUENCE_STORE_GHCR_TOKEN != '' && secrets.SEQUENCE_STORE_GHCR_TOKEN || secrets.GITHUB_TOKEN }}
+        run: |
+          echo "$GHCR_TOKEN" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+          docker pull ghcr.io/0xpolygon/sequence-store:${{ env.SEQUENCE_STORE_TAG }}
+          docker tag ghcr.io/0xpolygon/sequence-store:${{ env.SEQUENCE_STORE_TAG }} seqstore:local
+          docker images seqstore:local --format "{{.ID}} {{.CreatedAt}}"
+
+      - name: Kurtosis run
+        run: kurtosis run --args-file=kurtosis-sequencer-e2e.yml --enclave ${{ env.ENCLAVE_NAME }} .
+
+      - name: Inspect enclave
+        run: kurtosis enclave inspect ${{ env.ENCLAVE_NAME }}
+
+      - name: Run sequencer e2e tests
+        id: sequencer-tests
+        working-directory: pos-workflows/tests/sequencer_tests
+        run: bash kurtosis_sequencer_test.sh
+
+      - name: Collect network diagnostics and dump devnet state
+        if: always() && steps.sequencer-tests.outcome == 'failure'
+        uses: ./pos-workflows/.github/actions/network-diagnostics-and-state-dump
+        with:
+          enclave_name: ${{ env.ENCLAVE_NAME }}
+
+      - name: Post kurtosis run
+        if: always()
+        uses: ./.github/actions/kurtosis/cleanup
+        with:
+          main_branch: main
+          enclave_name: ${{ env.ENCLAVE_NAME }}

configs/kurtosis-sequencer-e2e.yml

@@ -0,0 +1,36 @@
+polygon_pos_package:
+  network_params:
+    # 4s blocks: the pre-Rio bootstrap (blocks 1-127, publisher fully gated)
+    # forks under host load at faster cadences on small runners. Rio stays at
+    # the package default of block 128, so the fork lands ~8.5 minutes in.
+    el_block_interval_seconds: 4
+
+  # In-enclave sequence store (requires kurtosis-pos with the sequencer
+  # launcher, main as of 0cdede0). The image defaults to seqstore:local,
+  # which CI loads by pulling ghcr.io/0xpolygon/sequence-store and
+  # retagging (or building the private repo from source).
+  sequence_store_params:
+    image: seqstore:local
+    redpanda_count: 1
+    gateway_count: 1
+
+  participants:
+    # 4 publishing validators: 3/4 stake tolerates one dead bor while
+    # heimdall's downtime span rotation stays quorate.
+    - kind: validator
+      cl_type: heimdall-v2
+      cl_image: heimdall-v2:local
+      el_type: bor
+      el_image: bor:local
+      el_bor_use_sequence_store: true
+      count: 4
+
+    # 1 RPC node on the same image; it follows the gateway stream for
+    # preconf receipts once the consumer phase lands.
+    - kind: rpc
+      cl_type: heimdall-v2
+      cl_image: heimdall-v2:local
+      el_type: bor
+      el_image: bor:local
+      el_bor_use_sequence_store: true
+      count: 1