feat: Add lane off-chain readiness preflight

deployment/preflight/lane_prereqs.go

@@ -0,0 +1,154 @@
+// Package preflight holds read-only checks that run before a lane is configured.
+//
+// The lane changesets in chainlink-ccip validate the topology and the on-chain state they
+// are about to write. They cannot see whether the off-chain components that serve a lane
+// exist, because the job state lives in this module's env metadata. These checks close
+// that gap: they answer "are the verifier and executor jobs for this chain deployed and
+// approved" before a lane is wired up.
+package preflight
+
+import (
+	"errors"
+	"fmt"
+	"slices"
+	"strconv"
+	"strings"
+
+	"github.com/smartcontractkit/chainlink-deployments-framework/datastore"
+
+	ccipoffchain "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/offchain"
+
+	ccvdeployment "github.com/smartcontractkit/chainlink-ccv/deployment"
+	"github.com/smartcontractkit/chainlink-ccv/deployment/shared"
+)
+
+var (
+	// ErrNoVerifierJob means a committee NOP has no verifier job for a lane chain's committee.
+	ErrNoVerifierJob = errors.New("committee NOP has no verifier job")
+	// ErrNoExecutorJob means a pool NOP has no executor job for a lane chain's executor pool.
+	ErrNoExecutorJob = errors.New("executor pool NOP has no executor job")
+	// ErrJobNotApproved means a job exists but its latest proposal is not approved.
+	ErrJobNotApproved = errors.New("job proposal is not approved")
+	// ErrJobMissingChain means a deployed job spec does not reference the lane chain, so the
+	// job predates the chain being added to the topology.
+	ErrJobMissingChain = errors.New("deployed job spec does not cover chain")
+)
+
+// CheckLaneOffchainReadiness reports whether the off-chain components serving the given
+// lane chains are deployed and approved.
+//
+// For each chain it walks the committees and executor pools that cover it in the topology
+// and, for every member NOP, requires a job that is approved and whose deployed spec
+// references the chain selector. A spec that does not mention the chain means the job was
+// generated before the chain joined the topology and has not been re-applied.
+//
+// Chains absent from every committee and pool are skipped: that is the lane changeset's
+// topology coverage check, and reporting it here would only duplicate the error.
+func CheckLaneOffchainReadiness(
+	ds datastore.DataStore,
+	topology *ccipoffchain.EnvironmentTopology,
+	chainSelectors []uint64,
+) error {
+	if topology == nil || topology.NOPTopology == nil {
+		return errors.New("topology is required")
+	}
+
+	// An environment that has never had CCV metadata written has no jobs, which the
+	// per-NOP checks below report as the missing jobs they are.
+	jobs, err := ccvdeployment.GetAllJobs(ds)
+	if err != nil && !errors.Is(err, datastore.ErrEnvMetadataNotSet) {
+		return fmt.Errorf("failed to read job state: %w", err)
+	}
+
+	for _, selector := range chainSelectors {
+		chainKey := strconv.FormatUint(selector, 10)
+
+		for _, qualifier := range committeeQualifiersCovering(topology, chainKey) {
+			chainCommittee := topology.NOPTopology.Committees[qualifier].ChainConfigs[chainKey]
+			for _, alias := range chainCommittee.NOPAliases {
+				err := requireJob(jobs, shared.NOPAlias(alias), jobSuffixVerifier, qualifier, chainKey, ErrNoVerifierJob)
+				if err != nil {
+					return fmt.Errorf("chain %s committee %q: %w", chainKey, qualifier, err)
+				}
+			}
+		}
+
+		for _, poolName := range executorPoolsCovering(topology, chainKey) {
+			chainPool := topology.ExecutorPools[poolName].ChainConfigs[chainKey]
+			for _, alias := range chainPool.NOPAliases {
+				err := requireJob(jobs, shared.NOPAlias(alias), jobSuffixExecutor, poolName, chainKey, ErrNoExecutorJob)
+				if err != nil {
+					return fmt.Errorf("chain %s executor pool %q: %w", chainKey, poolName, err)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+const (
+	jobSuffixVerifier = "verifier"
+	jobSuffixExecutor = "executor"
+)
+
+// requireJob finds the NOP's job for a qualifier and checks it is approved and covers the
+// chain. Job IDs are "<nop>-<qualifier>-<kind>", with an optional aggregator name in the
+// middle for per-aggregator verifier jobs, so they are matched on their parts rather than
+// reconstructed.
+func requireJob(
+	jobs shared.NOPJobs,
+	alias shared.NOPAlias,
+	kind string,
+	qualifier string,
+	chainKey string,
+	errMissing error,
+) error {
+	var found *shared.JobInfo
+	for jobID, info := range jobs[alias] {
+		if jobMatches(string(jobID), kind, qualifier) {
+			found = &info
+			break
+		}
+	}
+	if found == nil {
+		return fmt.Errorf("NOP %q: %w", alias, errMissing)
+	}
+	if !found.IsRunning() || found.LatestStatus() != shared.JobProposalStatusApproved {
+		return fmt.Errorf("NOP %q job %q status %q: %w",
+			alias, found.JobID, found.LatestStatus(), ErrJobNotApproved)
+	}
+	if !strings.Contains(found.Spec, chainKey) {
+		return fmt.Errorf("NOP %q job %q: %w", alias, found.JobID, ErrJobMissingChain)
+	}
+	return nil
+}
+
+// jobMatches reports whether a job ID belongs to the given kind and qualifier.
+func jobMatches(jobID, kind, qualifier string) bool {
+	return strings.HasSuffix(jobID, "-"+kind) && strings.Contains(jobID, "-"+qualifier+"-")
+}
+
+// committeeQualifiersCovering returns, sorted, the committees whose chain_configs include the chain.
+func committeeQualifiersCovering(topology *ccipoffchain.EnvironmentTopology, chainKey string) []string {
+	var qualifiers []string
+	for qualifier, committee := range topology.NOPTopology.Committees {
+		if _, ok := committee.ChainConfigs[chainKey]; ok {
+			qualifiers = append(qualifiers, qualifier)
+		}
+	}
+	slices.Sort(qualifiers)
+	return qualifiers
+}
+
+// executorPoolsCovering returns, sorted, the executor pools whose chain_configs include the chain.
+func executorPoolsCovering(topology *ccipoffchain.EnvironmentTopology, chainKey string) []string {
+	var pools []string
+	for poolName, pool := range topology.ExecutorPools {
+		if _, ok := pool.ChainConfigs[chainKey]; ok {
+			pools = append(pools, poolName)
+		}
+	}
+	slices.Sort(pools)
+	return pools
+}

deployment/preflight/lane_prereqs_test.go

@@ -0,0 +1,178 @@
+package preflight_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/smartcontractkit/chainlink-deployments-framework/datastore"
+
+	ccipoffchain "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/offchain"
+
+	ccvdeployment "github.com/smartcontractkit/chainlink-ccv/deployment"
+	"github.com/smartcontractkit/chainlink-ccv/deployment/preflight"
+	"github.com/smartcontractkit/chainlink-ccv/deployment/shared"
+)
+
+const (
+	laneChain   = uint64(6898391096552792247) // ethereum-testnet-sepolia-zksync-1
+	remoteChain = uint64(3478487238524512106) // ethereum-testnet-sepolia-arbitrum-1
+	nopAlias    = shared.NOPAlias("nop-1")
+	committeeQ  = "default"
+	executorQ   = "default"
+)
+
+// verifierSpecFor builds a job spec that references the given chain selectors, which is
+// how a deployed verifier spec records the chains it serves.
+func verifierSpecFor(selectors ...uint64) string {
+	var spec strings.Builder
+	spec.WriteString("[verifier]\n")
+	for _, selector := range selectors {
+		fmt.Fprintf(&spec, "[verifier.chains.%d]\nenabled = true\n", selector)
+	}
+	return spec.String()
+}
+
+func jobInfo(jobID shared.JobID, spec string, status shared.JobProposalStatus) shared.JobInfo {
+	info := shared.JobInfo{
+		JobID:    jobID,
+		NOPAlias: nopAlias,
+		Spec:     spec,
+	}
+	if status != "" {
+		info.ActiveProposalID = "proposal-1"
+		info.Proposals = map[string]shared.ProposalRevision{
+			"proposal-1": {ProposalID: "proposal-1", Revision: 1, Status: status},
+		}
+	}
+	return info
+}
+
+// topologyFor covers both lane chains in one committee and one executor pool.
+func topologyFor() *ccipoffchain.EnvironmentTopology {
+	chainCommittee := map[string]ccipoffchain.ChainCommitteeConfig{
+		fmt.Sprintf("%d", laneChain):   {NOPAliases: []string{string(nopAlias)}, Threshold: 1},
+		fmt.Sprintf("%d", remoteChain): {NOPAliases: []string{string(nopAlias)}, Threshold: 1},
+	}
+	chainPool := map[string]ccipoffchain.ChainExecutorPoolConfig{
+		fmt.Sprintf("%d", laneChain):   {NOPAliases: []string{string(nopAlias)}},
+		fmt.Sprintf("%d", remoteChain): {NOPAliases: []string{string(nopAlias)}},
+	}
+	return &ccipoffchain.EnvironmentTopology{
+		NOPTopology: &ccipoffchain.NOPTopology{
+			NOPs: []ccipoffchain.NOPConfig{{Alias: string(nopAlias), Name: "nop-1-name"}},
+			Committees: map[string]ccipoffchain.CommitteeConfig{
+				committeeQ: {Qualifier: committeeQ, ChainConfigs: chainCommittee},
+			},
+		},
+		ExecutorPools: map[string]ccipoffchain.ExecutorPoolConfig{
+			executorQ: {ChainConfigs: chainPool},
+		},
+	}
+}
+
+func dataStoreWithJobs(t *testing.T, jobs ...shared.JobInfo) datastore.DataStore {
+	t.Helper()
+	ds := datastore.NewMemoryDataStore()
+	if len(jobs) > 0 {
+		require.NoError(t, ccvdeployment.SaveJobs(ds, jobs))
+	}
+	return ds.Seal()
+}
+
+// The zkSync <-> Arbitrum lane was configured on chain while zkSync had no committee
+// verifier or executor jobs deployed, so messages were never verified.
+func TestCheckLaneOffchainReadiness(t *testing.T) {
+	verifierJob := shared.JobID(fmt.Sprintf("%s-%s-verifier", nopAlias, committeeQ))
+	executorJob := shared.JobID(fmt.Sprintf("%s-%s-executor", nopAlias, executorQ))
+	bothChains := verifierSpecFor(laneChain, remoteChain)
+
+	tests := []struct {
+		name    string
+		jobs    []shared.JobInfo
+		wantErr error
+	}{
+		{
+			name: "Success - verifier and executor jobs approved and cover the chain",
+			jobs: []shared.JobInfo{
+				jobInfo(verifierJob, bothChains, shared.JobProposalStatusApproved),
+				jobInfo(executorJob, bothChains, shared.JobProposalStatusApproved),
+			},
+		},
+		{
+			name:    "Failure - no jobs deployed at all",
+			jobs:    nil,
+			wantErr: preflight.ErrNoVerifierJob,
+		},
+		{
+			name: "Failure - verifier job missing for the committee",
+			jobs: []shared.JobInfo{
+				jobInfo(executorJob, bothChains, shared.JobProposalStatusApproved),
+			},
+			wantErr: preflight.ErrNoVerifierJob,
+		},
+		{
+			name: "Failure - verifier job still pending approval",
+			jobs: []shared.JobInfo{
+				jobInfo(verifierJob, bothChains, shared.JobProposalStatusPending),
+				jobInfo(executorJob, bothChains, shared.JobProposalStatusApproved),
+			},
+			wantErr: preflight.ErrJobNotApproved,
+		},
+		{
+			name: "Failure - verifier job rejected",
+			jobs: []shared.JobInfo{
+				jobInfo(verifierJob, bothChains, shared.JobProposalStatusRejected),
+				jobInfo(executorJob, bothChains, shared.JobProposalStatusApproved),
+			},
+			wantErr: preflight.ErrJobNotApproved,
+		},
+		{
+			name: "Failure - verifier spec does not cover the lane chain",
+			jobs: []shared.JobInfo{
+				jobInfo(verifierJob, verifierSpecFor(remoteChain), shared.JobProposalStatusApproved),
+				jobInfo(executorJob, bothChains, shared.JobProposalStatusApproved),
+			},
+			wantErr: preflight.ErrJobMissingChain,
+		},
+		{
+			name: "Failure - executor job missing for the pool",
+			jobs: []shared.JobInfo{
+				jobInfo(verifierJob, bothChains, shared.JobProposalStatusApproved),
+			},
+			wantErr: preflight.ErrNoExecutorJob,
+		},
+		{
+			name: "Failure - executor spec does not cover the lane chain",
+			jobs: []shared.JobInfo{
+				jobInfo(verifierJob, bothChains, shared.JobProposalStatusApproved),
+				jobInfo(executorJob, verifierSpecFor(remoteChain), shared.JobProposalStatusApproved),
+			},
+			wantErr: preflight.ErrJobMissingChain,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			err := preflight.CheckLaneOffchainReadiness(
+				dataStoreWithJobs(t, tc.jobs...), topologyFor(), []uint64{laneChain},
+			)
+			if tc.wantErr != nil {
+				require.ErrorIs(t, err, tc.wantErr)
+				require.Contains(t, err.Error(), fmt.Sprintf("%d", laneChain))
+				return
+			}
+			require.NoError(t, err)
+		})
+	}
+}
+
+// A chain outside every committee and pool is the lane topology check's job, not this
+// one, so readiness passes rather than reporting a missing job for it.
+func TestCheckLaneOffchainReadiness_ChainNotInTopology(t *testing.T) {
+	ds := dataStoreWithJobs(t)
+	err := preflight.CheckLaneOffchainReadiness(ds, topologyFor(), []uint64{99})
+	require.NoError(t, err)
+}