From ad39b46cd922e92091546ef8a2711213dc5dc954 Mon Sep 17 00:00:00 2001 From: Jasmin Bakalovic Date: Tue, 8 Sep 2026 10:08:33 -0700 Subject: [PATCH 1/2] CCIP-13390:Aadd lane off-chain readiness preflight New preflight package checks that the verifier and executor jobs serving a lane chain exist, are approved, and that their deployed spec covers that chain. The lane changesets in chainlink-ccip cannot read this job state, so this closes the gap before a lane is wired up. --- deployment/preflight/lane_prereqs.go | 154 +++++++++++++++++++ deployment/preflight/lane_prereqs_test.go | 176 ++++++++++++++++++++++ 2 files changed, 330 insertions(+) create mode 100644 deployment/preflight/lane_prereqs.go create mode 100644 deployment/preflight/lane_prereqs_test.go diff --git a/deployment/preflight/lane_prereqs.go b/deployment/preflight/lane_prereqs.go new file mode 100644 index 000000000..16f59cb5d --- /dev/null +++ b/deployment/preflight/lane_prereqs.go @@ -0,0 +1,154 @@ +// Package preflight holds read-only checks that run before a lane is configured. +// +// The lane changesets in chainlink-ccip validate the topology and the on-chain state they +// are about to write. They cannot see whether the off-chain components that serve a lane +// exist, because the job state lives in this module's env metadata. These checks close +// that gap: they answer "are the verifier and executor jobs for this chain deployed and +// approved" before a lane is wired up. +package preflight + +import ( + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/smartcontractkit/chainlink-deployments-framework/datastore" + + ccipoffchain "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/offchain" + + ccvdeployment "github.com/smartcontractkit/chainlink-ccv/deployment" + "github.com/smartcontractkit/chainlink-ccv/deployment/shared" +) + +var ( + // ErrNoVerifierJob means a committee NOP has no verifier job for a lane chain's committee. + ErrNoVerifierJob = errors.New("committee NOP has no verifier job") + // ErrNoExecutorJob means a pool NOP has no executor job for a lane chain's executor pool. + ErrNoExecutorJob = errors.New("executor pool NOP has no executor job") + // ErrJobNotApproved means a job exists but its latest proposal is not approved. + ErrJobNotApproved = errors.New("job proposal is not approved") + // ErrJobMissingChain means a deployed job spec does not reference the lane chain, so the + // job predates the chain being added to the topology. + ErrJobMissingChain = errors.New("deployed job spec does not cover chain") +) + +// CheckLaneOffchainReadiness reports whether the off-chain components serving the given +// lane chains are deployed and approved. +// +// For each chain it walks the committees and executor pools that cover it in the topology +// and, for every member NOP, requires a job that is approved and whose deployed spec +// references the chain selector. A spec that does not mention the chain means the job was +// generated before the chain joined the topology and has not been re-applied. +// +// Chains absent from every committee and pool are skipped: that is the lane changeset's +// topology coverage check, and reporting it here would only duplicate the error. +func CheckLaneOffchainReadiness( + ds datastore.DataStore, + topology *ccipoffchain.EnvironmentTopology, + chainSelectors []uint64, +) error { + if topology == nil || topology.NOPTopology == nil { + return errors.New("topology is required") + } + + // An environment that has never had CCV metadata written has no jobs, which the + // per-NOP checks below report as the missing jobs they are. + jobs, err := ccvdeployment.GetAllJobs(ds) + if err != nil && !errors.Is(err, datastore.ErrEnvMetadataNotSet) { + return fmt.Errorf("failed to read job state: %w", err) + } + + for _, selector := range chainSelectors { + chainKey := strconv.FormatUint(selector, 10) + + for _, qualifier := range committeeQualifiersCovering(topology, chainKey) { + chainCommittee := topology.NOPTopology.Committees[qualifier].ChainConfigs[chainKey] + for _, alias := range chainCommittee.NOPAliases { + err := requireJob(jobs, shared.NOPAlias(alias), jobSuffixVerifier, qualifier, chainKey, ErrNoVerifierJob) + if err != nil { + return fmt.Errorf("chain %s committee %q: %w", chainKey, qualifier, err) + } + } + } + + for _, poolName := range executorPoolsCovering(topology, chainKey) { + chainPool := topology.ExecutorPools[poolName].ChainConfigs[chainKey] + for _, alias := range chainPool.NOPAliases { + err := requireJob(jobs, shared.NOPAlias(alias), jobSuffixExecutor, poolName, chainKey, ErrNoExecutorJob) + if err != nil { + return fmt.Errorf("chain %s executor pool %q: %w", chainKey, poolName, err) + } + } + } + } + + return nil +} + +const ( + jobSuffixVerifier = "verifier" + jobSuffixExecutor = "executor" +) + +// requireJob finds the NOP's job for a qualifier and checks it is approved and covers the +// chain. Job IDs are "--", with an optional aggregator name in the +// middle for per-aggregator verifier jobs, so they are matched on their parts rather than +// reconstructed. +func requireJob( + jobs shared.NOPJobs, + alias shared.NOPAlias, + kind string, + qualifier string, + chainKey string, + errMissing error, +) error { + var found *shared.JobInfo + for jobID, info := range jobs[alias] { + if jobMatches(string(jobID), kind, qualifier) { + found = &info + break + } + } + if found == nil { + return fmt.Errorf("NOP %q: %w", alias, errMissing) + } + if !found.IsRunning() || found.LatestStatus() != shared.JobProposalStatusApproved { + return fmt.Errorf("NOP %q job %q status %q: %w", + alias, found.JobID, found.LatestStatus(), ErrJobNotApproved) + } + if !strings.Contains(found.Spec, chainKey) { + return fmt.Errorf("NOP %q job %q: %w", alias, found.JobID, ErrJobMissingChain) + } + return nil +} + +// jobMatches reports whether a job ID belongs to the given kind and qualifier. +func jobMatches(jobID, kind, qualifier string) bool { + return strings.HasSuffix(jobID, "-"+kind) && strings.Contains(jobID, "-"+qualifier+"-") +} + +// committeeQualifiersCovering returns, sorted, the committees whose chain_configs include the chain. +func committeeQualifiersCovering(topology *ccipoffchain.EnvironmentTopology, chainKey string) []string { + var qualifiers []string + for qualifier, committee := range topology.NOPTopology.Committees { + if _, ok := committee.ChainConfigs[chainKey]; ok { + qualifiers = append(qualifiers, qualifier) + } + } + slices.Sort(qualifiers) + return qualifiers +} + +// executorPoolsCovering returns, sorted, the executor pools whose chain_configs include the chain. +func executorPoolsCovering(topology *ccipoffchain.EnvironmentTopology, chainKey string) []string { + var pools []string + for poolName, pool := range topology.ExecutorPools { + if _, ok := pool.ChainConfigs[chainKey]; ok { + pools = append(pools, poolName) + } + } + slices.Sort(pools) + return pools +} diff --git a/deployment/preflight/lane_prereqs_test.go b/deployment/preflight/lane_prereqs_test.go new file mode 100644 index 000000000..bc3ea04fd --- /dev/null +++ b/deployment/preflight/lane_prereqs_test.go @@ -0,0 +1,176 @@ +package preflight_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-deployments-framework/datastore" + + ccipoffchain "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/offchain" + + ccvdeployment "github.com/smartcontractkit/chainlink-ccv/deployment" + "github.com/smartcontractkit/chainlink-ccv/deployment/preflight" + "github.com/smartcontractkit/chainlink-ccv/deployment/shared" +) + +const ( + laneChain = uint64(6898391096552792247) // ethereum-testnet-sepolia-zksync-1 + remoteChain = uint64(3478487238524512106) // ethereum-testnet-sepolia-arbitrum-1 + nopAlias = shared.NOPAlias("nop-1") + committeeQ = "default" + executorQ = "default" +) + +// verifierSpecFor builds a job spec that references the given chain selectors, which is +// how a deployed verifier spec records the chains it serves. +func verifierSpecFor(selectors ...uint64) string { + spec := "[verifier]\n" + for _, selector := range selectors { + spec += fmt.Sprintf("[verifier.chains.%d]\nenabled = true\n", selector) + } + return spec +} + +func jobInfo(jobID shared.JobID, spec string, status shared.JobProposalStatus) shared.JobInfo { + info := shared.JobInfo{ + JobID: jobID, + NOPAlias: nopAlias, + Spec: spec, + } + if status != "" { + info.ActiveProposalID = "proposal-1" + info.Proposals = map[string]shared.ProposalRevision{ + "proposal-1": {ProposalID: "proposal-1", Revision: 1, Status: status}, + } + } + return info +} + +// topologyFor covers both lane chains in one committee and one executor pool. +func topologyFor() *ccipoffchain.EnvironmentTopology { + chainCommittee := map[string]ccipoffchain.ChainCommitteeConfig{ + fmt.Sprintf("%d", laneChain): {NOPAliases: []string{string(nopAlias)}, Threshold: 1}, + fmt.Sprintf("%d", remoteChain): {NOPAliases: []string{string(nopAlias)}, Threshold: 1}, + } + chainPool := map[string]ccipoffchain.ChainExecutorPoolConfig{ + fmt.Sprintf("%d", laneChain): {NOPAliases: []string{string(nopAlias)}}, + fmt.Sprintf("%d", remoteChain): {NOPAliases: []string{string(nopAlias)}}, + } + return &ccipoffchain.EnvironmentTopology{ + NOPTopology: &ccipoffchain.NOPTopology{ + NOPs: []ccipoffchain.NOPConfig{{Alias: string(nopAlias), Name: "nop-1-name"}}, + Committees: map[string]ccipoffchain.CommitteeConfig{ + committeeQ: {Qualifier: committeeQ, ChainConfigs: chainCommittee}, + }, + }, + ExecutorPools: map[string]ccipoffchain.ExecutorPoolConfig{ + executorQ: {ChainConfigs: chainPool}, + }, + } +} + +func dataStoreWithJobs(t *testing.T, jobs ...shared.JobInfo) datastore.DataStore { + t.Helper() + ds := datastore.NewMemoryDataStore() + if len(jobs) > 0 { + require.NoError(t, ccvdeployment.SaveJobs(ds, jobs)) + } + return ds.Seal() +} + +// The zkSync <-> Arbitrum lane was configured on chain while zkSync had no committee +// verifier or executor jobs deployed, so messages were never verified. +func TestCheckLaneOffchainReadiness(t *testing.T) { + verifierJob := shared.JobID(fmt.Sprintf("%s-%s-verifier", nopAlias, committeeQ)) + executorJob := shared.JobID(fmt.Sprintf("%s-%s-executor", nopAlias, executorQ)) + bothChains := verifierSpecFor(laneChain, remoteChain) + + tests := []struct { + name string + jobs []shared.JobInfo + wantErr error + }{ + { + name: "Success - verifier and executor jobs approved and cover the chain", + jobs: []shared.JobInfo{ + jobInfo(verifierJob, bothChains, shared.JobProposalStatusApproved), + jobInfo(executorJob, bothChains, shared.JobProposalStatusApproved), + }, + }, + { + name: "Failure - no jobs deployed at all", + jobs: nil, + wantErr: preflight.ErrNoVerifierJob, + }, + { + name: "Failure - verifier job missing for the committee", + jobs: []shared.JobInfo{ + jobInfo(executorJob, bothChains, shared.JobProposalStatusApproved), + }, + wantErr: preflight.ErrNoVerifierJob, + }, + { + name: "Failure - verifier job still pending approval", + jobs: []shared.JobInfo{ + jobInfo(verifierJob, bothChains, shared.JobProposalStatusPending), + jobInfo(executorJob, bothChains, shared.JobProposalStatusApproved), + }, + wantErr: preflight.ErrJobNotApproved, + }, + { + name: "Failure - verifier job rejected", + jobs: []shared.JobInfo{ + jobInfo(verifierJob, bothChains, shared.JobProposalStatusRejected), + jobInfo(executorJob, bothChains, shared.JobProposalStatusApproved), + }, + wantErr: preflight.ErrJobNotApproved, + }, + { + name: "Failure - verifier spec does not cover the lane chain", + jobs: []shared.JobInfo{ + jobInfo(verifierJob, verifierSpecFor(remoteChain), shared.JobProposalStatusApproved), + jobInfo(executorJob, bothChains, shared.JobProposalStatusApproved), + }, + wantErr: preflight.ErrJobMissingChain, + }, + { + name: "Failure - executor job missing for the pool", + jobs: []shared.JobInfo{ + jobInfo(verifierJob, bothChains, shared.JobProposalStatusApproved), + }, + wantErr: preflight.ErrNoExecutorJob, + }, + { + name: "Failure - executor spec does not cover the lane chain", + jobs: []shared.JobInfo{ + jobInfo(verifierJob, bothChains, shared.JobProposalStatusApproved), + jobInfo(executorJob, verifierSpecFor(remoteChain), shared.JobProposalStatusApproved), + }, + wantErr: preflight.ErrJobMissingChain, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := preflight.CheckLaneOffchainReadiness( + dataStoreWithJobs(t, tc.jobs...), topologyFor(), []uint64{laneChain}, + ) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + require.Contains(t, err.Error(), fmt.Sprintf("%d", laneChain)) + return + } + require.NoError(t, err) + }) + } +} + +// A chain outside every committee and pool is the lane topology check's job, not this +// one, so readiness passes rather than reporting a missing job for it. +func TestCheckLaneOffchainReadiness_ChainNotInTopology(t *testing.T) { + ds := dataStoreWithJobs(t) + err := preflight.CheckLaneOffchainReadiness(ds, topologyFor(), []uint64{99}) + require.NoError(t, err) +} From 4bbc66f68077cc4275bcaf36c6867adeb287fbde Mon Sep 17 00:00:00 2001 From: Jasmin Bakalovic Date: Tue, 8 Sep 2026 10:24:54 -0700 Subject: [PATCH 2/2] CCIP-13390: Resolve linter issues by introducing strings.Builder instead of string concatenation --- deployment/preflight/lane_prereqs_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/deployment/preflight/lane_prereqs_test.go b/deployment/preflight/lane_prereqs_test.go index bc3ea04fd..637ca22d4 100644 --- a/deployment/preflight/lane_prereqs_test.go +++ b/deployment/preflight/lane_prereqs_test.go @@ -2,6 +2,7 @@ package preflight_test import ( "fmt" + "strings" "testing" "github.com/stretchr/testify/require" @@ -26,11 +27,12 @@ const ( // verifierSpecFor builds a job spec that references the given chain selectors, which is // how a deployed verifier spec records the chains it serves. func verifierSpecFor(selectors ...uint64) string { - spec := "[verifier]\n" + var spec strings.Builder + spec.WriteString("[verifier]\n") for _, selector := range selectors { - spec += fmt.Sprintf("[verifier.chains.%d]\nenabled = true\n", selector) + fmt.Fprintf(&spec, "[verifier.chains.%d]\nenabled = true\n", selector) } - return spec + return spec.String() } func jobInfo(jobID shared.JobID, spec string, status shared.JobProposalStatus) shared.JobInfo {