diff --git a/internal/compiler/bundle.go b/internal/compiler/bundle.go index d5976a08..4b6e97b7 100644 --- a/internal/compiler/bundle.go +++ b/internal/compiler/bundle.go @@ -134,14 +134,14 @@ func CompileBundlePlansContext(ctx context.Context, path string, source, eventSo if job.GitHubToken == nil { continue } - if ir.Jobs[i].tokenPolicyNarrowed && !warnedReusablePermissions { - bundle.IR.Warnings = append(bundle.IR.Warnings, reusableWorkflowTokenWarning(ir.Jobs[i].reusableCall)) + if ir.Jobs[i].reusable.tokenPolicyNarrowed && !warnedReusablePermissions { + bundle.IR.Warnings = append(bundle.IR.Warnings, reusableWorkflowTokenWarning(ir.Jobs[i].reusable.reusableCall)) warnedReusablePermissions = true } - if (ir.Jobs[i].jobPermissionsIgnored || jobPermissionsIgnored(job.GitHubToken.Permissions, ir.Jobs[i].Permissions)) && !warnedJobPermissions { + if (ir.Jobs[i].reusable.jobPermissionsIgnored || jobPermissionsIgnored(job.GitHubToken.Permissions, ir.Jobs[i].Permissions)) && !warnedJobPermissions { position := ir.Jobs[i].Source.Start - if ir.Jobs[i].jobPermissionsIgnored && ir.Jobs[i].reusableCall.Line != 0 { - position = ir.Jobs[i].reusableCall + if ir.Jobs[i].reusable.jobPermissionsIgnored && ir.Jobs[i].reusable.reusableCall.Line != 0 { + position = ir.Jobs[i].reusable.reusableCall } bundle.IR.Warnings = append(bundle.IR.Warnings, jobWorkflowTokenWarning(position)) warnedJobPermissions = true diff --git a/internal/compiler/compiler.go b/internal/compiler/compiler.go index 9b63632a..c43251b2 100644 --- a/internal/compiler/compiler.go +++ b/internal/compiler/compiler.go @@ -96,10 +96,7 @@ type JobInstance struct { RemoteWorkflow *RemoteWorkflowSource `json:"remote_workflow,omitempty"` RepositoryRoot string `json:"-"` Source workflow.Span `json:"source"` - secretAuthority secretAuthority - tokenPolicyNarrowed bool - jobPermissionsIgnored bool - reusableCall workflow.Position + reusable reusableJobAuthority } // CallGuard is one immutable caller-scoped condition inherited by a flattened diff --git a/internal/compiler/compiler_test.go b/internal/compiler/compiler_test.go index f4053b41..09df3dd2 100644 --- a/internal/compiler/compiler_test.go +++ b/internal/compiler/compiler_test.go @@ -1263,6 +1263,96 @@ jobs: } } +func TestCompilePreservesNestedReusableCallContext(t *testing.T) { + repository := t.TempDir() + callerPath := writeWorkflow(t, repository, "caller.yml", `on: push +permissions: + contents: write +jobs: + producer: + runs-on: ubuntu-latest + outputs: + subject: ${{ steps.emit.outputs.subject }} + steps: + - id: emit + run: echo subject=value >> "$GITHUB_OUTPUT" + delegated: + needs: producer + if: needs.producer.result == 'success' + uses: ./.github/workflows/middle.yml + with: + subject: ${{ needs.producer.outputs.subject }} + secrets: inherit +`) + writeWorkflow(t, repository, "middle.yml", `on: + workflow_call: + inputs: + subject: {type: string, required: true} +permissions: + contents: read +jobs: + nested: + if: inputs.subject != '' + uses: ./.github/workflows/leaf.yml + with: + subject: ${{ inputs.subject }} + secrets: inherit +`) + leafPath := writeWorkflow(t, repository, "leaf.yml", `on: + workflow_call: + inputs: + subject: {type: string, required: true} +jobs: + consume: + runs-on: ubuntu-latest + env: + TOKEN: ${{ secrets.LEAF_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - run: echo ${{ inputs.subject }} +`) + + plans, err := compileUntrustedPlans(callerPath, readFile(t, callerPath), pushEvent(t), "0.0.0-test", testDistributionDigest, "gha-untrusted") + if err != nil { + t.Fatal(err) + } + if len(plans) != 2 { + t.Fatalf("plans = %d, want producer and flattened nested consumer", len(plans)) + } + producer, consumer := plans[0], plans[1] + if consumer.Workflow.LogicalJobID != "delegated.nested.consume" || consumer.Workflow.Path != "./.github/workflows/leaf.yml" || consumer.Workflow.Digest != "sha256:"+sha256Sum(readFile(t, leafPath)) { + t.Fatalf("nested consumer provenance = %#v", consumer.Workflow) + } + if !slices.Equal(consumer.RequiredSecrets, []string{"LEAF_TOKEN"}) || consumer.GitHubToken == nil || !reflect.DeepEqual(consumer.GitHubToken.Permissions, map[string]string{"contents": "write"}) { + t.Fatalf("nested consumer authority = secrets %#v, token %#v", consumer.RequiredSecrets, consumer.GitHubToken) + } + deferred := consumer.DeferredInputs["subject"] + if len(deferred.Sources) != 1 || deferred.Sources[0].StepKey != producer.Target.StepKey || len(deferred.Outputs) != 1 || deferred.Outputs[0] != (plan.NeedOutput{Name: "value", StepKey: producer.Target.StepKey, Output: "subject"}) { + t.Fatalf("nested deferred input = %#v", deferred) + } + if len(consumer.NeedSources["producer"]) != 1 || consumer.NeedSources["producer"][0].StepKey != producer.Target.StepKey || len(consumer.NeedOutputs["producer"]) != 0 { + t.Fatalf("nested prerequisite projection = %#v / %#v", consumer.NeedSources, consumer.NeedOutputs) + } + if len(consumer.CallGuards) != 2 || consumer.CallGuards[0].Condition != "(needs.producer.result == 'success')" || consumer.CallGuards[1].Condition != "(inputs.subject != '')" { + t.Fatalf("nested call guards = %#v", consumer.CallGuards) + } + if !reflect.DeepEqual(consumer.CallGuards[1].DeferredInputs["subject"], deferred) || !slices.Equal(consumer.Dependencies, []string{producer.Target.StepKey}) { + t.Fatalf("nested guard/dependencies = %#v / %#v", consumer.CallGuards[1], consumer.Dependencies) + } + encoded, err := plan.Encode(consumer) + if err != nil { + t.Fatal(err) + } + decoded, err := plan.Decode(encoded) + if err != nil { + t.Fatal(err) + } + reencoded, err := plan.Encode(decoded) + if err != nil || !bytes.Equal(reencoded, encoded) { + t.Fatalf("nested reusable context plan round trip changed bytes: %v", err) + } +} + func TestCompileRejectsUnavailableReusableCallConditionContexts(t *testing.T) { for _, contextName := range []string{"matrix.target", "strategy.job-index", "secrets.TOKEN", "env.FLAG", "runner.os", "steps.build.outcome"} { t.Run(contextName, func(t *testing.T) { diff --git a/internal/compiler/job_graph_expansion.go b/internal/compiler/job_graph_expansion.go index 66020d4a..8d56910e 100644 --- a/internal/compiler/job_graph_expansion.go +++ b/internal/compiler/job_graph_expansion.go @@ -4,11 +4,9 @@ import ( "context" "errors" "fmt" - "slices" "sort" "github.com/buildkite/buildkite-gha/internal/expression" - "github.com/buildkite/buildkite-gha/internal/plan" "github.com/buildkite/buildkite-gha/internal/workflow" ) @@ -60,7 +58,7 @@ func processingJobs(path string, parsed *workflow.Workflow, resolved []sourcedJo continue } seen[sourced.ID] = true - jobs = append(jobs, ParsedJob{ID: sourced.ID, Path: sourced.path, Source: sourced.Span}) + jobs = append(jobs, ParsedJob{ID: sourced.ID, Path: sourced.sourcePath(), Source: sourced.Span}) } return jobs } @@ -108,12 +106,12 @@ func (e *jobGraphExpansion) acceptJobs(resolved []sourcedJob) { for _, sourced := range resolved { job := sourced.Job if _, exists := e.acceptedIndex[job.ID]; exists { - e.diagnostics = append(e.diagnostics, attributedProcessingFinding(StageGraph, CodeGraphInvalid, "compatibility", sourced.path, 0, 0, job.ID, "", "", 0, jobError(sourced.path, job, fmt.Sprintf("flattened job id %q collides with another job", job.ID)))) + e.diagnostics = append(e.diagnostics, attributedProcessingFinding(StageGraph, CodeGraphInvalid, "compatibility", sourced.sourcePath(), 0, 0, job.ID, "", "", 0, jobError(sourced.sourcePath(), job, fmt.Sprintf("flattened job id %q collides with another job", job.ID)))) continue } e.acceptedIndex[job.ID] = len(e.accepted) e.accepted = append(e.accepted, sourced) - if err := supported(sourced.path, job); err != nil { + if err := supported(sourced.sourcePath(), job); err != nil { e.diagnostics = append(e.diagnostics, err) e.failedJobs[job.ID] = true } @@ -123,15 +121,7 @@ func (e *jobGraphExpansion) acceptJobs(resolved []sourcedJob) { func (e *jobGraphExpansion) orderJobs() { e.topologyJobs = make(map[string]workflow.Job, len(e.accepted)) for _, sourced := range e.accepted { - job := sourced.Job - for _, guard := range sourced.callGuards { - job.Needs = append(job.Needs, bindingMembers(guard.needBindings)...) - job.Needs = append(job.Needs, bindingMembers(guard.inputs.deferred)...) - } - job.Needs = append(job.Needs, bindingMembers(sourced.inputs.deferred)...) - sort.Strings(job.Needs) - job.Needs = slices.Compact(job.Needs) - e.topologyJobs[sourced.ID] = job + e.topologyJobs[sourced.ID] = sourced.topologyJob() } order, err := topologicalOrder(e.path, e.topologyJobs) if err != nil { @@ -146,7 +136,7 @@ func (e *jobGraphExpansion) expandMatrices() { for _, id := range e.order { sourced := e.accepted[e.acceptedIndex[id]] job := sourced.Job - descriptor, deferred, err := describeRuntimeMatrix(job, sourced.path, sourced.digest, sourced.needBindings, e.topologyJobs, e.matricesByJob) + descriptor, deferred, err := sourced.describeRuntimeMatrix(e.topologyJobs, e.matricesByJob) var matrices []map[string]any if deferred { e.result.runtimeMatrixBoundary = true @@ -160,15 +150,15 @@ func (e *jobGraphExpansion) expandMatrices() { e.result.runtimeMatrices = append(e.result.runtimeMatrices, descriptor) err = errors.New("runtime matrix source is valid, but continuation upload is disabled because Buildkite transport has no authoritative current-attempt fence and durable idempotency boundary") } - err = locatedJobError(sourced.path, job, position.Line, position.Column, err.Error()) + err = locatedJobError(sourced.sourcePath(), job, position.Line, position.Column, err.Error()) } else { - matrices, err = expandMatrix(sourced.path, job, e.context) + matrices, err = expandMatrix(sourced.sourcePath(), job, e.context) } if err != nil { line, column := matrixErrorPosition(job) e.diagnostics = append(e.diagnostics, &ProcessingFinding{ Stage: StageMatrix, Code: CodeMatrixInvalid, Category: "compatibility", - Path: sourced.path, Line: line, Column: column, Job: job.ID, + Path: sourced.sourcePath(), Line: line, Column: column, Job: job.ID, Message: "matrix could not be expanded or validated", Err: err, }) e.failedMatrices[id] = true @@ -210,13 +200,12 @@ func (e *jobGraphExpansion) expandInstances() { func (e *jobGraphExpansion) expandJobInstances(id string) { sourced := e.accepted[e.acceptedIndex[id]] job := sourced.Job - jobPath := sourced.path - jobBlocked := e.jobBlocked(sourced) + jobPath := sourced.sourcePath() + jobBlocked := sourced.blockedBy(e.failedJobs) jobFailed := e.failedJobs[id] matrices := e.matricesByJob[id] concurrencyGroups := make(map[string]struct{}, len(matrices)) - jobContext := e.context - jobContext.Inputs = sourced.inputs.values + jobContext := sourced.compileContext(e.context) for matrixIndex, matrix := range matrices { strategy := matrixStrategy(job, matrixIndex, len(matrices)) instanceContext := jobContext @@ -246,7 +235,7 @@ func (e *jobGraphExpansion) expandJobInstances(id string) { resolvedContainer, containerErr := resolveCompileContainer(instanceJob.Container, instanceContext) instanceJob.Container = resolvedContainer resolvedServices, serviceErr := resolveCompileServices(instanceJob.Services, instanceContext) - candidate := newJobCandidate(sourced, instanceJob, matrix, key, resolvedServices) + candidate := sourced.newCandidate(instanceJob, matrix, key, resolvedServices) valid := true if containerErr != nil { @@ -311,7 +300,9 @@ func (e *jobGraphExpansion) expandJobInstances(id string) { instance.Platform = target.Platform instance.RuntimeImage = target.Image instance.ConcurrencyGroup = concurrencyGroup - if e.bindInstanceDependencies(sourced, job, key, &instance) { + diagnostics, blocked := sourced.bindInstanceDependencies(key, &instance, e.byLogicalID) + e.diagnostics = append(e.diagnostics, diagnostics...) + if blocked { jobFailed = true continue } @@ -336,226 +327,6 @@ func matrixStrategy(job workflow.Job, index, total int) map[string]any { return strategy } -func newJobCandidate(sourced sourcedJob, job workflow.Job, matrix map[string]any, key string, services []workflow.Service) JobInstance { - candidate := JobInstance{ - Key: key, LogicalJobID: job.ID, Matrix: matrix, Inputs: cloneAnyMap(sourced.inputs.values), - FailFast: job.FailFast, MaxParallel: job.MaxParallel, Steps: append([]workflow.Step(nil), job.Steps...), - Env: cloneMap(job.Env), Permissions: permissionScopes(job.Permissions), If: job.If, - ContinueOnError: job.ContinueOnError, TimeoutMinutes: job.TimeoutMinutes, - DefaultShell: job.DefaultShell, DefaultWorkingDirectory: job.DefaultWorkingDirectory, - Outputs: cloneMap(job.Outputs), Container: job.Container, Services: services, - ServicesExpression: job.ServicesExpression, SourcePath: sourced.path, SourceDigest: sourced.digest, - RemoteWorkflow: cloneRemoteWorkflowSource(sourced.remote), RepositoryRoot: sourced.root, Source: job.Span, - secretAuthority: sourced.secretAuthority, tokenPolicyNarrowed: sourced.tokenPolicyNarrowed, - jobPermissionsIgnored: sourced.jobPermissionsIgnored, reusableCall: sourced.reusableCall, - } - for _, guard := range sourced.callGuards { - candidate.CallGuards = append(candidate.CallGuards, CallGuard{Condition: guard.condition, Inputs: cloneAnyMap(guard.inputs.values)}) - } - return candidate -} - -func (e *jobGraphExpansion) jobBlocked(sourced sourcedJob) bool { - if bindingsFailed(sourced.needBindings, e.failedJobs) || bindingsFailed(sourced.inputs.deferred, e.failedJobs) { - return true - } - for _, guard := range sourced.callGuards { - if bindingsFailed(guard.needBindings, e.failedJobs) || bindingsFailed(guard.inputs.deferred, e.failedJobs) { - return true - } - } - return false -} - -func bindingsFailed(bindings map[string]needBinding, failedJobs map[string]bool) bool { - for _, binding := range bindings { - for _, member := range binding.members { - if failedJobs[member] { - return true - } - } - } - return false -} - -func (e *jobGraphExpansion) bindInstanceDependencies(sourced sourcedJob, job workflow.Job, key string, instance *JobInstance) bool { - for _, need := range sortedKeys(sourced.needBindings) { - binding := sourced.needBindings[need] - var members []string - for _, member := range binding.members { - for _, prerequisite := range e.byLogicalID[member] { - members = append(members, prerequisite.Key) - } - } - sort.Strings(members) - if len(members) == 0 { - e.diagnostics = append(e.diagnostics, attributedProcessingFinding(StageGraph, CodeGraphInvalid, "compatibility", sourced.path, 0, 0, job.ID, key, "", 0, jobError(sourced.path, job, fmt.Sprintf("prerequisite %q has no expanded instances", need)))) - return true - } - if instance.NeedGroups == nil { - instance.NeedGroups = make(map[string][]string, len(sourced.needBindings)) - } - instance.NeedGroups[need] = members - instance.Needs = append(instance.Needs, members...) - if binding.projectOutputs { - e.projectNeedOutputs(sourced, need, binding, instance) - } - } - deferredInputs, dependencies, err := resolveDeferredInputBindings(sourced.inputs.deferred, e.byLogicalID) - if err != nil { - e.diagnostics = append(e.diagnostics, attributedProcessingFinding(StageGraph, CodeGraphInvalid, "compatibility", sourced.path, 0, 0, job.ID, key, "", 0, jobError(sourced.path, job, fmt.Sprintf("resolve deferred reusable-workflow inputs: %v", err)))) - return true - } - instance.DeferredInputs = deferredInputs - instance.Needs = append(instance.Needs, dependencies...) - for guardIndex, guard := range sourced.callGuards { - groups, outputs, dependencies, err := resolveCallGuardBindings(guard.needBindings, e.byLogicalID) - if err != nil { - e.diagnostics = append(e.diagnostics, attributedProcessingFinding(StageGraph, CodeGraphInvalid, "compatibility", sourced.path, 0, 0, job.ID, key, "", 0, jobError(sourced.path, job, fmt.Sprintf("resolve reusable-workflow call guard: %v", err)))) - return true - } - instance.CallGuards[guardIndex].NeedGroups = groups - instance.CallGuards[guardIndex].NeedOutputs = outputs - instance.Needs = append(instance.Needs, dependencies...) - instance.CallGuards[guardIndex].DeferredInputs, dependencies, err = resolveDeferredInputBindings(guard.inputs.deferred, e.byLogicalID) - if err != nil { - e.diagnostics = append(e.diagnostics, attributedProcessingFinding(StageGraph, CodeGraphInvalid, "compatibility", sourced.path, 0, 0, job.ID, key, "", 0, jobError(sourced.path, job, fmt.Sprintf("resolve reusable-workflow call guard inputs: %v", err)))) - return true - } - instance.Needs = append(instance.Needs, dependencies...) - } - sort.Strings(instance.Needs) - instance.Needs = slices.Compact(instance.Needs) - return false -} - -func (e *jobGraphExpansion) projectNeedOutputs(sourced sourcedJob, need string, binding needBinding, instance *JobInstance) { - if instance.NeedOutputs == nil { - instance.NeedOutputs = make(map[string][]NeedOutput) - } - projected := []NeedOutput{} - for _, output := range binding.outputs { - producers := e.byLogicalID[output.member] - if len(producers) == 0 { - e.diagnostics = append(e.diagnostics, processingFinding(StageGraph, CodeGraphInvalid, "compatibility", fmt.Errorf("%s:%d:%d: workflow_call output %q selects unexpanded job %q", output.path, output.span.Start.Line, output.span.Start.Column, output.name, output.member))) - continue - } - if len(projected)+len(producers) > plan.MaxNeedOutputs { - e.diagnostics = append(e.diagnostics, processingFinding(StageGraph, CodeGraphInvalid, "compatibility", fmt.Errorf("%s:%d:%d: workflow_call output %q expands call projections beyond the maximum of %d", output.path, output.span.Start.Line, output.span.Start.Column, output.name, plan.MaxNeedOutputs))) - continue - } - for _, producer := range producers { - projected = append(projected, NeedOutput{Name: output.name, StepKey: producer.Key, Output: output.output}) - } - } - sortNeedOutputs(projected) - instance.NeedOutputs[need] = projected -} - -func resolveDeferredInputBindings(bindings map[string]needBinding, byLogicalID map[string][]JobInstance) (map[string]DeferredInput, []string, error) { - if len(bindings) == 0 { - return nil, nil, nil - } - resolved := make(map[string]DeferredInput, len(bindings)) - var dependencies []string - for _, name := range sortedKeys(bindings) { - input, err := resolveDeferredInputBinding(bindings[name], byLogicalID) - if err != nil { - return nil, nil, fmt.Errorf("input %q: %w", name, err) - } - resolved[name] = input - dependencies = append(dependencies, input.Sources...) - } - return resolved, dependencies, nil -} - -func resolveDeferredInputBinding(binding needBinding, byLogicalID map[string][]JobInstance) (DeferredInput, error) { - var deferred DeferredInput - for _, member := range binding.members { - producers := byLogicalID[member] - if len(producers) == 0 { - return DeferredInput{}, fmt.Errorf("source job %q has no expanded instances", member) - } - for _, producer := range producers { - deferred.Sources = append(deferred.Sources, producer.Key) - } - } - sort.Strings(deferred.Sources) - deferred.Sources = slices.Compact(deferred.Sources) - if len(deferred.Sources) > plan.MaxNeedProducers { - return DeferredInput{}, fmt.Errorf("has %d producers, maximum is %d", len(deferred.Sources), plan.MaxNeedProducers) - } - for _, output := range binding.outputs { - producers := byLogicalID[output.member] - if len(deferred.Outputs)+len(producers) > plan.MaxNeedOutputs { - return DeferredInput{}, fmt.Errorf("output %q expands beyond the maximum of %d projections", output.output, plan.MaxNeedOutputs) - } - for _, producer := range producers { - deferred.Outputs = append(deferred.Outputs, NeedOutput{Name: output.name, StepKey: producer.Key, Output: output.output}) - } - } - sortNeedOutputs(deferred.Outputs) - return deferred, nil -} - -func resolveCallGuardBindings(bindings map[string]needBinding, byLogicalID map[string][]JobInstance) (map[string][]string, map[string][]NeedOutput, []string, error) { - if len(bindings) == 0 { - return nil, nil, nil, nil - } - groups := make(map[string][]string, len(bindings)) - var projected map[string][]NeedOutput - var dependencies []string - for _, name := range sortedKeys(bindings) { - binding := bindings[name] - var members []string - for _, member := range binding.members { - for _, producer := range byLogicalID[member] { - members = append(members, producer.Key) - } - } - sort.Strings(members) - if len(members) == 0 { - return nil, nil, nil, fmt.Errorf("prerequisite %q has no expanded instances", name) - } - groups[name] = members - dependencies = append(dependencies, members...) - if !binding.projectOutputs { - continue - } - if projected == nil { - projected = make(map[string][]NeedOutput) - } - outputs := []NeedOutput{} - for _, output := range binding.outputs { - producers := byLogicalID[output.member] - if len(producers) == 0 { - return nil, nil, nil, fmt.Errorf("output %q selects unexpanded job %q", output.name, output.member) - } - if len(outputs)+len(producers) > plan.MaxNeedOutputs { - return nil, nil, nil, fmt.Errorf("output %q expands projections beyond the maximum of %d", output.name, plan.MaxNeedOutputs) - } - for _, producer := range producers { - outputs = append(outputs, NeedOutput{Name: output.name, StepKey: producer.Key, Output: output.output}) - } - } - sortNeedOutputs(outputs) - projected[name] = outputs - } - return groups, projected, dependencies, nil -} - -func sortNeedOutputs(outputs []NeedOutput) { - sort.Slice(outputs, func(i, j int) bool { - if outputs[i].Name != outputs[j].Name { - return outputs[i].Name < outputs[j].Name - } - if outputs[i].StepKey != outputs[j].StepKey { - return outputs[i].StepKey < outputs[j].StepKey - } - return outputs[i].Output < outputs[j].Output - }) -} - func topologicalOrder(path string, jobs map[string]workflow.Job) ([]string, error) { indegree := make(map[string]int, len(jobs)) dependents := make(map[string][]string, len(jobs)) diff --git a/internal/compiler/plan_builder.go b/internal/compiler/plan_builder.go index 40f13b17..46c6a6be 100644 --- a/internal/compiler/plan_builder.go +++ b/internal/compiler/plan_builder.go @@ -125,18 +125,6 @@ func (b planBuilder) buildPlan(instance JobInstance, runtimeDistributionDigest s if err := addContainerCapabilities(instance, actionRefs, &actions); err != nil { return plan.Job{}, PlanAuthorization{}, nil, err } - needSources, err := buildPlanNeedSources(instance, b.planDigests) - if err != nil { - return plan.Job{}, PlanAuthorization{}, nil, err - } - deferredInputs, err := buildPlanDeferredInputs(instance.DeferredInputs, b.planDigests) - if err != nil { - return plan.Job{}, PlanAuthorization{}, nil, fmt.Errorf("build plan for job %q: %w", instance.LogicalJobID, err) - } - callGuards, err := buildPlanCallGuards(instance, b.planDigests) - if err != nil { - return plan.Job{}, PlanAuthorization{}, nil, err - } secrets, secretMappings, githubToken, err := b.authorizePlanSecrets(instance, &actions) if err != nil { return plan.Job{}, PlanAuthorization{}, nil, err @@ -149,7 +137,10 @@ func (b planBuilder) buildPlan(instance JobInstance, runtimeDistributionDigest s if instance.Platform == PlatformDarwinARM64 && slices.Contains(actions.capabilities, "docker") { return plan.Job{}, PlanAuthorization{}, nil, fmt.Errorf("%s:%d:%d: job %q requires Docker, which is unavailable on darwin/arm64", instance.SourcePath, instance.Source.Start.Line, instance.Source.Start.Column, instance.LogicalJobID) } - job := b.lowerPlanJob(instance, runtimeDistributionDigest, steps, actions, needSources, buildPlanNeedOutputs(instance), deferredInputs, callGuards, secrets, secretMappings, githubToken) + job, err := b.lowerPlanJob(instance, runtimeDistributionDigest, steps, actions, secrets, secretMappings, githubToken) + if err != nil { + return plan.Job{}, PlanAuthorization{}, nil, err + } if err := job.Validate(); err != nil { return plan.Job{}, PlanAuthorization{}, nil, fmt.Errorf("build plan for job %q: %w", instance.LogicalJobID, err) } @@ -497,95 +488,6 @@ func addContainerCapabilities(instance JobInstance, actionRefs []string, built * return nil } -func buildPlanNeedSources(instance JobInstance, planDigests map[string]string) (map[string][]plan.NeedSource, error) { - needSources := make(map[string][]plan.NeedSource, len(instance.NeedGroups)) - for _, logicalNeed := range sortedKeys(instance.NeedGroups) { - dependencies := instance.NeedGroups[logicalNeed] - if len(dependencies) > plan.MaxNeedProducers { - return nil, fmt.Errorf("build plan for job %q: prerequisite %q has %d producers, maximum is %d", instance.LogicalJobID, logicalNeed, len(dependencies), plan.MaxNeedProducers) - } - for _, dependency := range dependencies { - digest, ok := planDigests[dependency] - if !ok { - return nil, fmt.Errorf("build plan for job %q: prerequisite %q has no earlier plan digest", instance.LogicalJobID, dependency) - } - needSources[logicalNeed] = append(needSources[logicalNeed], plan.NeedSource{StepKey: dependency, PlanDigest: digest}) - } - } - return needSources, nil -} - -func buildPlanNeedOutputs(instance JobInstance) map[string][]plan.NeedOutput { - if len(instance.NeedOutputs) == 0 { - return nil - } - needOutputs := make(map[string][]plan.NeedOutput, len(instance.NeedOutputs)) - for _, logicalNeed := range sortedKeys(instance.NeedOutputs) { - outputs := instance.NeedOutputs[logicalNeed] - needOutputs[logicalNeed] = make([]plan.NeedOutput, len(outputs)) - for i, output := range outputs { - needOutputs[logicalNeed][i] = plan.NeedOutput{Name: output.Name, StepKey: output.StepKey, Output: output.Output} - } - } - return needOutputs -} - -func buildPlanDeferredInputs(inputs map[string]DeferredInput, planDigests map[string]string) (map[string]plan.DeferredInput, error) { - if len(inputs) == 0 { - return nil, nil - } - resolved := make(map[string]plan.DeferredInput, len(inputs)) - for _, name := range sortedKeys(inputs) { - input := inputs[name] - deferred := plan.DeferredInput{Outputs: make([]plan.NeedOutput, len(input.Outputs))} - for _, source := range input.Sources { - digest, ok := planDigests[source] - if !ok { - return nil, fmt.Errorf("deferred input %q source %q has no earlier plan digest", name, source) - } - deferred.Sources = append(deferred.Sources, plan.NeedSource{StepKey: source, PlanDigest: digest}) - } - for i, output := range input.Outputs { - deferred.Outputs[i] = plan.NeedOutput{Name: output.Name, StepKey: output.StepKey, Output: output.Output} - } - resolved[name] = deferred - } - return resolved, nil -} - -func buildPlanCallGuards(instance JobInstance, planDigests map[string]string) ([]plan.CallGuard, error) { - callGuards := make([]plan.CallGuard, len(instance.CallGuards)) - for guardIndex, guard := range instance.CallGuards { - deferredInputs, err := buildPlanDeferredInputs(guard.DeferredInputs, planDigests) - if err != nil { - return nil, fmt.Errorf("build plan for job %q call guard %d: %w", instance.LogicalJobID, guardIndex+1, err) - } - planGuard := plan.CallGuard{Condition: guard.Condition, Inputs: cloneAnyMap(guard.Inputs), DeferredInputs: deferredInputs} - if len(guard.NeedGroups) != 0 { - planGuard.NeedSources = make(map[string][]plan.NeedSource, len(guard.NeedGroups)) - for _, logicalNeed := range sortedKeys(guard.NeedGroups) { - for _, dependency := range guard.NeedGroups[logicalNeed] { - digest, ok := planDigests[dependency] - if !ok { - return nil, fmt.Errorf("build plan for job %q: call guard prerequisite %q has no earlier plan digest", instance.LogicalJobID, dependency) - } - planGuard.NeedSources[logicalNeed] = append(planGuard.NeedSources[logicalNeed], plan.NeedSource{StepKey: dependency, PlanDigest: digest}) - } - } - } - if len(guard.NeedOutputs) != 0 { - planGuard.NeedOutputs = make(map[string][]plan.NeedOutput, len(guard.NeedOutputs)) - for _, logicalNeed := range sortedKeys(guard.NeedOutputs) { - for _, output := range guard.NeedOutputs[logicalNeed] { - planGuard.NeedOutputs[logicalNeed] = append(planGuard.NeedOutputs[logicalNeed], plan.NeedOutput{Name: output.Name, StepKey: output.StepKey, Output: output.Output}) - } - } - } - callGuards[guardIndex] = planGuard - } - return callGuards, nil -} - func (b planBuilder) authorizePlanSecrets(instance JobInstance, actions *builtPlanActions) ([]string, map[string]string, *plan.GitHubToken, error) { secrets, mappings, tokenAliases, referencesGitHubToken, err := requiredSecrets(instance, actions.requiredSecrets, actions.inputsInspected) if err != nil { @@ -616,20 +518,13 @@ func (b planBuilder) authorizePlanSecrets(instance JobInstance, actions *builtPl return secrets, mappings, &plan.GitHubToken{Workflow: policyWorkflow, Permissions: cloneMap(b.ir.Workflow.WorkflowTokenPermissions), Aliases: tokenAliases}, nil } -func (b planBuilder) lowerPlanJob(instance JobInstance, runtimeDistributionDigest string, steps []plan.Step, actions builtPlanActions, needSources map[string][]plan.NeedSource, needOutputs map[string][]plan.NeedOutput, deferredInputs map[string]plan.DeferredInput, callGuards []plan.CallGuard, secrets []string, secretMappings map[string]string, githubToken *plan.GitHubToken) plan.Job { +func (b planBuilder) lowerPlanJob(instance JobInstance, runtimeDistributionDigest string, steps []plan.Step, actions builtPlanActions, secrets []string, secretMappings map[string]string, githubToken *plan.GitHubToken) (plan.Job, error) { job := plan.Job{ Schema: plan.Schema, Compiler: plan.Compiler{ Version: b.compilerVersion, DistributionDigest: b.compilerDistributionDigest, }, Runtime: &plan.Runtime{DistributionDigest: runtimeDistributionDigest}, - Workflow: plan.Workflow{ - Path: instance.SourcePath, - Name: b.workflowName, - Digest: instance.SourceDigest, - LogicalJobID: instance.LogicalJobID, - Remote: planRemoteWorkflowSource(instance.RemoteWorkflow), - }, Event: plan.Event{ Provider: b.ir.Event.Provider, Name: b.ir.Event.Event, PayloadDigest: "sha256:" + hex.EncodeToString(b.eventDigest[:]), Repository: b.ir.Event.Repository.Owner + "/" + b.ir.Event.Repository.Name, @@ -643,13 +538,7 @@ func (b planBuilder) lowerPlanJob(instance JobInstance, runtimeDistributionDiges IDTokenPermission: instance.Permissions["id-token"], OIDC: cloneOIDCConfiguration(b.options.OIDC), Matrix: instance.Matrix, - Inputs: cloneAnyMap(instance.Inputs), - DeferredInputs: deferredInputs, Vars: cloneMap(b.ir.Vars), - Dependencies: append([]string(nil), instance.Needs...), - NeedSources: needSources, - NeedOutputs: needOutputs, - CallGuards: callGuards, Env: instance.Env, Condition: instance.If, ContinueOnError: instance.ContinueOnError, @@ -681,7 +570,10 @@ func (b planBuilder) lowerPlanJob(instance JobInstance, runtimeDistributionDiges job.Services[service.Name] = container job.ServiceOrder = append(job.ServiceOrder, service.Name) } - return job + if err := instance.lowerReusablePlan(&job, b.planDigests, b.workflowName); err != nil { + return plan.Job{}, err + } + return job, nil } func cloneOIDCConfiguration(configuration *plan.OIDCConfiguration) *plan.OIDCConfiguration { @@ -712,7 +604,7 @@ func requiredSecrets(instance JobInstance, actionRequired []string, actionInputs return err } for _, name := range names { - binding, ok := instance.secretAuthority.resolve(name) + binding, ok := instance.reusable.secrets.resolve(name) if strings.EqualFold(name, "GITHUB_TOKEN") || ok && binding.token { found[name] = name } @@ -860,7 +752,7 @@ func requiredSecrets(instance JobInstance, actionRequired []string, actionInputs tokenAliases = append(tokenAliases, alias) continue } - binding, ok := instance.secretAuthority.resolve(alias) + binding, ok := instance.reusable.secrets.resolve(alias) if !ok { continue } @@ -869,7 +761,7 @@ func requiredSecrets(instance JobInstance, actionRequired []string, actionInputs continue } sources[binding.source] = struct{}{} - if !instance.secretAuthority.unrestricted { + if !instance.reusable.secrets.unrestricted { mappings[alias] = binding.source } } diff --git a/internal/compiler/reusable.go b/internal/compiler/reusable.go index 6cf8d107..676a61cc 100644 --- a/internal/compiler/reusable.go +++ b/internal/compiler/reusable.go @@ -28,59 +28,6 @@ var staticInputCondition = regexp.MustCompile(`(?i)^\s*(?:\$\{\{\s*inputs\s*(?:\ var staticValueExpression = regexp.MustCompile(`(?i)^\s*\$\{\{\s*(inputs|matrix)\s*(?:\.\s*([A-Za-z_][A-Za-z0-9_-]*)|\[\s*'([A-Za-z0-9_-]{1,255})'\s*\])\s*\}\}\s*$`) var callOutputNamePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,255}$`) -type sourcedJob struct { - workflow.Job - path string - digest string - root string - remote *RemoteWorkflowSource - inputs reusableInputs - secretAuthority secretAuthority - needBindings map[string]needBinding - tokenPolicyNarrowed bool - jobPermissionsIgnored bool - reusableCall workflow.Position - callGuards []sourcedCallGuard -} - -type secretAuthority struct { - unrestricted bool - bindings map[string]secretBinding -} - -type secretBinding struct { - source string - token bool -} - -type sourcedCallGuard struct { - condition string - inputs reusableInputs - needBindings map[string]needBinding -} - -type reusableInputs struct { - values map[string]any - deferred map[string]needBinding -} - -type needBinding struct { - // members are flattened logical job IDs owned by one source-level need. - // projectOutputs distinguishes reusable-call/status-only boundaries from - // ordinary job needs, whose outputs pass through unchanged. - members []string - projectOutputs bool - outputs []needOutputBinding -} - -type needOutputBinding struct { - name string - member string - output string - path string - span workflow.Span -} - type reusableResolution struct { jobs []sourcedJob outputs []needOutputBinding @@ -128,7 +75,11 @@ func resolveReusableWorkflows(ctx context.Context, path string, source []byte, p for _, need := range job.Needs { bindings[need] = needBinding{members: []string{need}} } - jobs[i] = sourcedJob{Job: job, path: sourcePath, digest: digest, root: root, inputs: reusableInputs{values: cloneAnyMap(context.Inputs)}, secretAuthority: secretAuthority{unrestricted: true}, needBindings: bindings} + jobs[i] = sourcedJob{Job: job, reusable: reusableJobState{ + provenance: reusableJobProvenance{path: sourcePath, digest: digest, root: root}, + inputs: reusableInputs{values: cloneAnyMap(context.Inputs)}, needBindings: bindings, + authority: reusableJobAuthority{secrets: secretAuthority{unrestricted: true}}, + }} workflowJobs[job.ID] = job replacements[job.ID] = needBinding{members: []string{job.ID}} } @@ -152,7 +103,10 @@ func resolveReusableWorkflows(ctx context.Context, path string, source []byte, p } }() resolver.discoverRuntimeMatrixBoundaries(ctx, rootSource, parsed, 0, map[string]int{rootSource.identity.key(): 0}) - resolution, err := resolver.resolve(ctx, rootSource, digest, parsed, "", "", reusableInputs{values: context.Inputs}, nil, nil, secretAuthority{unrestricted: true}, false, workflow.Position{}, nil, 0) + resolution, err := resolver.resolve(ctx, parsed, reusableExpansion{ + source: rootSource, digest: digest, inputs: reusableInputs{values: context.Inputs}, + authority: reusableJobAuthority{secrets: secretAuthority{unrestricted: true}}, + }) return resolution.jobs, resolver.runtimeMatrixBoundary, err } @@ -202,8 +156,8 @@ func (resolver *reusableResolver) discoverRuntimeMatrixBoundaries(ctx context.Co } } -func (resolver *reusableResolver) resolve(ctx context.Context, current reusableWorkflowSource, digest string, parsed *workflow.Workflow, namespace, labelPrefix string, inputs reusableInputs, externalNeeds map[string]needBinding, permissionCeiling *workflow.Permissions, secrets secretAuthority, tokenPolicyNarrowed bool, reusableCallPosition workflow.Position, callGuards []sourcedCallGuard, depth int) (reusableResolution, error) { - path := current.displayPath +func (resolver *reusableResolver) resolve(ctx context.Context, parsed *workflow.Workflow, expansion reusableExpansion) (reusableResolution, error) { + path := expansion.source.displayPath resolver.runtimeMatrixBoundary = resolver.runtimeMatrixBoundary || hasRuntimeMatrixBoundary(parsed) jobs := make(map[string]workflow.Job, len(parsed.Jobs)) for _, job := range parsed.Jobs { @@ -219,11 +173,11 @@ func (resolver *reusableResolver) resolve(ctx context.Context, current reusableW for _, id := range order { job := jobs[id] declaredJobPermissions := job.Permissions - job.Permissions = effectivePermissions(job.Permissions, parsed.Permissions, permissionCeiling, depth != 0) - jobPermissionsIgnored := declaredJobPermissions != nil && repositoryPermissionsDiffer(resolver.rootPermissions, declaredJobPermissions) - jobTokenPolicyNarrowed := tokenPolicyNarrowed - if depth != 0 { - jobTokenPolicyNarrowed = jobTokenPolicyNarrowed || repositoryPermissionsNarrowed(resolver.rootPermissions, job.Permissions) + job.Permissions = effectivePermissions(job.Permissions, parsed.Permissions, expansion.permissionCeiling, expansion.depth != 0) + jobAuthority := expansion.authority + jobAuthority.jobPermissionsIgnored = jobAuthority.jobPermissionsIgnored || declaredJobPermissions != nil && repositoryPermissionsDiffer(resolver.rootPermissions, declaredJobPermissions) + if expansion.depth != 0 { + jobAuthority.tokenPolicyNarrowed = jobAuthority.tokenPolicyNarrowed || repositoryPermissionsNarrowed(resolver.rootPermissions, job.Permissions) idTokenPermission, hasIDTokenPermission := job.Permissions.Scopes["id-token"] job.Permissions = clonePermissions(resolver.rootPermissions) delete(job.Permissions.Scopes, "id-token") @@ -231,12 +185,12 @@ func (resolver *reusableResolver) resolve(ctx context.Context, current reusableW job.Permissions.Scopes["id-token"] = idTokenPermission } } - job, err = applyStaticInputs(path, job, inputs.values) + job, err = applyStaticInputs(path, job, expansion.inputs.values) if err != nil { return reusableResolution{}, err } if parsed.Callable { - if err := rejectUnresolvedInputExpressions(path, job, inputs.deferred); err != nil { + if err := rejectUnresolvedInputExpressions(path, job, expansion.inputs.deferred); err != nil { return reusableResolution{}, err } } @@ -248,7 +202,7 @@ func (resolver *reusableResolver) resolve(ctx context.Context, current reusableW callNeedBindings := replacementNeeds(job.Needs, replacements) needBindings := callNeedBindings if len(job.Needs) == 0 { - needBindings = cloneNeedBindings(externalNeeds) + needBindings = cloneNeedBindings(expansion.externalNeeds) for name, binding := range needBindings { binding.projectOutputs = true binding.outputs = nil @@ -257,34 +211,29 @@ func (resolver *reusableResolver) resolve(ctx context.Context, current reusableW } needs := bindingMembers(needBindings) if job.Reusable == nil { - job.ID = namespacedJobID(namespace, job.ID) + job.ID = namespacedJobID(expansion.namespace, job.ID) job.Needs = needs - if labelPrefix != "" { + if expansion.labelPrefix != "" { name := job.Name if name == "" { name = id } - job.Name = labelPrefix + " / " + name + job.Name = expansion.labelPrefix + " / " + name } resolver.expanded++ if resolver.expanded > maxFlattenedJobs { return reusableResolution{}, jobError(path, job, fmt.Sprintf("reusable-workflow graph expands beyond %d jobs", maxFlattenedJobs)) } - resolved = append(resolved, sourcedJob{ - Job: job, path: path, digest: digest, root: resolver.workspaceRoot, remote: cloneRemoteWorkflowSource(current.remote), inputs: cloneReusableInputs(inputs), - secretAuthority: cloneSecretAuthority(secrets), needBindings: needBindings, - tokenPolicyNarrowed: jobTokenPolicyNarrowed, jobPermissionsIgnored: jobPermissionsIgnored, reusableCall: reusableCallPosition, - callGuards: cloneSourcedCallGuards(callGuards), - }) + resolved = append(resolved, expansion.flattenedJob(job, needBindings, jobAuthority, resolver.workspaceRoot)) replacements[id] = needBinding{members: []string{job.ID}} continue } call := job.Reusable - calleeGuards := callGuards + calleeGuards := expansion.callGuards if strings.TrimSpace(job.If) != "" { conditionContext := resolver.context - conditionContext.Inputs = inputs.values + conditionContext.Inputs = expansion.inputs.values conditionContext.Matrix = nil conditionContext.Strategy = nil if err := expression.ValidateCompileCallCondition(job.If, conditionContext); err != nil { @@ -297,14 +246,14 @@ func (resolver *reusableResolver) resolve(ctx context.Context, current reusableW if err := expression.ValidateCallCondition(condition); err != nil { return reusableResolution{}, jobError(path, job, fmt.Sprintf("reusable-workflow call condition: %v", err)) } - calleeGuards = append(cloneSourcedCallGuards(callGuards), sourcedCallGuard{ - condition: condition, inputs: cloneReusableInputs(inputs), needBindings: cloneNeedBindings(callNeedBindings), + calleeGuards = append(cloneSourcedCallGuards(expansion.callGuards), sourcedCallGuard{ + condition: condition, inputs: cloneReusableInputs(expansion.inputs), needBindings: cloneNeedBindings(callNeedBindings), }) } - if depth >= MaxReusableWorkflowDepth { + if expansion.depth >= MaxReusableWorkflowDepth { return reusableResolution{}, locatedJobError(path, job, call.Span.Start.Line, call.Span.Start.Column, fmt.Sprintf("reusable-workflow nesting exceeds maximum depth %d", MaxReusableWorkflowDepth)) } - calleeSource, source, err := resolver.loadReusableWorkflow(ctx, current, call.Uses) + calleeSource, source, err := resolver.loadReusableWorkflow(ctx, expansion.source, call.Uses) if err != nil { return reusableResolution{}, locatedJobWrappedError(path, job, call.Span.Start.Line, call.Span.Start.Column, "", err) } @@ -322,7 +271,7 @@ func (resolver *reusableResolver) resolve(ctx context.Context, current reusableW if !callee.Callable { return reusableResolution{}, locatedJobError(path, job, call.Span.Start.Line, call.Span.Start.Column, fmt.Sprintf("reusable workflow %q does not declare on.workflow_call", call.Uses)) } - calleeSecrets, err := resolveCallSecretAuthority(path, job, call, callee, secrets) + calleeSecrets, err := resolveCallSecretAuthority(path, job, call, callee, expansion.authority.secrets) if err != nil { return reusableResolution{}, err } @@ -339,7 +288,7 @@ func (resolver *reusableResolver) resolve(ctx context.Context, current reusableW var callOutputs []needOutputBinding callNamespaces := make(map[string]struct{}, len(matrices)) for _, matrix := range matrices { - callInputs, err := resolveCallInputs(path, job, call, callee, inputs, callNeedBindings, matrix, resolver.context) + callInputs, err := resolveCallInputs(path, job, call, callee, expansion.inputs, callNeedBindings, matrix, resolver.context) if err != nil { return reusableResolution{}, err } @@ -351,7 +300,7 @@ func (resolver *reusableResolver) resolve(ctx context.Context, current reusableW } component += "-" + suffix } - callNamespace := namespacedJobID(namespace, component) + callNamespace := namespacedJobID(expansion.namespace, component) if _, exists := callNamespaces[callNamespace]; exists { return reusableResolution{}, jobError(path, job, fmt.Sprintf("reusable-workflow matrix produces duplicate namespace %q", callNamespace)) } @@ -363,20 +312,27 @@ func (resolver *reusableResolver) resolve(ctx context.Context, current reusableW if len(matrix) != 0 { callLabel = instanceLabel(job, matrix, expression.CompileContext{}) } - if labelPrefix != "" { - callLabel = labelPrefix + " / " + callLabel + if expansion.labelPrefix != "" { + callLabel = expansion.labelPrefix + " / " + callLabel } calleePermissionCeiling := job.Permissions if calleePermissionCeiling == nil { calleePermissionCeiling = &workflow.Permissions{Scopes: map[string]string{}, Span: call.Span} } - calleeCallPosition := reusableCallPosition - if depth == 0 { + calleeCallPosition := expansion.authority.reusableCall + if expansion.depth == 0 { calleeCallPosition = call.Span.Start } resolver.stack = append(resolver.stack, calleeSource.identity) - calleeResolution, err := resolver.resolve(ctx, calleeSource, calleeDigest, callee, callNamespace, callLabel, callInputs, needBindings, calleePermissionCeiling, calleeSecrets, jobTokenPolicyNarrowed, calleeCallPosition, calleeGuards, depth+1) + calleeAuthority := jobAuthority + calleeAuthority.secrets = calleeSecrets + calleeAuthority.reusableCall = calleeCallPosition + calleeResolution, err := resolver.resolve(ctx, callee, reusableExpansion{ + source: calleeSource, digest: calleeDigest, namespace: callNamespace, labelPrefix: callLabel, + inputs: callInputs, externalNeeds: needBindings, permissionCeiling: calleePermissionCeiling, + authority: calleeAuthority, callGuards: calleeGuards, depth: expansion.depth + 1, + }) resolver.stack = resolver.stack[:len(resolver.stack)-1] if err != nil { message := "reusable workflow could not be resolved" @@ -394,11 +350,6 @@ func (resolver *reusableResolver) resolve(ctx context.Context, current reusableW fmt.Sprintf("resolve reusable workflow %q", call.Uses), err), } } - if jobPermissionsIgnored { - for i := range calleeResolution.jobs { - calleeResolution.jobs[i].jobPermissionsIgnored = true - } - } resolved = append(resolved, calleeResolution.jobs...) callOutputs = append(callOutputs, calleeResolution.outputs...) for _, calleeJob := range calleeResolution.jobs { diff --git a/internal/compiler/reusable_job.go b/internal/compiler/reusable_job.go new file mode 100644 index 00000000..fec849b4 --- /dev/null +++ b/internal/compiler/reusable_job.go @@ -0,0 +1,472 @@ +package compiler + +import ( + "fmt" + "slices" + "sort" + + "github.com/buildkite/buildkite-gha/internal/expression" + "github.com/buildkite/buildkite-gha/internal/plan" + "github.com/buildkite/buildkite-gha/internal/workflow" +) + +// sourcedJob owns reusable-workflow state from recursive expansion until it is +// projected onto a concrete JobInstance. Graph expansion only orchestrates the +// ordered calls below; it does not interpret authority, provenance, deferred +// inputs, prerequisite projections, or call guards independently. +type sourcedJob struct { + workflow.Job + reusable reusableJobState +} + +type reusableExpansion struct { + source reusableWorkflowSource + digest string + namespace string + labelPrefix string + inputs reusableInputs + externalNeeds map[string]needBinding + permissionCeiling *workflow.Permissions + authority reusableJobAuthority + callGuards []sourcedCallGuard + depth int +} + +type reusableJobState struct { + provenance reusableJobProvenance + inputs reusableInputs + needBindings map[string]needBinding + authority reusableJobAuthority + callGuards []sourcedCallGuard +} + +type reusableJobProvenance struct { + path string + digest string + root string + remote *RemoteWorkflowSource +} + +type reusableJobAuthority struct { + secrets secretAuthority + tokenPolicyNarrowed bool + jobPermissionsIgnored bool + reusableCall workflow.Position +} + +type secretAuthority struct { + unrestricted bool + bindings map[string]secretBinding +} + +type secretBinding struct { + source string + token bool +} + +type sourcedCallGuard struct { + condition string + inputs reusableInputs + needBindings map[string]needBinding +} + +type reusableInputs struct { + values map[string]any + deferred map[string]needBinding +} + +type needBinding struct { + // members are flattened logical job IDs owned by one source-level need. + // projectOutputs distinguishes reusable-call/status-only boundaries from + // ordinary job needs, whose outputs pass through unchanged. + members []string + projectOutputs bool + outputs []needOutputBinding +} + +type needOutputBinding struct { + name string + member string + output string + path string + span workflow.Span +} + +func (expansion reusableExpansion) flattenedJob(job workflow.Job, needs map[string]needBinding, authority reusableJobAuthority, workspaceRoot string) sourcedJob { + authority.secrets = cloneSecretAuthority(authority.secrets) + return sourcedJob{Job: job, reusable: reusableJobState{ + provenance: reusableJobProvenance{ + path: expansion.source.displayPath, digest: expansion.digest, root: workspaceRoot, + remote: cloneRemoteWorkflowSource(expansion.source.remote), + }, + inputs: cloneReusableInputs(expansion.inputs), needBindings: cloneNeedBindings(needs), + authority: authority, callGuards: cloneSourcedCallGuards(expansion.callGuards), + }} +} + +func (sourced sourcedJob) sourcePath() string { + return sourced.reusable.provenance.path +} + +func (sourced sourcedJob) describeRuntimeMatrix(jobs map[string]workflow.Job, matricesByJob map[string][]map[string]any) (RuntimeMatrixDescriptor, bool, error) { + return describeRuntimeMatrix(sourced.Job, sourced.reusable.provenance.path, sourced.reusable.provenance.digest, sourced.reusable.needBindings, jobs, matricesByJob) +} + +func (sourced sourcedJob) topologyJob() workflow.Job { + job := sourced.Job + for _, guard := range sourced.reusable.callGuards { + job.Needs = append(job.Needs, bindingMembers(guard.needBindings)...) + job.Needs = append(job.Needs, bindingMembers(guard.inputs.deferred)...) + } + job.Needs = append(job.Needs, bindingMembers(sourced.reusable.inputs.deferred)...) + sort.Strings(job.Needs) + job.Needs = slices.Compact(job.Needs) + return job +} + +func (sourced sourcedJob) compileContext(context expression.CompileContext) expression.CompileContext { + context.Inputs = sourced.reusable.inputs.values + return context +} + +func (sourced sourcedJob) newCandidate(job workflow.Job, matrix map[string]any, key string, services []workflow.Service) JobInstance { + candidate := JobInstance{ + Key: key, LogicalJobID: job.ID, Matrix: matrix, Inputs: cloneAnyMap(sourced.reusable.inputs.values), + FailFast: job.FailFast, MaxParallel: job.MaxParallel, Steps: append([]workflow.Step(nil), job.Steps...), + Env: cloneMap(job.Env), Permissions: permissionScopes(job.Permissions), If: job.If, + ContinueOnError: job.ContinueOnError, TimeoutMinutes: job.TimeoutMinutes, + DefaultShell: job.DefaultShell, DefaultWorkingDirectory: job.DefaultWorkingDirectory, + Outputs: cloneMap(job.Outputs), Container: job.Container, Services: services, + ServicesExpression: job.ServicesExpression, SourcePath: sourced.reusable.provenance.path, SourceDigest: sourced.reusable.provenance.digest, + RemoteWorkflow: cloneRemoteWorkflowSource(sourced.reusable.provenance.remote), RepositoryRoot: sourced.reusable.provenance.root, Source: job.Span, + reusable: sourced.reusable.authority, + } + for _, guard := range sourced.reusable.callGuards { + candidate.CallGuards = append(candidate.CallGuards, CallGuard{Condition: guard.condition, Inputs: cloneAnyMap(guard.inputs.values)}) + } + return candidate +} + +func (sourced sourcedJob) blockedBy(failedJobs map[string]bool) bool { + if bindingsFailed(sourced.reusable.needBindings, failedJobs) || bindingsFailed(sourced.reusable.inputs.deferred, failedJobs) { + return true + } + for _, guard := range sourced.reusable.callGuards { + if bindingsFailed(guard.needBindings, failedJobs) || bindingsFailed(guard.inputs.deferred, failedJobs) { + return true + } + } + return false +} + +func bindingsFailed(bindings map[string]needBinding, failedJobs map[string]bool) bool { + for _, binding := range bindings { + for _, member := range binding.members { + if failedJobs[member] { + return true + } + } + } + return false +} + +func (sourced sourcedJob) bindInstanceDependencies(key string, instance *JobInstance, byLogicalID map[string][]JobInstance) ([]error, bool) { + job := sourced.Job + var diagnostics []error + for _, need := range sortedKeys(sourced.reusable.needBindings) { + binding := sourced.reusable.needBindings[need] + var members []string + for _, member := range binding.members { + for _, prerequisite := range byLogicalID[member] { + members = append(members, prerequisite.Key) + } + } + sort.Strings(members) + if len(members) == 0 { + diagnostics = append(diagnostics, sourced.graphFinding(key, jobError(sourced.reusable.provenance.path, job, fmt.Sprintf("prerequisite %q has no expanded instances", need)))) + return diagnostics, true + } + if instance.NeedGroups == nil { + instance.NeedGroups = make(map[string][]string, len(sourced.reusable.needBindings)) + } + instance.NeedGroups[need] = members + instance.Needs = append(instance.Needs, members...) + if binding.projectOutputs { + diagnostics = append(diagnostics, sourced.projectNeedOutputs(need, binding, instance, byLogicalID)...) + } + } + deferredInputs, dependencies, err := resolveDeferredInputBindings(sourced.reusable.inputs.deferred, byLogicalID) + if err != nil { + diagnostics = append(diagnostics, sourced.graphFinding(key, jobError(sourced.reusable.provenance.path, job, fmt.Sprintf("resolve deferred reusable-workflow inputs: %v", err)))) + return diagnostics, true + } + instance.DeferredInputs = deferredInputs + instance.Needs = append(instance.Needs, dependencies...) + for guardIndex, guard := range sourced.reusable.callGuards { + groups, outputs, dependencies, err := resolveCallGuardBindings(guard.needBindings, byLogicalID) + if err != nil { + diagnostics = append(diagnostics, sourced.graphFinding(key, jobError(sourced.reusable.provenance.path, job, fmt.Sprintf("resolve reusable-workflow call guard: %v", err)))) + return diagnostics, true + } + instance.CallGuards[guardIndex].NeedGroups = groups + instance.CallGuards[guardIndex].NeedOutputs = outputs + instance.Needs = append(instance.Needs, dependencies...) + instance.CallGuards[guardIndex].DeferredInputs, dependencies, err = resolveDeferredInputBindings(guard.inputs.deferred, byLogicalID) + if err != nil { + diagnostics = append(diagnostics, sourced.graphFinding(key, jobError(sourced.reusable.provenance.path, job, fmt.Sprintf("resolve reusable-workflow call guard inputs: %v", err)))) + return diagnostics, true + } + instance.Needs = append(instance.Needs, dependencies...) + } + sort.Strings(instance.Needs) + instance.Needs = slices.Compact(instance.Needs) + return diagnostics, false +} + +func (sourced sourcedJob) graphFinding(key string, err error) error { + return attributedProcessingFinding(StageGraph, CodeGraphInvalid, "compatibility", sourced.reusable.provenance.path, 0, 0, sourced.ID, key, "", 0, err) +} + +func (sourced sourcedJob) projectNeedOutputs(need string, binding needBinding, instance *JobInstance, byLogicalID map[string][]JobInstance) []error { + if instance.NeedOutputs == nil { + instance.NeedOutputs = make(map[string][]NeedOutput) + } + var diagnostics []error + projected := []NeedOutput{} + for _, output := range binding.outputs { + producers := byLogicalID[output.member] + if len(producers) == 0 { + diagnostics = append(diagnostics, processingFinding(StageGraph, CodeGraphInvalid, "compatibility", fmt.Errorf("%s:%d:%d: workflow_call output %q selects unexpanded job %q", output.path, output.span.Start.Line, output.span.Start.Column, output.name, output.member))) + continue + } + if len(projected)+len(producers) > plan.MaxNeedOutputs { + diagnostics = append(diagnostics, processingFinding(StageGraph, CodeGraphInvalid, "compatibility", fmt.Errorf("%s:%d:%d: workflow_call output %q expands call projections beyond the maximum of %d", output.path, output.span.Start.Line, output.span.Start.Column, output.name, plan.MaxNeedOutputs))) + continue + } + for _, producer := range producers { + projected = append(projected, NeedOutput{Name: output.name, StepKey: producer.Key, Output: output.output}) + } + } + sortNeedOutputs(projected) + instance.NeedOutputs[need] = projected + return diagnostics +} + +func (instance JobInstance) lowerReusablePlan(job *plan.Job, planDigests map[string]string, workflowName string) error { + needSources, err := buildPlanNeedSources(instance, planDigests) + if err != nil { + return err + } + deferredInputs, err := buildPlanDeferredInputs(instance.DeferredInputs, planDigests) + if err != nil { + return fmt.Errorf("build plan for job %q: %w", instance.LogicalJobID, err) + } + callGuards, err := buildPlanCallGuards(instance, planDigests) + if err != nil { + return err + } + job.Workflow = plan.Workflow{ + Path: instance.SourcePath, Name: workflowName, Digest: instance.SourceDigest, + LogicalJobID: instance.LogicalJobID, Remote: planRemoteWorkflowSource(instance.RemoteWorkflow), + } + job.Inputs = cloneAnyMap(instance.Inputs) + job.DeferredInputs = deferredInputs + job.Dependencies = append([]string(nil), instance.Needs...) + job.NeedSources = needSources + job.NeedOutputs = buildPlanNeedOutputs(instance) + job.CallGuards = callGuards + return nil +} + +func buildPlanNeedSources(instance JobInstance, planDigests map[string]string) (map[string][]plan.NeedSource, error) { + needSources := make(map[string][]plan.NeedSource, len(instance.NeedGroups)) + for _, logicalNeed := range sortedKeys(instance.NeedGroups) { + dependencies := instance.NeedGroups[logicalNeed] + if len(dependencies) > plan.MaxNeedProducers { + return nil, fmt.Errorf("build plan for job %q: prerequisite %q has %d producers, maximum is %d", instance.LogicalJobID, logicalNeed, len(dependencies), plan.MaxNeedProducers) + } + for _, dependency := range dependencies { + digest, ok := planDigests[dependency] + if !ok { + return nil, fmt.Errorf("build plan for job %q: prerequisite %q has no earlier plan digest", instance.LogicalJobID, dependency) + } + needSources[logicalNeed] = append(needSources[logicalNeed], plan.NeedSource{StepKey: dependency, PlanDigest: digest}) + } + } + return needSources, nil +} + +func buildPlanNeedOutputs(instance JobInstance) map[string][]plan.NeedOutput { + if len(instance.NeedOutputs) == 0 { + return nil + } + needOutputs := make(map[string][]plan.NeedOutput, len(instance.NeedOutputs)) + for _, logicalNeed := range sortedKeys(instance.NeedOutputs) { + outputs := instance.NeedOutputs[logicalNeed] + needOutputs[logicalNeed] = make([]plan.NeedOutput, len(outputs)) + for i, output := range outputs { + needOutputs[logicalNeed][i] = plan.NeedOutput{Name: output.Name, StepKey: output.StepKey, Output: output.Output} + } + } + return needOutputs +} + +func buildPlanDeferredInputs(inputs map[string]DeferredInput, planDigests map[string]string) (map[string]plan.DeferredInput, error) { + if len(inputs) == 0 { + return nil, nil + } + resolved := make(map[string]plan.DeferredInput, len(inputs)) + for _, name := range sortedKeys(inputs) { + input := inputs[name] + deferred := plan.DeferredInput{Outputs: make([]plan.NeedOutput, len(input.Outputs))} + for _, source := range input.Sources { + digest, ok := planDigests[source] + if !ok { + return nil, fmt.Errorf("deferred input %q source %q has no earlier plan digest", name, source) + } + deferred.Sources = append(deferred.Sources, plan.NeedSource{StepKey: source, PlanDigest: digest}) + } + for i, output := range input.Outputs { + deferred.Outputs[i] = plan.NeedOutput{Name: output.Name, StepKey: output.StepKey, Output: output.Output} + } + resolved[name] = deferred + } + return resolved, nil +} + +func buildPlanCallGuards(instance JobInstance, planDigests map[string]string) ([]plan.CallGuard, error) { + callGuards := make([]plan.CallGuard, len(instance.CallGuards)) + for guardIndex, guard := range instance.CallGuards { + deferredInputs, err := buildPlanDeferredInputs(guard.DeferredInputs, planDigests) + if err != nil { + return nil, fmt.Errorf("build plan for job %q call guard %d: %w", instance.LogicalJobID, guardIndex+1, err) + } + planGuard := plan.CallGuard{Condition: guard.Condition, Inputs: cloneAnyMap(guard.Inputs), DeferredInputs: deferredInputs} + if len(guard.NeedGroups) != 0 { + planGuard.NeedSources = make(map[string][]plan.NeedSource, len(guard.NeedGroups)) + for _, logicalNeed := range sortedKeys(guard.NeedGroups) { + for _, dependency := range guard.NeedGroups[logicalNeed] { + digest, ok := planDigests[dependency] + if !ok { + return nil, fmt.Errorf("build plan for job %q: call guard prerequisite %q has no earlier plan digest", instance.LogicalJobID, dependency) + } + planGuard.NeedSources[logicalNeed] = append(planGuard.NeedSources[logicalNeed], plan.NeedSource{StepKey: dependency, PlanDigest: digest}) + } + } + } + if len(guard.NeedOutputs) != 0 { + planGuard.NeedOutputs = make(map[string][]plan.NeedOutput, len(guard.NeedOutputs)) + for _, logicalNeed := range sortedKeys(guard.NeedOutputs) { + for _, output := range guard.NeedOutputs[logicalNeed] { + planGuard.NeedOutputs[logicalNeed] = append(planGuard.NeedOutputs[logicalNeed], plan.NeedOutput{Name: output.Name, StepKey: output.StepKey, Output: output.Output}) + } + } + } + callGuards[guardIndex] = planGuard + } + return callGuards, nil +} + +func resolveDeferredInputBindings(bindings map[string]needBinding, byLogicalID map[string][]JobInstance) (map[string]DeferredInput, []string, error) { + if len(bindings) == 0 { + return nil, nil, nil + } + resolved := make(map[string]DeferredInput, len(bindings)) + var dependencies []string + for _, name := range sortedKeys(bindings) { + input, err := resolveDeferredInputBinding(bindings[name], byLogicalID) + if err != nil { + return nil, nil, fmt.Errorf("input %q: %w", name, err) + } + resolved[name] = input + dependencies = append(dependencies, input.Sources...) + } + return resolved, dependencies, nil +} + +func resolveDeferredInputBinding(binding needBinding, byLogicalID map[string][]JobInstance) (DeferredInput, error) { + var deferred DeferredInput + for _, member := range binding.members { + producers := byLogicalID[member] + if len(producers) == 0 { + return DeferredInput{}, fmt.Errorf("source job %q has no expanded instances", member) + } + for _, producer := range producers { + deferred.Sources = append(deferred.Sources, producer.Key) + } + } + sort.Strings(deferred.Sources) + deferred.Sources = slices.Compact(deferred.Sources) + if len(deferred.Sources) > plan.MaxNeedProducers { + return DeferredInput{}, fmt.Errorf("has %d producers, maximum is %d", len(deferred.Sources), plan.MaxNeedProducers) + } + for _, output := range binding.outputs { + producers := byLogicalID[output.member] + if len(deferred.Outputs)+len(producers) > plan.MaxNeedOutputs { + return DeferredInput{}, fmt.Errorf("output %q expands beyond the maximum of %d projections", output.output, plan.MaxNeedOutputs) + } + for _, producer := range producers { + deferred.Outputs = append(deferred.Outputs, NeedOutput{Name: output.name, StepKey: producer.Key, Output: output.output}) + } + } + sortNeedOutputs(deferred.Outputs) + return deferred, nil +} + +func resolveCallGuardBindings(bindings map[string]needBinding, byLogicalID map[string][]JobInstance) (map[string][]string, map[string][]NeedOutput, []string, error) { + if len(bindings) == 0 { + return nil, nil, nil, nil + } + groups := make(map[string][]string, len(bindings)) + var projected map[string][]NeedOutput + var dependencies []string + for _, name := range sortedKeys(bindings) { + binding := bindings[name] + var members []string + for _, member := range binding.members { + for _, producer := range byLogicalID[member] { + members = append(members, producer.Key) + } + } + sort.Strings(members) + if len(members) == 0 { + return nil, nil, nil, fmt.Errorf("prerequisite %q has no expanded instances", name) + } + groups[name] = members + dependencies = append(dependencies, members...) + if !binding.projectOutputs { + continue + } + if projected == nil { + projected = make(map[string][]NeedOutput) + } + outputs := []NeedOutput{} + for _, output := range binding.outputs { + producers := byLogicalID[output.member] + if len(producers) == 0 { + return nil, nil, nil, fmt.Errorf("output %q selects unexpanded job %q", output.name, output.member) + } + if len(outputs)+len(producers) > plan.MaxNeedOutputs { + return nil, nil, nil, fmt.Errorf("output %q expands projections beyond the maximum of %d", output.name, plan.MaxNeedOutputs) + } + for _, producer := range producers { + outputs = append(outputs, NeedOutput{Name: output.name, StepKey: producer.Key, Output: output.output}) + } + } + sortNeedOutputs(outputs) + projected[name] = outputs + } + return groups, projected, dependencies, nil +} + +func sortNeedOutputs(outputs []NeedOutput) { + sort.Slice(outputs, func(i, j int) bool { + if outputs[i].Name != outputs[j].Name { + return outputs[i].Name < outputs[j].Name + } + if outputs[i].StepKey != outputs[j].StepKey { + return outputs[i].StepKey < outputs[j].StepKey + } + return outputs[i].Output < outputs[j].Output + }) +}