Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion e2e/consumermodule/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ require (
github.com/aws/smithy-go v1.27.7 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/block/mysql v0.0.0-20260906224346-ee0a93fe50d6 // indirect
github.com/block/pg-sprite v0.2.0 // indirect
github.com/block/pg-sprite v0.3.1 // indirect
github.com/block/spirit v0.17.1-0.20260907005557-10804bbe247c // indirect
github.com/bradleyfalzon/ghinstallation/v2 v2.18.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
Expand Down
4 changes: 2 additions & 2 deletions e2e/consumermodule/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/block/mysql v0.0.0-20260906224346-ee0a93fe50d6 h1:GvubwsqXHanJkhBotCs4XdEmSnwzhHQe7DVGrn+NFok=
github.com/block/mysql v0.0.0-20260906224346-ee0a93fe50d6/go.mod h1:KEo73lbxXs9cFlq+x3Z35UqGg3MTxAPfjDOR/ob/iik=
github.com/block/pg-sprite v0.2.0 h1:H6w/MNJf1rc7XtdVEI0Sq63I2+MkiifPgkS3qfZ9Rz8=
github.com/block/pg-sprite v0.2.0/go.mod h1:vZxHdTMrCOPAYgswveB7PSjOaOuRgnDLGRw6WoOizRg=
github.com/block/pg-sprite v0.3.1 h1:l2w3aAFql+RvKnXoE7aHh1xx0Umj6pRsnjhsSUeEB0g=
github.com/block/pg-sprite v0.3.1/go.mod h1:vZxHdTMrCOPAYgswveB7PSjOaOuRgnDLGRw6WoOizRg=
github.com/block/spirit v0.17.1-0.20260907005557-10804bbe247c h1:Gdd1vWs0UKLlvq84+4wMveyls7QkFqhQDpF21KR2cIA=
github.com/block/spirit v0.17.1-0.20260907005557-10804bbe247c/go.mod h1:Lg97/e4zr2X3AQXRUrDVAQZOVqFDZ09px503h4v/Yss=
github.com/block/vitess v0.0.0-20260907005807-88d15fda31ea h1:t9VROoN/aCzwwh7Ap90TLijZz3rZHZ6HUgWymC+yVS8=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.44.5
github.com/aws/aws-sdk-go-v2/service/sts v1.43.3
github.com/block/mysql v0.0.0-20260906224346-ee0a93fe50d6
github.com/block/pg-sprite v0.2.0
github.com/block/pg-sprite v0.3.1
github.com/block/spirit v0.17.1-0.20260907005557-10804bbe247c
github.com/bradleyfalzon/ghinstallation/v2 v2.18.0
github.com/charmbracelet/bubbles v1.0.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/block/mysql v0.0.0-20260906224346-ee0a93fe50d6 h1:GvubwsqXHanJkhBotCs4XdEmSnwzhHQe7DVGrn+NFok=
github.com/block/mysql v0.0.0-20260906224346-ee0a93fe50d6/go.mod h1:KEo73lbxXs9cFlq+x3Z35UqGg3MTxAPfjDOR/ob/iik=
github.com/block/pg-sprite v0.2.0 h1:H6w/MNJf1rc7XtdVEI0Sq63I2+MkiifPgkS3qfZ9Rz8=
github.com/block/pg-sprite v0.2.0/go.mod h1:vZxHdTMrCOPAYgswveB7PSjOaOuRgnDLGRw6WoOizRg=
github.com/block/pg-sprite v0.3.1 h1:l2w3aAFql+RvKnXoE7aHh1xx0Umj6pRsnjhsSUeEB0g=
github.com/block/pg-sprite v0.3.1/go.mod h1:vZxHdTMrCOPAYgswveB7PSjOaOuRgnDLGRw6WoOizRg=
github.com/block/spirit v0.17.1-0.20260907005557-10804bbe247c h1:Gdd1vWs0UKLlvq84+4wMveyls7QkFqhQDpF21KR2cIA=
github.com/block/spirit v0.17.1-0.20260907005557-10804bbe247c/go.mod h1:Lg97/e4zr2X3AQXRUrDVAQZOVqFDZ09px503h4v/Yss=
github.com/block/vitess v0.0.0-20260907005807-88d15fda31ea h1:t9VROoN/aCzwwh7Ap90TLijZz3rZHZ6HUgWymC+yVS8=
Expand Down
136 changes: 97 additions & 39 deletions pkg/engine/postgres/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,17 +177,21 @@ func (e *Engine) runOptimisticApply(ctx context.Context, conn targetConn, change
}

var invalidErr *executor.InvalidIndexError
if errors.As(err, &invalidErr) {
// An invalid index — pre-existing or a build's own unrecovered
// leftover — is operational: an operator clears it and a retry can
// succeed. Checked before the refusal and budget arms because the
// verdict wraps the build failure that produced it (a budget-
// cancelled build leaves its own invalid index), and that inner
// cause must not be read as the outcome — the index the operator
// clears is. The detail is built from the typed identifiers and
// verdict code, never the wrapped build or cleanup errors, which
// may carry raw server text; the full cause lands in the server
// log below it.
if errors.As(err, &invalidErr) && !invalidErr.Code().Permanent() {
// An invalid index an operator can clear — a build's own leftover,
// abandoned debris, another backend's build to wait out, or a
// builder this role cannot observe — is operational: once it is
// cleared a retry can succeed. Checked before the refusal and budget
// arms because the verdict wraps the build failure that produced it
// (a budget-cancelled build leaves its own invalid index), and that
// inner cause must not be read as the outcome — the index the
// operator clears is. The permanent verdicts (the name is occupied
// on another table, or by an index the server will not drop
// concurrently) fall through to classifyRefusal: retrying unchanged
// reproduces them. The detail is built from the typed identifiers
// and verdict code, never the wrapped build or cleanup errors, which
// may carry raw server text; the full cause lands in the server log
// below it.
logger.Error("PostgreSQL concurrent index build left or found an invalid index",
"namespace", change.namespace, "table", change.table,
"index_schema", invalidErr.Schema, "index", invalidErr.Index, "error", err)
Expand Down Expand Up @@ -259,28 +263,63 @@ func classifyApplyFailure(err error, table string) applyFailure {
return applyFailure{detail: "PostgreSQL schema change failed; see server logs", retryable: true}
}

// invalidIndexDetail renders the operator-facing next step for an
// invalidIndexAdvice renders the operator-facing cause and next step for an
// invalid-index verdict, matching pg-sprite's own ownership standard: a drop
// is named only when the entry is proven this build's own leftover. A
// pre-existing invalid entry may be another actor's still-running build, and
// an unproven verdict may sit on a healthy index — both get investigation
// steps, never a statement to run. Only the typed identifiers are
// is named only where the executor proved the entry is a failed build's
// debris on the target table — this build's own leftover, or an abandoned
// entry with no builder. A build still in flight says wait; an entry on
// another table, one the server will not drop concurrently, one whose
// builder this role cannot see, and an unproven verdict get investigation
// steps, never a statement to run — the index under the name may be healthy
// or may be exactly what it is meant to be. Only the typed identifiers are
// interpolated, never the wrapped build or cleanup errors, which may carry
// raw server text.
func invalidIndexDetail(invalidErr *executor.InvalidIndexError) string {
// raw server text. The two halves leave unsanitized so classifyRefusal can
// compose a sequence-step clause between them and sanitize the whole; the
// operational path composes them through invalidIndexDetail.
func invalidIndexAdvice(invalidErr *executor.InvalidIndexError) (cause, remedy string) {
name := fmt.Sprintf("%q.%q", invalidErr.Schema, invalidErr.Index)
var advice string
switch invalidErr.Code() {
case executor.CodeInvalidIndexOwnLeftover:
advice = fmt.Sprintf("this build left its own invalid index %s on the target; drop the invalid index, then retry", name)
case executor.CodeInvalidIndexPreexisting:
advice = fmt.Sprintf("an invalid index %s already occupies the name on the target and may be another actor's build still in progress; check pg_stat_activity before any recovery, then retry", name)
return fmt.Sprintf("this build left its own invalid index %s on the target", name),
"drop the invalid index, then retry"
case executor.CodeInvalidIndexAbandoned:
return fmt.Sprintf("an abandoned invalid index %s occupies the name on the target table with no backend building it", name),
"confirm it is still invalid with no builder, drop the invalid index, then retry"
case executor.CodeInvalidIndexBuildInFlight:
return fmt.Sprintf("an invalid index %s occupies the name and backend %d is still building it", name, invalidErr.BuilderPID),
"wait for that build to finish or fail, then retry"
case executor.CodeInvalidIndexBuilderUnobservable:
return fmt.Sprintf("an invalid index %s occupies the name on the target table and the engine role cannot observe whether a backend is building it", name),
"check pg_stat_progress_create_index with a role granted pg_read_all_stats before any recovery, then retry"
case executor.CodeInvalidIndexOtherTable:
return fmt.Sprintf("an invalid index %s already occupies the name on a different table%s", name, invalidIndexTableSuffix(invalidErr)),
"this change cannot claim it — rename the index in the schema file and re-plan, or clear the entry through that table's own change"
case executor.CodeInvalidIndexNotDroppable:
return fmt.Sprintf("an invalid index %s occupies the name and is a partitioned table's index, an index partition, or a constraint's index rather than a failed build's leftover", name),
"an operator must resolve it on the target, or rename the index in the schema file and re-plan"
default:
// CodeInvalidIndexUnproven and any future verdict fail safe with
// investigation steps: the index under the name may be healthy.
advice = fmt.Sprintf("index %s may be invalid but its catalog state could not be verified; inspect pg_index.indisvalid on the target before any recovery, then retry", name)
return fmt.Sprintf("index %s may be invalid but its catalog state could not be verified", name),
"inspect pg_index.indisvalid on the target before any recovery, then retry"
}
return sanitizeReasonText(advice)
}

// invalidIndexDetail composes the advice for the operational (retryable)
// publish path, where no sequence-step clause intervenes.
func invalidIndexDetail(invalidErr *executor.InvalidIndexError) string {
cause, remedy := invalidIndexAdvice(invalidErr)
return sanitizeReasonText(cause + "; " + remedy)
}

// invalidIndexTableSuffix names the table the invalid index sits on when the
// catalog inspection saw it; the verdict carries no table when the state
// could not be inspected.
func invalidIndexTableSuffix(invalidErr *executor.InvalidIndexError) string {
if invalidErr.Table == "" {
return ""
}
return fmt.Sprintf(" (%q)", invalidErr.Table)
}

// refusal is a typed apply outcome that retrying cannot fix: the schema
Expand Down Expand Up @@ -388,15 +427,21 @@ func refusalForCause(err error, table string) *refusal {
cause: fmt.Sprintf("the engine role lacks access for %s %s", privilegeErr.Tier, object),
remedy: remedy}
}
// An invalid-index verdict is operational even when the build failure it
// wraps would classify as a refusal on its own — a budget-cancelled
// concurrent build leaves its own invalid index, and the index the
// operator clears is the outcome, not the inner budget exhaustion.
// Declined before the budget arm so the nested cause can never shadow
// the verdict.
// An invalid-index verdict is decided by its own code, never by the build
// failure it wraps — a budget-cancelled concurrent build leaves its own
// invalid index, and the index the operator clears is the outcome, not
// the inner budget exhaustion. Decided before the budget arm so the
// nested cause can never shadow the verdict. Only the permanent members
// of the family refuse: the name is occupied on another table, or by an
// index the server will not drop concurrently, so retrying unchanged
// reproduces the verdict. Every other member is operational.
var invalidErr *executor.InvalidIndexError
if errors.As(err, &invalidErr) {
return nil
r, _ := refusalForOutcome(invalidErr.Code(), table)
if r != nil {
r.cause, r.remedy = invalidIndexAdvice(invalidErr)
}
return r
}
var budgetErr *executor.BudgetError
if errors.As(err, &budgetErr) && budgetErr.Cause == executor.CauseStatement {
Expand Down Expand Up @@ -485,14 +530,27 @@ func refusalForOutcome(code executor.Code, table string) (*refusal, bool) {
return &refusal{reason: "engine-invariant-violation",
cause: fmt.Sprintf("the engine's safety invariants did not hold while changing table %q", table),
remedy: "inspect the target and server logs before re-running"}, true
case executor.CodeBudgetLockExceeded, executor.CodeCancelledExternally,
executor.CodeInvalidIndexOwnLeftover, executor.CodeInvalidIndexPreexisting,
executor.CodeInvalidIndexUnproven, executor.CodePoolTooSmall,
executor.CodeExecutionFailed:
// Operational outcomes: a bounded lock race, an external stop, an
// invalid-index state an operator clears, engine pool sizing, or a
// failure outside the typed set. A retry can succeed once
// conditions change, so none is a permanent refusal.
case executor.CodeInvalidIndexOtherTable, executor.CodeInvalidIndexNotDroppable:
// The permanent members of the invalid-index family: the requested
// name is held by an entry this change can never clear — an invalid
// index on a different table, or one the server will not drop
// concurrently (a partitioned table's index, an index partition, a
// constraint's index). Retrying unchanged reproduces the verdict.
// The typed-verdict path replaces this cause and remedy with the
// code's own advice; this mapping keeps the vocabulary total.
return &refusal{reason: "invalid-index-occupied",
cause: fmt.Sprintf("an invalid index already occupies a name the change to %q needs and is not a failed build's leftover", table),
remedy: "rename the index in the schema file and re-plan, or resolve the entry on the target"}, true
case executor.CodeBudgetLockExceeded, executor.CodeCancelledByCaller,
executor.CodeCancelledExternally, executor.CodeInvalidIndexOwnLeftover,
executor.CodeInvalidIndexAbandoned, executor.CodeInvalidIndexBuildInFlight,
executor.CodeInvalidIndexBuilderUnobservable, executor.CodeInvalidIndexUnproven,
executor.CodePoolTooSmall, executor.CodeExecutionFailed:
// Operational outcomes: a bounded lock race, the caller's own
// context ending or an external stop, an invalid-index state an
// operator clears or waits out, engine pool sizing, or a failure
// outside the typed set. A retry can succeed once conditions
// change, so none is a permanent refusal.
return nil, true
}
return nil, false
Expand Down
Loading
Loading