feat(watcher): add --required_annotations to hold the finalizer until extra keys are true

cmd/watcher/main.go

@@ -85,6 +85,7 @@ var (
 	storeDeadline                = flag.Duration("store_deadline", 10*time.Minute, "How long to wait for storing the PipelineRun and TaskRun resources before aborting and clearing the finalizer in case of delete event")
 	forwardBuffer                = flag.Duration("forward_buffer", 150*time.Second, "This determines duration since completion time of TaskRun to wait for forwarder to finish")
 	managedByValues              = flag.String("managed_by_values", "", "Comma-separated list of additional spec.managedBy values the watcher will process. Runs with unset, empty, whitespace-only or \"tekton.dev/pipeline\" managedBy values are always accepted.")
+	requiredAnnotations          = flag.String("required_annotations", "", "Comma-separated list of annotation keys that must all be present and set to \"true\" before the watcher clears its finalizer. The stored annotation is always implicitly required. When empty, only the stored annotation is checked.")
 )
 
 func main() {
@@ -139,6 +140,7 @@ func main() {
 		SummaryAnnotations:           *summaryAnnotations,
 		DisableStoringIncompleteRuns: *disableStoringIncompleteRuns,
 		AllowedManagedByValues:       reconciler.ParseManagedByValues(*managedByValues),
+		RequiredAnnotations:          reconciler.ParseRequiredAnnotations(*requiredAnnotations),
 	}
 
 	log.Printf("dynamic reconcile timeout %s and update log timeout is %s", cfg.DynamicReconcileTimeout.String(), cfg.UpdateLogTimeout.String())

docs/watcher/README.md

@@ -76,6 +76,27 @@ Watcher implements a finalizer to block deletion by an external pruner when obje
 
 When deletion request comes, it will block until completion time + `completed_run_grace_period` period is passed. A hard limit could be set as `store_deadline` (default 10m), after which the object will be removed from the cluster even without confirmation it's been stored in the DB.
 
+### Required annotations for finalizer release
+
+The `required_annotations` flag accepts a comma-separated list of annotation keys that must all be present and set to `"true"` on a PipelineRun or TaskRun before the Watcher clears its finalizer. This is useful in multicluster scenarios where external controllers (e.g. a Hub scheduler or syncer service) need to write annotations on a resource before it can be safely deleted.
+
+The `results.tekton.dev/stored` annotation is always implicitly required and does not need to be listed. When the flag is empty or unset (the default), only the stored annotation is checked and behavior is unchanged from previous versions.
+
+For example, to require that a Hub scheduler has annotated the resource before the finalizer is cleared:
+
+```
+--required_annotations=hub.example.com/scheduled
+```
+
+Multiple annotations:
+
+```
+--required_annotations=hub.example.com/scheduled,syncer.example.com/synced
+```
+
+If any required annotation is missing or not set to `"true"`, the Watcher re-queues the resource and checks again after the `FinalizerRequeueInterval` (10 seconds). The `store_deadline` safety limit still applies — if the deadline passes, the finalizer is cleared regardless of whether the required annotations are present.
+
+> **Note:** This flag applies to both PipelineRun and TaskRun resources. CustomRuns are not affected.
 
 ## Filtering by `spec.managedBy`
 

pkg/watcher/reconciler/config.go

@@ -16,6 +16,7 @@
 package reconciler
 
 import (
+	"strings"
 	"time"
 
 	"k8s.io/apimachinery/pkg/labels"
@@ -83,6 +84,13 @@ type Config struct {
 	// included. Runs with nil or empty managedBy are also always accepted.
 	// This set must not be mutated after initialization to avoid data races.
 	AllowedManagedByValues sets.Set[string]
+
+	// RequiredAnnotations is a list of annotation keys that must all be
+	// present and set to "true" on a resource before the watcher clears
+	// its finalizer. The stored annotation (results.tekton.dev/stored) is
+	// always implicitly required and does not need to be listed.
+	// When empty, only the stored annotation is checked (existing behavior).
+	RequiredAnnotations []string
 }
 
 // GetDisableAnnotationUpdate returns whether annotation updates should be
@@ -129,3 +137,33 @@ func (c *Config) SetLabelSelector(selector string) error {
 	c.labelSelector = parsedSelector
 	return nil
 }
+
+// ParseRequiredAnnotations parses a comma-separated list of annotation keys
+// into a string slice. Empty entries are ignored.
+func ParseRequiredAnnotations(raw string) []string {
+	if raw == "" {
+		return nil
+	}
+	var result []string
+	for _, v := range strings.Split(raw, ",") {
+		if trimmed := strings.TrimSpace(v); trimmed != "" {
+			result = append(result, trimmed)
+		}
+	}
+	return result
+}
+
+// AreRequiredAnnotationsReady checks whether all required annotations are
+// present and set to "true" on the given annotations map. Returns the key of
+// the first missing/non-true annotation, or "" if all are satisfied.
+func (c *Config) AreRequiredAnnotationsReady(annotations map[string]string) (missingKey string, ready bool) {
+	if c == nil || len(c.RequiredAnnotations) == 0 {
+		return "", true
+	}
+	for _, key := range c.RequiredAnnotations {
+		if annotations[key] != "true" {
+			return key, false
+		}
+	}
+	return "", true
+}

pkg/watcher/reconciler/config_test.go

@@ -44,6 +44,110 @@ func TestGetDisableAnnotationUpdate(t *testing.T) {
 	}
 }
 
+func TestParseRequiredAnnotations(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		{name: "empty string", input: "", want: nil},
+		{name: "single value", input: "hub.example.com/scheduled", want: []string{"hub.example.com/scheduled"}},
+		{name: "multiple values", input: "hub.example.com/scheduled,syncer.example.com/synced", want: []string{"hub.example.com/scheduled", "syncer.example.com/synced"}},
+		{name: "whitespace handling", input: " hub.example.com/scheduled , syncer.example.com/synced ", want: []string{"hub.example.com/scheduled", "syncer.example.com/synced"}},
+		{name: "trailing comma ignored", input: "hub.example.com/scheduled,", want: []string{"hub.example.com/scheduled"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			got := ParseRequiredAnnotations(tc.input)
+			if len(got) != len(tc.want) {
+				t.Fatalf("ParseRequiredAnnotations(%q) = %v (len %d), want %v (len %d)", tc.input, got, len(got), tc.want, len(tc.want))
+			}
+			for i := range got {
+				if got[i] != tc.want[i] {
+					t.Errorf("ParseRequiredAnnotations(%q)[%d] = %q, want %q", tc.input, i, got[i], tc.want[i])
+				}
+			}
+		})
+	}
+}
+
+func TestAreRequiredAnnotationsReady(t *testing.T) {
+	for _, tc := range []struct {
+		name        string
+		cfg         *Config
+		annotations map[string]string
+		wantReady   bool
+		wantMissing string
+	}{
+		{
+			name:        "nil config is always ready",
+			cfg:         nil,
+			annotations: nil,
+			wantReady:   true,
+		},
+		{
+			name:        "empty required list is always ready",
+			cfg:         &Config{},
+			annotations: nil,
+			wantReady:   true,
+		},
+		{
+			name:        "required annotation present and true",
+			cfg:         &Config{RequiredAnnotations: []string{"hub.example.com/scheduled"}},
+			annotations: map[string]string{"hub.example.com/scheduled": "true"},
+			wantReady:   true,
+		},
+		{
+			name:        "required annotation missing",
+			cfg:         &Config{RequiredAnnotations: []string{"hub.example.com/scheduled"}},
+			annotations: map[string]string{},
+			wantReady:   false,
+			wantMissing: "hub.example.com/scheduled",
+		},
+		{
+			name:        "required annotation present but not true",
+			cfg:         &Config{RequiredAnnotations: []string{"hub.example.com/scheduled"}},
+			annotations: map[string]string{"hub.example.com/scheduled": "false"},
+			wantReady:   false,
+			wantMissing: "hub.example.com/scheduled",
+		},
+		{
+			name:        "nil annotations map",
+			cfg:         &Config{RequiredAnnotations: []string{"hub.example.com/scheduled"}},
+			annotations: nil,
+			wantReady:   false,
+			wantMissing: "hub.example.com/scheduled",
+		},
+		{
+			name: "multiple required - all satisfied",
+			cfg:  &Config{RequiredAnnotations: []string{"hub.example.com/scheduled", "syncer.example.com/synced"}},
+			annotations: map[string]string{
+				"hub.example.com/scheduled": "true",
+				"syncer.example.com/synced": "true",
+			},
+			wantReady: true,
+		},
+		{
+			name: "multiple required - one missing",
+			cfg:  &Config{RequiredAnnotations: []string{"hub.example.com/scheduled", "syncer.example.com/synced"}},
+			annotations: map[string]string{
+				"hub.example.com/scheduled": "true",
+			},
+			wantReady:   false,
+			wantMissing: "syncer.example.com/synced",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			missingKey, ready := tc.cfg.AreRequiredAnnotationsReady(tc.annotations)
+			if ready != tc.wantReady {
+				t.Errorf("AreRequiredAnnotationsReady() ready = %t, want %t", ready, tc.wantReady)
+			}
+			if missingKey != tc.wantMissing {
+				t.Errorf("AreRequiredAnnotationsReady() missingKey = %q, want %q", missingKey, tc.wantMissing)
+			}
+		})
+	}
+}
+
 func TestCompletedResourceGracePeriod(t *testing.T) {
 	for _, tc := range []struct {
 		cfg  *Config

pkg/watcher/reconciler/pipelinerun/reconciler.go

@@ -318,6 +318,12 @@ func (r *Reconciler) finalize(ctx context.Context, pr *pipelinev1.PipelineRun, r
 		return controller.NewRequeueAfter(r.cfg.FinalizerRequeueInterval)
 	}
 
+	if missingKey, ready := r.cfg.AreRequiredAnnotationsReady(pr.Annotations); !ready {
+		logging.FromContext(ctx).Debugf("required annotation %q is not yet true on pipelinerun %s/%s, requeuing",
+			missingKey, pr.Namespace, pr.Name)
+		return controller.NewRequeueAfter(r.cfg.FinalizerRequeueInterval)
+	}
+
 	return nil
 }
 

pkg/watcher/reconciler/pipelinerun/reconciler_test.go

@@ -458,6 +458,164 @@ func TestFinalize(t *testing.T) {
 			cfg:  cfg,
 			want: nil,
 		},
+		{
+			name: "required annotations not ready - requeue",
+			pr: &pipelinev1.PipelineRun{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-pr",
+					Namespace: "test-ns",
+					Annotations: map[string]string{
+						resultsannotation.Stored: "true",
+					},
+				},
+				Status: pipelinev1.PipelineRunStatus{
+					Status: duckv1.Status{
+						Conditions: duckv1.Conditions{
+							apis.Condition{
+								Type:   apis.ConditionSucceeded,
+								Status: corev1.ConditionTrue,
+							},
+						},
+					},
+					PipelineRunStatusFields: pipelinev1.PipelineRunStatusFields{
+						CompletionTime: &metav1.Time{Time: time.Now()},
+					},
+				},
+			},
+			cfg: &reconciler.Config{
+				StoreDeadline:            &storeDeadline,
+				FinalizerRequeueInterval: finalizerRequeueInterval,
+				RequiredAnnotations:      []string{"hub.example.com/scheduled"},
+			},
+			want: controller.NewRequeueAfter(finalizerRequeueInterval),
+		},
+		{
+			name: "required annotations all satisfied - allow finalization",
+			pr: &pipelinev1.PipelineRun{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-pr",
+					Namespace: "test-ns",
+					Annotations: map[string]string{
+						resultsannotation.Stored:    "true",
+						"hub.example.com/scheduled": "true",
+					},
+				},
+				Status: pipelinev1.PipelineRunStatus{
+					Status: duckv1.Status{
+						Conditions: duckv1.Conditions{
+							apis.Condition{
+								Type:   apis.ConditionSucceeded,
+								Status: corev1.ConditionTrue,
+							},
+						},
+					},
+					PipelineRunStatusFields: pipelinev1.PipelineRunStatusFields{
+						CompletionTime: &metav1.Time{Time: time.Now()},
+					},
+				},
+			},
+			cfg: &reconciler.Config{
+				StoreDeadline:            &storeDeadline,
+				FinalizerRequeueInterval: finalizerRequeueInterval,
+				RequiredAnnotations:      []string{"hub.example.com/scheduled"},
+			},
+			want: nil,
+		},
+		{
+			name: "multiple required annotations - one missing - requeue",
+			pr: &pipelinev1.PipelineRun{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-pr",
+					Namespace: "test-ns",
+					Annotations: map[string]string{
+						resultsannotation.Stored:    "true",
+						"hub.example.com/scheduled": "true",
+					},
+				},
+				Status: pipelinev1.PipelineRunStatus{
+					Status: duckv1.Status{
+						Conditions: duckv1.Conditions{
+							apis.Condition{
+								Type:   apis.ConditionSucceeded,
+								Status: corev1.ConditionTrue,
+							},
+						},
+					},
+					PipelineRunStatusFields: pipelinev1.PipelineRunStatusFields{
+						CompletionTime: &metav1.Time{Time: time.Now()},
+					},
+				},
+			},
+			cfg: &reconciler.Config{
+				StoreDeadline:            &storeDeadline,
+				FinalizerRequeueInterval: finalizerRequeueInterval,
+				RequiredAnnotations:      []string{"hub.example.com/scheduled", "syncer.example.com/synced"},
+			},
+			want: controller.NewRequeueAfter(finalizerRequeueInterval),
+		},
+		{
+			name: "required annotation present but not true - requeue",
+			pr: &pipelinev1.PipelineRun{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-pr",
+					Namespace: "test-ns",
+					Annotations: map[string]string{
+						resultsannotation.Stored:    "true",
+						"hub.example.com/scheduled": "false",
+					},
+				},
+				Status: pipelinev1.PipelineRunStatus{
+					Status: duckv1.Status{
+						Conditions: duckv1.Conditions{
+							apis.Condition{
+								Type:   apis.ConditionSucceeded,
+								Status: corev1.ConditionTrue,
+							},
+						},
+					},
+					PipelineRunStatusFields: pipelinev1.PipelineRunStatusFields{
+						CompletionTime: &metav1.Time{Time: time.Now()},
+					},
+				},
+			},
+			cfg: &reconciler.Config{
+				StoreDeadline:            &storeDeadline,
+				FinalizerRequeueInterval: finalizerRequeueInterval,
+				RequiredAnnotations:      []string{"hub.example.com/scheduled"},
+			},
+			want: controller.NewRequeueAfter(finalizerRequeueInterval),
+		},
+		{
+			name: "store deadline passed - required annotations ignored",
+			pr: &pipelinev1.PipelineRun{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-pr",
+					Namespace: "test-ns",
+					Annotations: map[string]string{
+						resultsannotation.Stored: "true",
+					},
+				},
+				Status: pipelinev1.PipelineRunStatus{
+					Status: duckv1.Status{
+						Conditions: duckv1.Conditions{
+							apis.Condition{
+								Type:   apis.ConditionSucceeded,
+								Status: corev1.ConditionTrue,
+							},
+						},
+					},
+					PipelineRunStatusFields: pipelinev1.PipelineRunStatusFields{
+						CompletionTime: &metav1.Time{Time: time.Now().Add(-2 * time.Hour)},
+					},
+				},
+			},
+			cfg: &reconciler.Config{
+				StoreDeadline:            &storeDeadline,
+				FinalizerRequeueInterval: finalizerRequeueInterval,
+				RequiredAnnotations:      []string{"hub.example.com/scheduled"},
+			},
+			want: nil,
+		},
 		{
 			name: "verify finalizer requeue interval",
 			pr: &pipelinev1.PipelineRun{