x-amz-expiration header support

weed/s3api/s3_constants/extend_key.go

@@ -30,6 +30,12 @@ const (
 	ExtChecksumAlgorithm = "x-seaweedfs-checksum-algorithm"
 	ExtChecksumValue     = "x-seaweedfs-checksum-value"
 
+	// Lifecycle expiration annotation (use x-seaweedfs- prefix to avoid leaking in generic header loop).
+	// Stored by the lifecycle walker for not-yet-due objects; read by GET/HEAD handlers to emit
+	// the x-amz-expiration response header.
+	// Format: expiry-date="Mon, 02 Jan 2006 15:04:05 GMT", rule-id="<rule-id>"
+	ExtExpirationKey = "x-seaweedfs-expiration"
+
 	// Bucket Policy
 	ExtBucketPolicyKey = "Seaweed-X-Amz-Bucket-Policy"
 

weed/s3api/s3_constants/header.go

@@ -57,6 +57,9 @@ const (
 	AmzPartNumberMarker = "X-Amz-Part-Number-Marker"
 	AmzDeleteMarker     = "X-Amz-Delete-Marker"
 
+	// S3 lifecycle expiration response header
+	AmzExpiration = "x-amz-expiration"
+
 	SeaweedFSUploadId                = "X-Seaweedfs-Upload-Id"
 	SeaweedFSMultipartPartsCount     = "X-Seaweedfs-Multipart-Parts-Count"
 	SeaweedFSMultipartPartBoundaries = "X-Seaweedfs-Multipart-Part-Boundaries" // JSON: [{part:1,start:0,end:2,etag:"abc"},{part:2,start:2,end:3,etag:"def"}]

weed/s3api/s3api_object_handlers.go

@@ -2048,6 +2048,13 @@ func (s3a *S3ApiServer) setResponseHeaders(w http.ResponseWriter, r *http.Reques
 		}
 	}
 
+	// Set x-amz-expiration if the lifecycle walker annotated this object.
+	if entry.Extended != nil {
+		if v, ok := entry.Extended[s3_constants.ExtExpirationKey]; ok {
+			w.Header().Set(s3_constants.AmzExpiration, string(v))
+		}
+	}
+
 	// Apply S3 passthrough headers from query parameters
 	// AWS S3 supports overriding response headers via query parameters like:
 	// ?response-cache-control=no-cache&response-content-type=application/json

weed/s3api/s3lifecycle/bootstrap/walker.go

@@ -57,6 +57,11 @@ type ListFunc func(ctx context.Context, bucket, start string, cb func(*Entry) er
 // the caller decides whether to retry from the recorded last_scanned_path.
 type Dispatcher interface {
 	Delete(ctx context.Context, action *engine.CompiledAction, entry *Entry) error
+	// Annotate persists the computed expiration date for a not-yet-due object so
+	// GET/HEAD handlers can return the x-amz-expiration response header without
+	// re-evaluating lifecycle rules on every request. Errors are non-fatal: a
+	// missing annotation is a missing header, not a data-loss event.
+	Annotate(ctx context.Context, bucket string, entry *Entry, expiresAt time.Time, ruleID string) error
 }
 
 // Checkpoint is the resume state. Caller persists it under
@@ -137,6 +142,12 @@ func walkEntry(ctx context.Context, snap *engine.Snapshot, bucket string, entry
 		NoncurrentIndex:  entry.NoncurrentIndex,
 		Tags:             entry.Tags,
 	}
+	// earliestExpiry tracks the soonest expiration across all matching expiration-kind
+	// rules that are not yet due. Used to annotate the object after the loop.
+	var earliestExpiry time.Time
+	var expiryRuleID string
+	dispatched := false
+
 	for _, key := range keys {
 		action := snap.Action(key)
 		if action == nil {
@@ -167,15 +178,34 @@ func walkEntry(ctx context.Context, snap *engine.Snapshot, bucket string, entry
 		}
 		res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, now)
 		if res.Action == s3lifecycle.ActionNone {
+			// Not yet due: capture the earliest expiration date so we can annotate.
+			if key.ActionKind == s3lifecycle.ActionKindExpirationDays ||
+				key.ActionKind == s3lifecycle.ActionKindExpirationDate {
+				dueAt := s3lifecycle.ComputeDueAt(action.Rule, key.ActionKind, info)
+				if !dueAt.IsZero() && (earliestExpiry.IsZero() || dueAt.Before(earliestExpiry)) {
+					earliestExpiry = dueAt
+					expiryRuleID = action.Rule.ID
+				}
+			}
 			continue
 		}
 		if err := dispatch.Delete(ctx, action, entry); err != nil {
 			glog.Warningf("lifecycle bootstrap: dispatch %s/%s kind=%s: %v",
 				bucket, entry.Path, key.ActionKind, err)
 			return err
 		}
+		dispatched = true
 		stats.S3LifecycleBootstrapDispatchCounter.WithLabelValues(bucket, key.ActionKind.String()).Inc()
 	}
+
+	// Annotate not-yet-expired objects so GET/HEAD can return x-amz-expiration.
+	// Skip if the object was already dispatched for deletion.
+	if !dispatched && !earliestExpiry.IsZero() {
+		if err := dispatch.Annotate(ctx, bucket, entry, earliestExpiry, expiryRuleID); err != nil {
+			glog.Warningf("lifecycle bootstrap: annotate %s/%s: %v", bucket, entry.Path, err)
+			// Non-fatal: a missing annotation is just a missing response header.
+		}
+	}
 	return nil
 }
 

weed/s3api/s3lifecycle/bootstrap/walker_test.go

@@ -14,15 +14,24 @@ import (
 
 // recorder captures dispatched (action, entry) pairs for assertion.
 type recorder struct {
-	calls []dispatchCall
-	err   error // when set, every Delete returns this error
+	calls         []dispatchCall
+	err           error // when set, every Delete returns this error
+	annotateCalls []annotateCall
+	annotateErr   error // when set, every Annotate returns this error
 }
 
 type dispatchCall struct {
 	kind s3lifecycle.ActionKind
 	path string
 }
 
+type annotateCall struct {
+	bucket    string
+	path      string
+	expiresAt time.Time
+	ruleID    string
+}
+
 func (r *recorder) Delete(ctx context.Context, action *engine.CompiledAction, entry *Entry) error {
 	if r.err != nil {
 		return r.err
@@ -31,6 +40,19 @@ func (r *recorder) Delete(ctx context.Context, action *engine.CompiledAction, en
 	return nil
 }
 
+func (r *recorder) Annotate(_ context.Context, bucket string, entry *Entry, expiresAt time.Time, ruleID string) error {
+	if r.annotateErr != nil {
+		return r.annotateErr
+	}
+	r.annotateCalls = append(r.annotateCalls, annotateCall{
+		bucket:    bucket,
+		path:      entry.Path,
+		expiresAt: expiresAt,
+		ruleID:    ruleID,
+	})
+	return nil
+}
+
 func mustTime(t *testing.T, s string) time.Time {
 	t.Helper()
 	tm, err := time.Parse(time.RFC3339, s)
@@ -401,6 +423,168 @@ func TestWalk_NonMPUDirectorySkipped(t *testing.T) {
 	}
 }
 
+func TestWalk_NotYetDueExpirationDaysAnnotates(t *testing.T) {
+	rule := &s3lifecycle.Rule{
+		ID:             "exp-rule",
+		Status:         s3lifecycle.StatusEnabled,
+		ExpirationDays: 30,
+	}
+	snap := compileEvDriven(t, "bk", rule)
+	mod := mustTime(t, "2024-01-01T00:00:00Z")
+	now := mod.Add(s3lifecycle.DaysToDuration(10)) // 10d in, not yet due at 30d
+
+	rec := &recorder{}
+	_, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
+		{Path: "logs/a", IsLatest: true, ModTime: mod},
+	}), rec, WalkOptions{Now: now})
+	if err != nil {
+		t.Fatalf("Walk: %v", err)
+	}
+	if len(rec.calls) != 0 {
+		t.Fatalf("not-yet-due entry must not dispatch delete, got %v", rec.calls)
+	}
+	if len(rec.annotateCalls) != 1 {
+		t.Fatalf("want 1 Annotate call, got %d", len(rec.annotateCalls))
+	}
+	ac := rec.annotateCalls[0]
+	if ac.bucket != "bk" {
+		t.Fatalf("annotate bucket want bk, got %q", ac.bucket)
+	}
+	if ac.path != "logs/a" {
+		t.Fatalf("annotate path want logs/a, got %q", ac.path)
+	}
+	if ac.ruleID != "exp-rule" {
+		t.Fatalf("annotate ruleID want exp-rule, got %q", ac.ruleID)
+	}
+	wantExpiry := mod.Add(s3lifecycle.DaysToDuration(30))
+	if !ac.expiresAt.Equal(wantExpiry) {
+		t.Fatalf("annotate expiresAt want %v, got %v", wantExpiry, ac.expiresAt)
+	}
+}
+
+func TestWalk_NotYetDueDateAnnotates(t *testing.T) {
+	expDate := mustTime(t, "2099-01-01T00:00:00Z")
+	rule := &s3lifecycle.Rule{
+		ID:             "date-rule",
+		Status:         s3lifecycle.StatusEnabled,
+		ExpirationDate: expDate,
+	}
+	snap := compileEvDriven(t, "bk", rule)
+	now := mustTime(t, "2024-06-01T00:00:00Z") // well before the expiration date
+
+	rec := &recorder{}
+	_, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
+		{Path: "obj/a", IsLatest: true, ModTime: mustTime(t, "2024-01-01T00:00:00Z")},
+	}), rec, WalkOptions{Now: now})
+	if err != nil {
+		t.Fatalf("Walk: %v", err)
+	}
+	if len(rec.calls) != 0 {
+		t.Fatalf("pre-date entry must not dispatch delete, got %v", rec.calls)
+	}
+	if len(rec.annotateCalls) != 1 {
+		t.Fatalf("want 1 Annotate call, got %d", len(rec.annotateCalls))
+	}
+	if !rec.annotateCalls[0].expiresAt.Equal(expDate) {
+		t.Fatalf("annotate expiresAt want %v, got %v", expDate, rec.annotateCalls[0].expiresAt)
+	}
+	if rec.annotateCalls[0].ruleID != "date-rule" {
+		t.Fatalf("annotate ruleID want date-rule, got %q", rec.annotateCalls[0].ruleID)
+	}
+}
+
+func TestWalk_DueActionDoesNotAnnotate(t *testing.T) {
+	rule := &s3lifecycle.Rule{
+		ID:             "exp-rule",
+		Status:         s3lifecycle.StatusEnabled,
+		ExpirationDays: 30,
+	}
+	snap := compileEvDriven(t, "bk", rule)
+	mod := mustTime(t, "2024-01-01T00:00:00Z")
+	now := mod.Add(s3lifecycle.DaysToDuration(60)) // well past the 30d threshold
+
+	rec := &recorder{}
+	_, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
+		{Path: "obj/a", IsLatest: true, ModTime: mod},
+	}), rec, WalkOptions{Now: now})
+	if err != nil {
+		t.Fatalf("Walk: %v", err)
+	}
+	if len(rec.calls) != 1 || rec.calls[0].path != "obj/a" {
+		t.Fatalf("due entry must dispatch delete, got %v", rec.calls)
+	}
+	if len(rec.annotateCalls) != 0 {
+		t.Fatalf("dispatched-for-delete entry must not be annotated, got %v", rec.annotateCalls)
+	}
+}
+
+func TestWalk_EarliestExpirationAnnotated(t *testing.T) {
+	// Two rules both matching the object; r2 expires sooner. Walker must
+	// annotate with the earliest expiry date and its rule ID.
+	mod := mustTime(t, "2024-01-01T00:00:00Z")
+	r1 := &s3lifecycle.Rule{
+		ID:             "r1",
+		Status:         s3lifecycle.StatusEnabled,
+		ExpirationDays: 30, // expires in 30d
+	}
+	r2 := &s3lifecycle.Rule{
+		ID:             "r2",
+		Status:         s3lifecycle.StatusEnabled,
+		ExpirationDays: 10, // expires in 10d — earliest
+	}
+	snap := compileEvDriven(t, "bk", r1, r2)
+	now := mod.Add(s3lifecycle.DaysToDuration(5)) // 5d in, neither is due
+
+	rec := &recorder{}
+	_, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
+		{Path: "obj/a", IsLatest: true, ModTime: mod},
+	}), rec, WalkOptions{Now: now})
+	if err != nil {
+		t.Fatalf("Walk: %v", err)
+	}
+	if len(rec.calls) != 0 {
+		t.Fatalf("neither rule is due, no delete expected, got %v", rec.calls)
+	}
+	if len(rec.annotateCalls) != 1 {
+		t.Fatalf("want exactly 1 Annotate call, got %d", len(rec.annotateCalls))
+	}
+	wantExpiry := mod.Add(s3lifecycle.DaysToDuration(10))
+	if !rec.annotateCalls[0].expiresAt.Equal(wantExpiry) {
+		t.Fatalf("want earliest expiry %v, got %v", wantExpiry, rec.annotateCalls[0].expiresAt)
+	}
+	if rec.annotateCalls[0].ruleID != "r2" {
+		t.Fatalf("want ruleID r2 (earliest), got %q", rec.annotateCalls[0].ruleID)
+	}
+}
+
+func TestWalk_AnnotateErrorIsNonFatal(t *testing.T) {
+	// Annotate failure must not halt the walk — a missing annotation is
+	// a missing response header, not a data-loss event.
+	rule := &s3lifecycle.Rule{
+		ID:             "r",
+		Status:         s3lifecycle.StatusEnabled,
+		ExpirationDays: 30,
+	}
+	snap := compileEvDriven(t, "bk", rule)
+	mod := mustTime(t, "2024-01-01T00:00:00Z")
+	now := mod.Add(s3lifecycle.DaysToDuration(5)) // not yet due
+
+	rec := &recorder{annotateErr: errors.New("filer write failed")}
+	cp, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
+		{Path: "obj/a", IsLatest: true, ModTime: mod},
+		{Path: "obj/b", IsLatest: true, ModTime: mod},
+	}), rec, WalkOptions{Now: now})
+	if err != nil {
+		t.Fatalf("Walk must not fail on Annotate error, got %v", err)
+	}
+	if !cp.Completed {
+		t.Fatalf("walk should complete despite Annotate error")
+	}
+	if cp.LastScannedPath != "obj/b" {
+		t.Fatalf("walk should scan all entries, checkpoint want obj/b, got %q", cp.LastScannedPath)
+	}
+}
+
 func TestWalk_MPUInitDoesNotFireNoncurrent(t *testing.T) {
 	// Same rule covers both AbortMPU and NoncurrentVersionExpiration; the
 	// MPU init record must dispatch only the AbortMPU action. Without the

weed/s3api/s3lifecycle/dailyrun/walk_buckets_test.go

@@ -31,6 +31,10 @@ func (d *recordingDispatcher) Delete(_ context.Context, action *engine.CompiledA
 	return d.err
 }
 
+func (d *recordingDispatcher) Annotate(_ context.Context, _ string, _ *bootstrap.Entry, _ time.Time, _ string) error {
+	return nil
+}
+
 // findShardForPath returns the shard ID for an entry with given path
 // in given bucket. Helper for tests that want to force entries onto
 // a known shard.