From 8d2fb4c49174d60a52daeda867aa8f038b578364 Mon Sep 17 00:00:00 2001 From: John Ramsden Date: Wed, 22 Jul 2026 13:38:43 -0700 Subject: [PATCH] feat: add radosgw-admin multisite read wrappers Add the inert read-only plumbing for RGW multisite replication (CE062): a radosgw-admin exec wrapper with a remote-cluster variant and typed, JSON-backed wrappers for realm get, zonegroup get, zone get, metadata sync status, per-source data sync status, and the mdlog/datalog head markers. Catch-up verdicts are computed deterministically from local sync markers versus the peer's log heads (readable over RADOS through an imported remote), using the same per-shard comparison rule the radosgw-admin sync status text applies internally; that text is not parsed at all since the command ignores --format json on both squid and tentacle. Failing commands yield zero-value results with a nil error, mirroring the RBD wrapper contract, so the upcoming replication handler can treat an unconfigured gateway as ordinary disabled state. Fixtures are captured from a live two-site squid deployment, covering master, secondary and unconfigured views. Nothing calls this code yet; the replication handler arrives in a follow-up. Assisted-by: claude-code:claude-fable-5 Signed-off-by: John Ramsden --- microceph/ceph/rgw_multisite.go | 342 ++++++++++++++++++ microceph/ceph/rgw_multisite_test.go | 253 +++++++++++++ .../test_assets/rgw_data_sync_status.json | 49 +++ .../ceph/test_assets/rgw_datalog_status.json | 14 + .../ceph/test_assets/rgw_mdlog_status.json | 18 + .../rgw_metadata_sync_status_master.json | 16 + .../rgw_metadata_sync_status_secondary.json | 53 +++ microceph/ceph/test_assets/rgw_realm_get.json | 6 + microceph/ceph/test_assets/rgw_zone_get.json | 43 +++ .../ceph/test_assets/rgw_zonegroup_get.json | 116 ++++++ 10 files changed, 910 insertions(+) create mode 100644 microceph/ceph/rgw_multisite.go create mode 100644 microceph/ceph/rgw_multisite_test.go create mode 100644 microceph/ceph/test_assets/rgw_data_sync_status.json create mode 100644 microceph/ceph/test_assets/rgw_datalog_status.json create mode 100644 microceph/ceph/test_assets/rgw_mdlog_status.json create mode 100644 microceph/ceph/test_assets/rgw_metadata_sync_status_master.json create mode 100644 microceph/ceph/test_assets/rgw_metadata_sync_status_secondary.json create mode 100644 microceph/ceph/test_assets/rgw_realm_get.json create mode 100644 microceph/ceph/test_assets/rgw_zone_get.json create mode 100644 microceph/ceph/test_assets/rgw_zonegroup_get.json diff --git a/microceph/ceph/rgw_multisite.go b/microceph/ceph/rgw_multisite.go new file mode 100644 index 00000000..dd0691e4 --- /dev/null +++ b/microceph/ceph/rgw_multisite.go @@ -0,0 +1,342 @@ +package ceph + +import ( + "encoding/json" + "fmt" + + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microceph/microceph/logger" +) + +// radosgwAdminRun runs radosgw-admin with the given arguments. +func radosgwAdminRun(args ...string) (string, error) { + return common.ProcessExec.RunCommand("radosgw-admin", args...) +} + +// radosgwAdminRunRemote runs radosgw-admin against an imported remote +// cluster when the cluster/client pair is non-empty, and locally otherwise. +func radosgwAdminRunRemote(cluster string, client string, args ...string) (string, error) { + args = appendRemoteClusterArgs(args, cluster, client) + return radosgwAdminRun(args...) +} + +// RgwRealm holds the subset of `radosgw-admin realm get` output consumed by +// RGW replication. +type RgwRealm struct { + ID string `json:"id"` + Name string `json:"name"` + CurrentPeriod string `json:"current_period"` + Epoch int `json:"epoch"` +} + +// RgwZoneGroupZone is one zone entry in a zonegroup map. +type RgwZoneGroupZone struct { + ID string `json:"id"` + Name string `json:"name"` + Endpoints []string `json:"endpoints"` + ReadOnly bool `json:"read_only"` +} + +// RgwZoneGroup holds the subset of `radosgw-admin zonegroup get` output +// consumed by RGW replication. +type RgwZoneGroup struct { + ID string `json:"id"` + Name string `json:"name"` + IsMaster bool `json:"is_master"` + Endpoints []string `json:"endpoints"` + MasterZone string `json:"master_zone"` + Zones []RgwZoneGroupZone `json:"zones"` + RealmID string `json:"realm_id"` +} + +// RgwZoneSystemKey is the S3 key pair bound to a zone for inter-zone sync. +type RgwZoneSystemKey struct { + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` +} + +// RgwZone holds the subset of `radosgw-admin zone get` output consumed by +// RGW replication. +type RgwZone struct { + ID string `json:"id"` + Name string `json:"name"` + SystemKey RgwZoneSystemKey `json:"system_key"` +} + +// GetRgwRealm fetches the default realm. Pass a non-empty cluster/client +// pair to target an imported remote cluster instead of the local one. +// A failing command (e.g. no realm configured) yields a zero-value realm +// with a nil error +func GetRgwRealm(cluster string, client string) (RgwRealm, error) { + response := RgwRealm{} + + output, err := radosgwAdminRunRemote(cluster, client, "realm", "get") + if err != nil { + logger.Warnf("REPRGW: failed realm get operation: %v", err) + return response, nil + } + + err = json.Unmarshal([]byte(output), &response) + if err != nil { + return response, fmt.Errorf("cannot unmarshal realm get output: %w", err) + } + + return response, nil +} + +// GetRgwZoneGroup fetches the default zonegroup. Pass a non-empty +// cluster/client pair to target an imported remote cluster instead of the +// local one. A failing command yields a zero-value zonegroup with a nil +// error +func GetRgwZoneGroup(cluster string, client string) (RgwZoneGroup, error) { + response := RgwZoneGroup{} + + output, err := radosgwAdminRunRemote(cluster, client, "zonegroup", "get") + if err != nil { + logger.Warnf("REPRGW: failed zonegroup get operation: %v", err) + return response, nil + } + + err = json.Unmarshal([]byte(output), &response) + if err != nil { + return response, fmt.Errorf("cannot unmarshal zonegroup get output: %w", err) + } + + return response, nil +} + +// GetRgwZone fetches the default zone. Pass a non-empty cluster/client pair +// to target an imported remote cluster instead of the local one. A failing +// command yields a zero-value zone with a nil error +func GetRgwZone(cluster string, client string) (RgwZone, error) { + response := RgwZone{} + + output, err := radosgwAdminRunRemote(cluster, client, "zone", "get") + if err != nil { + logger.Warnf("REPRGW: failed zone get operation: %v", err) + return response, nil + } + + err = json.Unmarshal([]byte(output), &response) + if err != nil { + return response, fmt.Errorf("cannot unmarshal zone get output: %w", err) + } + + return response, nil +} + +// RgwSyncInfo is the info block shared by `metadata sync status` and +// `data sync status` output. Period and RealmEpoch are metadata-only. +type RgwSyncInfo struct { + Status string `json:"status"` + NumShards int `json:"num_shards"` + Period string `json:"period"` + RealmEpoch int `json:"realm_epoch"` +} + +// RgwMetadataSyncMarker is one shard's metadata sync position. State is +// numeric in this output: 0 = full sync, 1 = incremental sync. +type RgwMetadataSyncMarker struct { + State int `json:"state"` + Marker string `json:"marker"` +} + +// RgwMetadataSyncShard pairs a shard id with its metadata sync marker. +type RgwMetadataSyncShard struct { + Key int `json:"key"` + Val RgwMetadataSyncMarker `json:"val"` +} + +// RgwMetadataSyncStatus is the parsed form of `radosgw-admin metadata sync +// status`. On the metadata master the command reports Status "init" with +// zero shards - the master runs no metadata sync. +type RgwMetadataSyncStatus struct { + Info RgwSyncInfo + Markers []RgwMetadataSyncShard +} + +// RgwDataSyncMarker is one shard's data sync position. Unlike the metadata +// variant, state is a string here: "full-sync" or "incremental-sync". +type RgwDataSyncMarker struct { + Status string `json:"status"` + Marker string `json:"marker"` +} + +// RgwDataSyncShard pairs a shard id with its data sync marker. +type RgwDataSyncShard struct { + Key int `json:"key"` + Val RgwDataSyncMarker `json:"val"` +} + +// RgwDataSyncStatus is the parsed form of `radosgw-admin data sync status +// --source-zone=` for one sync source. +type RgwDataSyncStatus struct { + Info RgwSyncInfo + Markers []RgwDataSyncShard +} + +type rgwMetadataSyncEnvelope struct { + SyncStatus struct { + Info RgwSyncInfo `json:"info"` + Markers []RgwMetadataSyncShard `json:"markers"` + } `json:"sync_status"` +} + +type rgwDataSyncEnvelope struct { + SyncStatus struct { + Info RgwSyncInfo `json:"info"` + Markers []RgwDataSyncShard `json:"markers"` + } `json:"sync_status"` +} + +// GetRgwMetadataSyncStatus fetches the typed metadata sync state (JSON, +// local sync markers only - no peer comparison). Pass a non-empty +// cluster/client pair to target an imported remote cluster. A failing +// command yields a zero-value status with a nil error +func GetRgwMetadataSyncStatus(cluster string, client string) (RgwMetadataSyncStatus, error) { + envelope := rgwMetadataSyncEnvelope{} + + output, err := radosgwAdminRunRemote(cluster, client, "metadata", "sync", "status") + if err != nil { + logger.Warnf("REPRGW: failed metadata sync status operation: %v", err) + return RgwMetadataSyncStatus{}, nil + } + + err = json.Unmarshal([]byte(output), &envelope) + if err != nil { + return RgwMetadataSyncStatus{}, fmt.Errorf("cannot unmarshal metadata sync status output: %w", err) + } + + return RgwMetadataSyncStatus{Info: envelope.SyncStatus.Info, Markers: envelope.SyncStatus.Markers}, nil +} + +// GetRgwDataSyncStatus fetches the typed data sync state for one source +// zone (JSON, local sync markers only - no peer comparison). Pass a +// non-empty cluster/client pair to target an imported remote cluster. A +// failing command yields a zero-value status with a nil error, mirroring +// the RBD wrapper contract. +func GetRgwDataSyncStatus(sourceZone string, cluster string, client string) (RgwDataSyncStatus, error) { + envelope := rgwDataSyncEnvelope{} + + output, err := radosgwAdminRunRemote(cluster, client, "data", "sync", "status", "--source-zone", sourceZone) + if err != nil { + logger.Warnf("REPRGW: failed data sync status operation for source(%s): %v", sourceZone, err) + return RgwDataSyncStatus{}, nil + } + + err = json.Unmarshal([]byte(output), &envelope) + if err != nil { + return RgwDataSyncStatus{}, fmt.Errorf("cannot unmarshal data sync status output: %w", err) + } + + return RgwDataSyncStatus{Info: envelope.SyncStatus.Info, Markers: envelope.SyncStatus.Markers}, nil +} + +// RgwLogShard is one shard entry of `mdlog status` or `datalog status` +// output: the log head position on the cluster that owns the log. The +// array index is the shard id. +type RgwLogShard struct { + Marker string `json:"marker"` + LastUpdate string `json:"last_update"` +} + +// GetRgwMdlogStatus fetches the metadata log head markers, one entry per +// shard. Run it against the metadata master (via the cluster/client pair) +// when computing a secondary's catch-up verdict. A failing command yields +// nil with a nil error +func GetRgwMdlogStatus(cluster string, client string) ([]RgwLogShard, error) { + shards := []RgwLogShard{} + + output, err := radosgwAdminRunRemote(cluster, client, "mdlog", "status") + if err != nil { + logger.Warnf("REPRGW: failed mdlog status operation: %v", err) + return nil, nil + } + + err = json.Unmarshal([]byte(output), &shards) + if err != nil { + return nil, fmt.Errorf("cannot unmarshal mdlog status output: %w", err) + } + + return shards, nil +} + +// GetRgwDatalogStatus fetches the data log head markers, one entry per +// shard. Run it against the source zone's cluster (via the cluster/client +// pair) when computing the catch-up verdict for sync from that source. A +// failing command yields nil with a nil error +func GetRgwDatalogStatus(cluster string, client string) ([]RgwLogShard, error) { + shards := []RgwLogShard{} + + output, err := radosgwAdminRunRemote(cluster, client, "datalog", "status") + if err != nil { + logger.Warnf("REPRGW: failed datalog status operation: %v", err) + return nil, nil + } + + err = json.Unmarshal([]byte(output), &shards) + if err != nil { + return nil, fmt.Errorf("cannot unmarshal datalog status output: %w", err) + } + + return shards, nil +} + +// RgwSyncVerdict is the deterministically computed catch-up verdict for one +// sync relationship: local sync markers compared against the peer's log +// heads, using the same per-shard rule radosgw-admin's own `sync status` +// applies (a shard still in full sync, or an incremental shard whose peer +// head is past the local marker, counts as behind). It omits upstream's +// entry-listing prune step, so a shard whose peer log was trimmed may +// transiently over-report as behind. +type RgwSyncVerdict struct { + CaughtUp bool + BehindShards []int + FullSyncShards int + PeriodMismatch bool +} + +// ComputeRgwMetadataSyncVerdict compares a secondary's metadata sync +// markers with the master's mdlog heads. currentPeriod is the realm's +// current period id; a secondary syncing an older period is reported as +// PeriodMismatch without a per-shard comparison, as upstream does. +func ComputeRgwMetadataSyncVerdict(local RgwMetadataSyncStatus, masterLog []RgwLogShard, currentPeriod string) RgwSyncVerdict { + verdict := RgwSyncVerdict{} + + if local.Info.Period != "" && currentPeriod != "" && local.Info.Period != currentPeriod { + verdict.PeriodMismatch = true + return verdict + } + + for _, shard := range local.Markers { + if shard.Val.State != 1 { + verdict.FullSyncShards++ + continue + } + if shard.Key < len(masterLog) && masterLog[shard.Key].Marker > shard.Val.Marker { + verdict.BehindShards = append(verdict.BehindShards, shard.Key) + } + } + + verdict.CaughtUp = len(verdict.BehindShards) == 0 && verdict.FullSyncShards == 0 + return verdict +} + +// ComputeRgwDataSyncVerdict compares local data sync markers for one source +// zone with that source's datalog heads. +func ComputeRgwDataSyncVerdict(local RgwDataSyncStatus, sourceLog []RgwLogShard) RgwSyncVerdict { + verdict := RgwSyncVerdict{} + + for _, shard := range local.Markers { + if shard.Val.Status != "incremental-sync" { + verdict.FullSyncShards++ + continue + } + if shard.Key < len(sourceLog) && sourceLog[shard.Key].Marker > shard.Val.Marker { + verdict.BehindShards = append(verdict.BehindShards, shard.Key) + } + } + + verdict.CaughtUp = len(verdict.BehindShards) == 0 && verdict.FullSyncShards == 0 + return verdict +} diff --git a/microceph/ceph/rgw_multisite_test.go b/microceph/ceph/rgw_multisite_test.go new file mode 100644 index 00000000..ca956e5f --- /dev/null +++ b/microceph/ceph/rgw_multisite_test.go @@ -0,0 +1,253 @@ +package ceph + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microceph/microceph/mocks" + "github.com/canonical/microceph/microceph/tests" +) + +type RgwMultisiteSuite struct { + tests.BaseSuite +} + +func TestRgwMultisite(t *testing.T) { + suite.Run(t, new(RgwMultisiteSuite)) +} + +func (s *RgwMultisiteSuite) SetupTest() { + s.BaseSuite.SetupTest() +} + +func (s *RgwMultisiteSuite) TestGetRgwRealm() { + r := mocks.NewRunner(s.T()) + + output, _ := os.ReadFile("./test_assets/rgw_realm_get.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "realm", "get"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + realm, err := GetRgwRealm("", "") + assert.NoError(s.T(), err) + assert.Equal(s.T(), "microceph", realm.Name) + assert.Equal(s.T(), "cf90947b-b444-488d-abd3-779c3c6062d7", realm.ID) + assert.Equal(s.T(), "9b9a5a4f-ecb4-42a1-b2ff-b31fc0ef5b1b", realm.CurrentPeriod) + assert.Equal(s.T(), 2, realm.Epoch) +} + +func (s *RgwMultisiteSuite) TestGetRgwRealmRemote() { + r := mocks.NewRunner(s.T()) + + output, _ := os.ReadFile("./test_assets/rgw_realm_get.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "realm", "get", "--cluster", "siteb", "--id", "sitea"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + realm, err := GetRgwRealm("siteb", "sitea") + assert.NoError(s.T(), err) + assert.Equal(s.T(), "microceph", realm.Name) +} + +func (s *RgwMultisiteSuite) TestGetRgwRealmUnconfigured() { + r := mocks.NewRunner(s.T()) + + // A realm-less gateway fails realm get; the wrapper swallows the exec + // error into a zero-value realm (the RBD wrapper contract). + r.On("RunCommand", []interface{}{ + "radosgw-admin", "realm", "get"}...).Return("", fmt.Errorf("failed to load realm: (2) No such file or directory")).Once() + common.ProcessExec = r + + realm, err := GetRgwRealm("", "") + assert.NoError(s.T(), err) + assert.Empty(s.T(), realm.ID) + assert.Empty(s.T(), realm.Name) +} + +func (s *RgwMultisiteSuite) TestGetRgwZoneGroup() { + r := mocks.NewRunner(s.T()) + + output, _ := os.ReadFile("./test_assets/rgw_zonegroup_get.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "zonegroup", "get"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + zonegroup, err := GetRgwZoneGroup("", "") + assert.NoError(s.T(), err) + assert.Equal(s.T(), "microceph", zonegroup.Name) + assert.True(s.T(), zonegroup.IsMaster) + assert.Equal(s.T(), "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", zonegroup.MasterZone) + assert.Len(s.T(), zonegroup.Zones, 2) + + names := []string{zonegroup.Zones[0].Name, zonegroup.Zones[1].Name} + assert.Contains(s.T(), names, "sitea") + assert.Contains(s.T(), names, "siteb") + assert.NotEmpty(s.T(), zonegroup.Zones[0].Endpoints) +} + +func (s *RgwMultisiteSuite) TestGetRgwZone() { + r := mocks.NewRunner(s.T()) + + output, _ := os.ReadFile("./test_assets/rgw_zone_get.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "zone", "get"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + zone, err := GetRgwZone("", "") + assert.NoError(s.T(), err) + assert.Equal(s.T(), "sitea", zone.Name) + assert.Equal(s.T(), "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", zone.ID) + assert.NotEmpty(s.T(), zone.SystemKey.AccessKey) + assert.NotEmpty(s.T(), zone.SystemKey.SecretKey) +} + +func (s *RgwMultisiteSuite) TestGetRgwMetadataSyncStatusSecondary() { + r := mocks.NewRunner(s.T()) + + output, _ := os.ReadFile("./test_assets/rgw_metadata_sync_status_secondary.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "metadata", "sync", "status", "--cluster", "siteb", "--id", "sitea"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + status, err := GetRgwMetadataSyncStatus("siteb", "sitea") + assert.NoError(s.T(), err) + assert.Equal(s.T(), "sync", status.Info.Status) + assert.Equal(s.T(), 64, status.Info.NumShards) + assert.NotEmpty(s.T(), status.Info.Period) + assert.Equal(s.T(), 2, status.Info.RealmEpoch) + assert.NotEmpty(s.T(), status.Markers) + assert.Equal(s.T(), 1, status.Markers[0].Val.State) // incremental +} + +func (s *RgwMultisiteSuite) TestGetRgwMetadataSyncStatusMaster() { + r := mocks.NewRunner(s.T()) + + // The metadata master runs no metadata sync: status "init", no shards. + output, _ := os.ReadFile("./test_assets/rgw_metadata_sync_status_master.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "metadata", "sync", "status"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + status, err := GetRgwMetadataSyncStatus("", "") + assert.NoError(s.T(), err) + assert.Equal(s.T(), "init", status.Info.Status) + assert.Equal(s.T(), 0, status.Info.NumShards) + assert.Empty(s.T(), status.Markers) +} + +func (s *RgwMultisiteSuite) TestGetRgwDataSyncStatus() { + r := mocks.NewRunner(s.T()) + + output, _ := os.ReadFile("./test_assets/rgw_data_sync_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "data", "sync", "status", "--source-zone", "sitea", + "--cluster", "siteb", "--id", "sitea"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + status, err := GetRgwDataSyncStatus("sitea", "siteb", "sitea") + assert.NoError(s.T(), err) + assert.Equal(s.T(), "sync", status.Info.Status) + assert.Equal(s.T(), 128, status.Info.NumShards) + assert.NotEmpty(s.T(), status.Markers) + assert.Equal(s.T(), "incremental-sync", status.Markers[0].Val.Status) +} + +func (s *RgwMultisiteSuite) TestGetRgwMdlogStatus() { + r := mocks.NewRunner(s.T()) + + output, _ := os.ReadFile("./test_assets/rgw_mdlog_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "mdlog", "status"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + shards, err := GetRgwMdlogStatus("", "") + assert.NoError(s.T(), err) + assert.Len(s.T(), shards, 4) + assert.Empty(s.T(), shards[0].Marker) + assert.NotEmpty(s.T(), shards[2].Marker) // an active shard's head +} + +func (s *RgwMultisiteSuite) TestGetRgwDatalogStatusRemote() { + r := mocks.NewRunner(s.T()) + + output, _ := os.ReadFile("./test_assets/rgw_datalog_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "datalog", "status", "--cluster", "siteb", "--id", "sitea"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + shards, err := GetRgwDatalogStatus("siteb", "sitea") + assert.NoError(s.T(), err) + assert.Len(s.T(), shards, 3) + assert.NotEmpty(s.T(), shards[2].Marker) +} + +func (s *RgwMultisiteSuite) TestComputeRgwMetadataSyncVerdictCaughtUp() { + local := RgwMetadataSyncStatus{ + Info: RgwSyncInfo{Status: "sync", NumShards: 3, Period: "p1"}, + Markers: []RgwMetadataSyncShard{ + {Key: 0, Val: RgwMetadataSyncMarker{State: 1, Marker: ""}}, + {Key: 1, Val: RgwMetadataSyncMarker{State: 1, Marker: "1_100_5.1"}}, + {Key: 2, Val: RgwMetadataSyncMarker{State: 1, Marker: "1_200_7.1"}}, + }, + } + masterLog := []RgwLogShard{{Marker: ""}, {Marker: "1_100_5.1"}, {Marker: "1_200_7.1"}} + + verdict := ComputeRgwMetadataSyncVerdict(local, masterLog, "p1") + assert.True(s.T(), verdict.CaughtUp) + assert.Empty(s.T(), verdict.BehindShards) + assert.Zero(s.T(), verdict.FullSyncShards) + assert.False(s.T(), verdict.PeriodMismatch) +} + +func (s *RgwMultisiteSuite) TestComputeRgwMetadataSyncVerdictBehind() { + local := RgwMetadataSyncStatus{ + Info: RgwSyncInfo{Status: "sync", NumShards: 3, Period: "p1"}, + Markers: []RgwMetadataSyncShard{ + {Key: 0, Val: RgwMetadataSyncMarker{State: 0, Marker: ""}}, // still full sync + {Key: 1, Val: RgwMetadataSyncMarker{State: 1, Marker: "1_100_5.1"}}, // behind + {Key: 2, Val: RgwMetadataSyncMarker{State: 1, Marker: "1_200_7.1"}}, // caught up + }, + } + masterLog := []RgwLogShard{{Marker: ""}, {Marker: "1_150_6.1"}, {Marker: "1_200_7.1"}} + + verdict := ComputeRgwMetadataSyncVerdict(local, masterLog, "p1") + assert.False(s.T(), verdict.CaughtUp) + assert.Equal(s.T(), []int{1}, verdict.BehindShards) + assert.Equal(s.T(), 1, verdict.FullSyncShards) +} + +func (s *RgwMultisiteSuite) TestComputeRgwMetadataSyncVerdictPeriodMismatch() { + local := RgwMetadataSyncStatus{ + Info: RgwSyncInfo{Status: "sync", NumShards: 1, Period: "p-old"}, + Markers: []RgwMetadataSyncShard{ + {Key: 0, Val: RgwMetadataSyncMarker{State: 1, Marker: "x"}}, + }, + } + + verdict := ComputeRgwMetadataSyncVerdict(local, nil, "p-new") + assert.False(s.T(), verdict.CaughtUp) + assert.True(s.T(), verdict.PeriodMismatch) + assert.Empty(s.T(), verdict.BehindShards) // comparison skipped, as upstream does +} + +func (s *RgwMultisiteSuite) TestComputeRgwDataSyncVerdict() { + local := RgwDataSyncStatus{ + Info: RgwSyncInfo{Status: "sync", NumShards: 3}, + Markers: []RgwDataSyncShard{ + {Key: 0, Val: RgwDataSyncMarker{Status: "incremental-sync", Marker: "1_50_1.1"}}, + {Key: 1, Val: RgwDataSyncMarker{Status: "full-sync", Marker: ""}}, + {Key: 5, Val: RgwDataSyncMarker{Status: "incremental-sync", Marker: ""}}, // out of log bounds + }, + } + sourceLog := []RgwLogShard{{Marker: "1_60_2.1"}, {Marker: ""}, {Marker: ""}} + + verdict := ComputeRgwDataSyncVerdict(local, sourceLog) + assert.False(s.T(), verdict.CaughtUp) + assert.Equal(s.T(), []int{0}, verdict.BehindShards) // shard 5 out of bounds: not counted + assert.Equal(s.T(), 1, verdict.FullSyncShards) +} diff --git a/microceph/ceph/test_assets/rgw_data_sync_status.json b/microceph/ceph/test_assets/rgw_data_sync_status.json new file mode 100644 index 00000000..4d571a14 --- /dev/null +++ b/microceph/ceph/test_assets/rgw_data_sync_status.json @@ -0,0 +1,49 @@ +{ + "sync_status": { + "info": { + "status": "sync", + "num_shards": 128, + "instance_id": 8413746528885553131 + }, + "markers": [ + { + "key": 0, + "val": { + "status": "incremental-sync", + "marker": "", + "next_step_marker": "", + "total_entries": 0, + "pos": 0, + "timestamp": "0.000000" + } + }, + { + "key": 1, + "val": { + "status": "incremental-sync", + "marker": "", + "next_step_marker": "", + "total_entries": 0, + "pos": 0, + "timestamp": "0.000000" + } + }, + { + "key": 2, + "val": { + "status": "incremental-sync", + "marker": "", + "next_step_marker": "", + "total_entries": 0, + "pos": 0, + "timestamp": "0.000000" + } + } + ] + }, + "full_sync": { + "total": 0, + "complete": 0 + }, + "current_time": "2026-07-23T00:22:24Z" +} diff --git a/microceph/ceph/test_assets/rgw_datalog_status.json b/microceph/ceph/test_assets/rgw_datalog_status.json new file mode 100644 index 00000000..a5af5ffd --- /dev/null +++ b/microceph/ceph/test_assets/rgw_datalog_status.json @@ -0,0 +1,14 @@ +[ + { + "marker": "", + "last_update": "0.000000" + }, + { + "marker": "", + "last_update": "0.000000" + }, + { + "marker": "00000000000000000000:00000000000000000512", + "last_update": "2026-07-22T00:49:04.648105Z" + } +] \ No newline at end of file diff --git a/microceph/ceph/test_assets/rgw_mdlog_status.json b/microceph/ceph/test_assets/rgw_mdlog_status.json new file mode 100644 index 00000000..af19a53d --- /dev/null +++ b/microceph/ceph/test_assets/rgw_mdlog_status.json @@ -0,0 +1,18 @@ +[ + { + "marker": "", + "last_update": "0.000000" + }, + { + "marker": "", + "last_update": "0.000000" + }, + { + "marker": "1_1784681341.806612_607.1", + "last_update": "2026-07-22T00:49:01.806612Z" + }, + { + "marker": "1_1784681399.801225_678.1", + "last_update": "2026-07-22T00:49:59.801225Z" + } +] \ No newline at end of file diff --git a/microceph/ceph/test_assets/rgw_metadata_sync_status_master.json b/microceph/ceph/test_assets/rgw_metadata_sync_status_master.json new file mode 100644 index 00000000..5b27c6c5 --- /dev/null +++ b/microceph/ceph/test_assets/rgw_metadata_sync_status_master.json @@ -0,0 +1,16 @@ +{ + "sync_status": { + "info": { + "status": "init", + "num_shards": 0, + "period": "", + "realm_epoch": 0 + }, + "markers": [] + }, + "full_sync": { + "total": 0, + "complete": 0 + }, + "current_time": "2026-07-23T00:22:24Z" +} diff --git a/microceph/ceph/test_assets/rgw_metadata_sync_status_secondary.json b/microceph/ceph/test_assets/rgw_metadata_sync_status_secondary.json new file mode 100644 index 00000000..c099a041 --- /dev/null +++ b/microceph/ceph/test_assets/rgw_metadata_sync_status_secondary.json @@ -0,0 +1,53 @@ +{ + "sync_status": { + "info": { + "status": "sync", + "num_shards": 64, + "period": "9b9a5a4f-ecb4-42a1-b2ff-b31fc0ef5b1b", + "realm_epoch": 2 + }, + "markers": [ + { + "key": 0, + "val": { + "state": 1, + "marker": "", + "next_step_marker": "", + "total_entries": 1, + "pos": 0, + "timestamp": "0.000000", + "realm_epoch": 2 + } + }, + { + "key": 1, + "val": { + "state": 1, + "marker": "", + "next_step_marker": "", + "total_entries": 0, + "pos": 0, + "timestamp": "0.000000", + "realm_epoch": 2 + } + }, + { + "key": 2, + "val": { + "state": 1, + "marker": "", + "next_step_marker": "", + "total_entries": 0, + "pos": 0, + "timestamp": "0.000000", + "realm_epoch": 2 + } + } + ] + }, + "full_sync": { + "total": 1, + "complete": 1 + }, + "current_time": "2026-07-23T00:22:24Z" +} diff --git a/microceph/ceph/test_assets/rgw_realm_get.json b/microceph/ceph/test_assets/rgw_realm_get.json new file mode 100644 index 00000000..f3cf3468 --- /dev/null +++ b/microceph/ceph/test_assets/rgw_realm_get.json @@ -0,0 +1,6 @@ +{ + "id": "cf90947b-b444-488d-abd3-779c3c6062d7", + "name": "microceph", + "current_period": "9b9a5a4f-ecb4-42a1-b2ff-b31fc0ef5b1b", + "epoch": 2 +} diff --git a/microceph/ceph/test_assets/rgw_zone_get.json b/microceph/ceph/test_assets/rgw_zone_get.json new file mode 100644 index 00000000..8fd70e4c --- /dev/null +++ b/microceph/ceph/test_assets/rgw_zone_get.json @@ -0,0 +1,43 @@ +{ + "id": "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", + "name": "sitea", + "domain_root": "sitea.rgw.meta:root", + "control_pool": "sitea.rgw.control", + "gc_pool": "sitea.rgw.log:gc", + "lc_pool": "sitea.rgw.log:lc", + "log_pool": "sitea.rgw.log", + "intent_log_pool": "sitea.rgw.log:intent", + "usage_log_pool": "sitea.rgw.log:usage", + "roles_pool": "sitea.rgw.meta:roles", + "reshard_pool": "sitea.rgw.log:reshard", + "user_keys_pool": "sitea.rgw.meta:users.keys", + "user_email_pool": "sitea.rgw.meta:users.email", + "user_swift_pool": "sitea.rgw.meta:users.swift", + "user_uid_pool": "sitea.rgw.meta:users.uid", + "otp_pool": "sitea.rgw.otp", + "notif_pool": "sitea.rgw.log:notif", + "topics_pool": "sitea.rgw.meta:topics", + "account_pool": "sitea.rgw.meta:accounts", + "group_pool": "sitea.rgw.meta:groups", + "system_key": { + "access_key": "VDS6K1QMOGKYIJA5WTQD", + "secret_key": "cE3yAkkOisWAwBj7fYBErJ5RedXeeAJX7JXZOERe" + }, + "placement_pools": [ + { + "key": "default-placement", + "val": { + "index_pool": "sitea.rgw.buckets.index", + "storage_classes": { + "STANDARD": { + "data_pool": "sitea.rgw.buckets.data" + } + }, + "data_extra_pool": "sitea.rgw.buckets.non-ec", + "index_type": 0, + "inline_data": true + } + } + ], + "realm_id": "cf90947b-b444-488d-abd3-779c3c6062d7" +} diff --git a/microceph/ceph/test_assets/rgw_zonegroup_get.json b/microceph/ceph/test_assets/rgw_zonegroup_get.json new file mode 100644 index 00000000..bd8c80f4 --- /dev/null +++ b/microceph/ceph/test_assets/rgw_zonegroup_get.json @@ -0,0 +1,116 @@ +{ + "id": "67be86c9-2912-4ce2-835d-9bdf91915363", + "name": "microceph", + "api_name": "microceph", + "is_master": true, + "endpoints": [ + "http://10.85.32.250:80" + ], + "hostnames": [], + "hostnames_s3website": [], + "master_zone": "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", + "zones": [ + { + "id": "58a9f4ec-c0b7-415d-93a5-8eb1c03818ae", + "name": "siteb", + "endpoints": [ + "http://10.85.32.128:80" + ], + "log_meta": false, + "log_data": true, + "bucket_index_max_shards": 11, + "read_only": false, + "tier_type": "", + "sync_from_all": true, + "sync_from": [], + "redirect_zone": "", + "supported_features": [ + "compress-encrypted", + "notification_v2", + "resharding" + ] + }, + { + "id": "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", + "name": "sitea", + "endpoints": [ + "http://10.85.32.250:80" + ], + "log_meta": false, + "log_data": true, + "bucket_index_max_shards": 11, + "read_only": false, + "tier_type": "", + "sync_from_all": true, + "sync_from": [], + "redirect_zone": "", + "supported_features": [ + "compress-encrypted", + "notification_v2", + "resharding" + ] + } + ], + "placement_targets": [ + { + "name": "default-placement", + "tags": [], + "storage_classes": [ + "STANDARD" + ] + } + ], + "default_placement": "default-placement", + "realm_id": "cf90947b-b444-488d-abd3-779c3c6062d7", + "sync_policy": { + "groups": [ + { + "id": "default", + "data_flow": { + "symmetrical": [ + { + "id": "sitea-siteb", + "zones": [ + "58a9f4ec-c0b7-415d-93a5-8eb1c03818ae", + "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7" + ] + } + ] + }, + "pipes": [ + { + "id": "all", + "source": { + "bucket": "*", + "zones": [ + "*" + ] + }, + "dest": { + "bucket": "*", + "zones": [ + "*" + ] + }, + "params": { + "source": { + "filter": { + "tags": [] + } + }, + "dest": {}, + "priority": 0, + "mode": "system", + "user": "" + } + } + ], + "status": "enabled" + } + ] + }, + "enabled_features": [ + "notification_v2", + "resharding" + ] +}