From 1868ddca5144254db24efc9c509cabdd87e9369d Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Fri, 10 Jul 2026 16:42:28 +0530 Subject: [PATCH 01/31] snap: stage samba/ctdb and add smbd service (experiment) Package samba, samba-vfs-modules and ctdb from the noble archive and expose an smbd app (disabled by default) driven by a config under SNAP_DATA/conf/samba. No placement or API integration; this is the packaging half of the native SMB experiment. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- snap/snapcraft.yaml | 32 ++++++++++++++++++++++++++++++++ snapcraft/commands/smbd.start | 19 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100755 snapcraft/commands/smbd.start diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 71728465..3656351c 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -45,6 +45,8 @@ layout: symlink: $SNAP/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/ceph /usr/lib/ganesha: symlink: $SNAP/lib/ganesha + /usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/samba: + symlink: $SNAP/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/samba /usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/rados-classes: symlink: $SNAP/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/rados-classes /etc/ceph: @@ -116,6 +118,18 @@ apps: - network - network-bind - process-control + smbd: + command: commands/smbd.start + daemon: simple + install-mode: disable + after: + - daemon + plugs: + - account-control + - network + - network-bind + - network-control + - process-control osd: command: commands/osd.start reload-command: commands/osd.reload @@ -463,6 +477,24 @@ parts: - lib/*/liburcu-bp.so* - lib/*/libwbclient.so* + samba: + plugin: nil + stage-packages: + - samba + - samba-vfs-modules + - ctdb + organize: + sbin/: bin/ + usr/bin/: bin/ + usr/sbin/: bin/ + usr/lib/: lib/ + usr/libexec/: libexec/ + prime: + - -lib/systemd + - -lib/tmpfiles.d + - -lib/sysusers.d + - -lib/*/avahi + logrotate: plugin: nil stage-packages: diff --git a/snapcraft/commands/smbd.start b/snapcraft/commands/smbd.start new file mode 100755 index 00000000..8cbb2e57 --- /dev/null +++ b/snapcraft/commands/smbd.start @@ -0,0 +1,19 @@ +#!/bin/bash + +. "${SNAP}/commands/common" + +limits + +wait_for_config + +conf="${SNAP_DATA}/conf/samba/smb.conf" + +mkdir -p \ + "${SNAP_DATA}/run/samba" \ + "${SNAP_COMMON}/logs/samba" \ + "${SNAP_COMMON}/data/samba/private" \ + "${SNAP_COMMON}/data/samba/lock" \ + "${SNAP_COMMON}/data/samba/state" \ + "${SNAP_COMMON}/data/samba/cache" + +exec smbd --foreground --no-process-group --configfile="${conf}" From 26fffa8cd3a7979638e9a162a94b12fe67a0db03 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 16:00:30 +0530 Subject: [PATCH 02/31] api: add SMBSpec types for native SMB Add Go types mirroring the upstream mgr/smb SMBSpec service spec JSON (service_spec.py): SMBSpec, SMBPlacementSpec, SMBPublicAddrSpec and the SMBDestination union decoder, plus the upstream cluster id regex from mgr/smb validation.py. Unknown fields are tolerated so newer mgr serializations do not break decoding. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/api/types/smb.go | 92 +++++++++++++++++++++++++++++++++ microceph/api/types/smb_test.go | 87 +++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 microceph/api/types/smb.go create mode 100644 microceph/api/types/smb_test.go diff --git a/microceph/api/types/smb.go b/microceph/api/types/smb.go new file mode 100644 index 00000000..ed541747 --- /dev/null +++ b/microceph/api/types/smb.go @@ -0,0 +1,92 @@ +package types + +import ( + "encoding/json" + "fmt" + "regexp" +) + +// SMBClusterIDRegex mirrors the upstream mgr/smb ID validation +// (src/pybind/mgr/smb/validation.py _name_re): 1-18 characters, +// alphanumeric with inner hyphens. +var SMBClusterIDRegex = regexp.MustCompile(`^[a-zA-Z0-9]($|[a-zA-Z0-9-]{0,16}[a-zA-Z0-9]$)`) + +// SMBSpec mirrors the JSON serialization of the upstream mgr/smb service +// spec (ceph src/python-common service_spec.py, SMBSpec). Field names match +// the python spec exactly; unknown fields are tolerated so newer mgr +// versions do not break decoding. Phase-1-unsupported fields are rejected +// at validation time from the raw payload, not modeled here. +type SMBSpec struct { + ServiceType string `json:"service_type" yaml:"service_type"` + ServiceID string `json:"service_id" yaml:"service_id"` + Placement SMBPlacementSpec `json:"placement" yaml:"placement"` + ClusterID string `json:"cluster_id" yaml:"cluster_id"` + Features []string `json:"features" yaml:"features"` + ConfigURI string `json:"config_uri" yaml:"config_uri"` + UserSources []string `json:"user_sources" yaml:"user_sources"` + ClusterMetaURI string `json:"cluster_meta_uri" yaml:"cluster_meta_uri"` + ClusterLockURI string `json:"cluster_lock_uri" yaml:"cluster_lock_uri"` + ClusterPublicAddrs []SMBPublicAddrSpec `json:"cluster_public_addrs" yaml:"cluster_public_addrs"` + IncludeCephUsers []string `json:"include_ceph_users" yaml:"include_ceph_users"` +} + +// SMBPlacementSpec is the subset of the ceph PlacementSpec serialization +// honored in Phase 1. Hosts entries are plain hostnames. CountPerHost and +// HostPattern are parsed only so validation can reject them explicitly. +type SMBPlacementSpec struct { + Hosts []string `json:"hosts" yaml:"hosts"` + Count int `json:"count" yaml:"count"` + Label string `json:"label" yaml:"label"` + CountPerHost int `json:"count_per_host" yaml:"count_per_host"` + HostPattern json.RawMessage `json:"host_pattern" yaml:"host_pattern"` +} + +// SMBService identifies an SMB cluster by its cluster id. +type SMBService struct { + ClusterID string `json:"cluster_id" yaml:"cluster_id"` +} + +// SMBServiceStatus describes one SMB cluster: its stored spec and the +// nodes it is currently placed on. +type SMBServiceStatus struct { + ClusterID string `json:"cluster_id" yaml:"cluster_id"` + Spec json.RawMessage `json:"spec" yaml:"spec"` + PlacedOn []string `json:"placed_on" yaml:"placed_on"` +} + +// SMBPublicAddrSpec mirrors SMBClusterPublicIPSpec: a CTDB public address +// with optional destination networks. +type SMBPublicAddrSpec struct { + Address string `json:"address" yaml:"address"` + Destination SMBDestination `json:"destination" yaml:"destination"` +} + +// SMBDestination decodes the python Union[str, List[str], None] shape of +// SMBClusterPublicIPSpec.destination into a flat string slice. +type SMBDestination []string + +// UnmarshalJSON accepts null, a single string, or a list of strings. +func (d *SMBDestination) UnmarshalJSON(data []byte) error { + // json.Unmarshal(null, &string) is a no-op success, so null must be + // handled before the single-string attempt. + if string(data) == "null" { + *d = nil + return nil + } + + var single string + err := json.Unmarshal(data, &single) + if err == nil { + *d = SMBDestination{single} + return nil + } + + var many []string + err = json.Unmarshal(data, &many) + if err == nil { + *d = many + return nil + } + + return fmt.Errorf("destination must be a string, list of strings, or null: %s", string(data)) +} diff --git a/microceph/api/types/smb_test.go b/microceph/api/types/smb_test.go new file mode 100644 index 00000000..24b13d34 --- /dev/null +++ b/microceph/api/types/smb_test.go @@ -0,0 +1,87 @@ +package types + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/suite" +) + +// sampleSMBSpec is the reference SMBSpec JSON emitted by mgr/smb +// (ceph service_spec.py serialization). +const sampleSMBSpec = `{ + "service_type": "smb", + "service_id": "dev", + "placement": {"hosts": ["smbdev-1", "smbdev-2", "smbdev-3"]}, + "cluster_id": "dev", + "features": [], + "config_uri": "rados://.smb/dev/scc.dev.json", + "user_sources": ["rados://.smb/dev/users.dev.json"], + "cluster_meta_uri": "rados://.smb/dev/cluster.meta.json", + "cluster_lock_uri": "rados://.smb/dev/cluster.meta.lock", + "cluster_public_addrs": [ + {"address": "10.105.154.245/24", "destination": null} + ] +}` + +type SMBSpecSuite struct { + suite.Suite +} + +func TestSMBSpecSuite(t *testing.T) { + suite.Run(t, new(SMBSpecSuite)) +} + +func (s *SMBSpecSuite) TestUnmarshalSampleSpec() { + var spec SMBSpec + err := json.Unmarshal([]byte(sampleSMBSpec), &spec) + s.NoError(err) + + s.Equal("smb", spec.ServiceType) + s.Equal("dev", spec.ServiceID) + s.Equal("dev", spec.ClusterID) + s.Equal([]string{"smbdev-1", "smbdev-2", "smbdev-3"}, spec.Placement.Hosts) + s.Empty(spec.Features) + s.Equal("rados://.smb/dev/scc.dev.json", spec.ConfigURI) + s.Equal([]string{"rados://.smb/dev/users.dev.json"}, spec.UserSources) + s.Equal("rados://.smb/dev/cluster.meta.json", spec.ClusterMetaURI) + s.Equal("rados://.smb/dev/cluster.meta.lock", spec.ClusterLockURI) + s.Require().Len(spec.ClusterPublicAddrs, 1) + s.Equal("10.105.154.245/24", spec.ClusterPublicAddrs[0].Address) + s.Empty(spec.ClusterPublicAddrs[0].Destination) +} + +func (s *SMBSpecSuite) TestUnmarshalPlacementCountAndLabel() { + var spec SMBSpec + err := json.Unmarshal([]byte(`{"placement": {"count": 3, "label": "smb"}}`), &spec) + s.NoError(err) + s.Equal(3, spec.Placement.Count) + s.Equal("smb", spec.Placement.Label) +} + +func (s *SMBSpecSuite) TestUnmarshalDestinationString() { + var addr SMBPublicAddrSpec + err := json.Unmarshal([]byte(`{"address": "10.0.0.1/24", "destination": "10.0.0.0/24"}`), &addr) + s.NoError(err) + s.Equal(SMBDestination{"10.0.0.0/24"}, addr.Destination) +} + +func (s *SMBSpecSuite) TestUnmarshalDestinationList() { + var addr SMBPublicAddrSpec + err := json.Unmarshal([]byte(`{"address": "10.0.0.1/24", "destination": ["10.0.0.0/24", "10.1.0.0/24"]}`), &addr) + s.NoError(err) + s.Equal(SMBDestination{"10.0.0.0/24", "10.1.0.0/24"}, addr.Destination) +} + +func (s *SMBSpecSuite) TestUnmarshalDestinationInvalid() { + var addr SMBPublicAddrSpec + err := json.Unmarshal([]byte(`{"address": "10.0.0.1/24", "destination": 42}`), &addr) + s.Error(err) +} + +func (s *SMBSpecSuite) TestUnknownFieldsTolerated() { + var spec SMBSpec + err := json.Unmarshal([]byte(`{"cluster_id": "dev", "some_future_field": {"x": 1}}`), &spec) + s.NoError(err) + s.Equal("dev", spec.ClusterID) +} From 20c7b71afabf693ada62250884cef4239fefb900 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 16:00:45 +0530 Subject: [PATCH 03/31] daemon: add smb service placement Add SMBServicePlacement implementing PlacementIntf: SMBSpec payload validation (upstream cluster id regex, feature gate accepting only 'clustered', rejection of Phase-1-unsupported fields), hospitality checks (port 445 free, smb clusters node-disjoint), and DB recording via GroupedServicesQuery with the spec JSON stored verbatim as group config. ServiceInit and PostPlacementCheck are documented no-ops until the M3 lifecycle work. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/service_placement_smb.go | 140 +++++++++++++ microceph/ceph/service_placement_smb_test.go | 200 +++++++++++++++++++ microceph/ceph/services_placement.go | 1 + microceph/database/grouped_service.go | 5 + 4 files changed, 346 insertions(+) create mode 100644 microceph/ceph/service_placement_smb.go create mode 100644 microceph/ceph/service_placement_smb_test.go diff --git a/microceph/ceph/service_placement_smb.go b/microceph/ceph/service_placement_smb.go new file mode 100644 index 00000000..145572bb --- /dev/null +++ b/microceph/ceph/service_placement_smb.go @@ -0,0 +1,140 @@ +package ceph + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/interfaces" +) + +// smbPort is the well-known TCP port smbd binds on every placed node. +const smbPort = 445 + +// isAddressAvailableFunc is injectable so tests can exercise the +// hospitality check without binding the fixed SMB port. +var isAddressAvailableFunc = isAddressAvailable + +// SMBServicePlacement implements PlacementIntf for CTDB-clustered Samba. +// The payload is the mgr/smb SMBSpec JSON, stored verbatim as the service +// group config. +type SMBServicePlacement struct { + Spec types.SMBSpec + rawSpec string +} + +// PopulateParams parses and validates the SMBSpec payload. +func (smb *SMBServicePlacement) PopulateParams(s interfaces.StateInterface, payload string) error { + err := json.Unmarshal([]byte(payload), &smb.Spec) + if err != nil { + return err + } + + err = checkSMBUnsupportedFields([]byte(payload)) + if err != nil { + return err + } + + if !types.SMBClusterIDRegex.MatchString(smb.Spec.ClusterID) { + return fmt.Errorf("cluster_id '%s' is not a valid ID (regex: '%s')", + smb.Spec.ClusterID, types.SMBClusterIDRegex.String()) + } + + for _, feature := range smb.Spec.Features { + switch feature { + case "clustered": + // CTDB clustering is the Phase 1 deployment model. + case "domain": + return fmt.Errorf("features: 'domain' (AD membership) is not supported in Phase 1") + default: + return fmt.Errorf("features: '%s' is not supported", feature) + } + } + + smb.rawSpec = payload + return nil +} + +// checkSMBUnsupportedFields rejects SMBSpec fields Phase 1 does not +// implement, so a spec requesting them fails loudly instead of being +// silently ignored. Unset serializations (null, [], {}) are tolerated. +func checkSMBUnsupportedFields(payload []byte) error { + var raw map[string]json.RawMessage + err := json.Unmarshal(payload, &raw) + if err != nil { + return err + } + + unsupported := func(key string) bool { + switch key { + case "bind_addrs", "custom_ports", "custom_dns": + return true + } + return strings.HasPrefix(key, "remote_control_") + } + + for key, value := range raw { + if !unsupported(key) { + continue + } + switch string(value) { + case "null", "[]", "{}": + continue + } + return fmt.Errorf("field '%s' is not supported in Phase 1", key) + } + + return nil +} + +// HospitalityCheck verifies the SMB port is free and the node is not +// already part of an smb cluster (smb clusters are node-disjoint: one +// ctdb/smbd instance per node). +func (smb *SMBServicePlacement) HospitalityCheck(s interfaces.StateInterface) error { + address := fmt.Sprintf("0.0.0.0:%d", smbPort) + available, err := isAddressAvailableFunc(address) + if err != nil { + return fmt.Errorf("error encountered during address availability check: %w", err) + } else if !available { + return fmt.Errorf("address '%s' is currently in use.", address) + } + + services, err := database.GroupedServicesQuery.GetGroupedServicesOnHost(context.Background(), s) + if err != nil { + return fmt.Errorf("failed to fetch smb group membership: %w", err) + } + + for _, service := range services { + if service.Service != "smb" { + continue + } + if service.GroupID == smb.Spec.ClusterID { + return fmt.Errorf("node is already a member of smb cluster '%s'", service.GroupID) + } + return fmt.Errorf("node is already a member of smb cluster '%s'; smb clusters must be node-disjoint", service.GroupID) + } + + return nil +} + +// ServiceInit is a no-op in Phase 1 milestone M2: config rendering and +// ctdbd lifecycle land with M3. +func (smb *SMBServicePlacement) ServiceInit(ctx context.Context, s interfaces.StateInterface) error { + return nil +} + +// PostPlacementCheck is a no-op until ServiceInit starts services (M3), +// after which it will verify ctdbd health. +func (smb *SMBServicePlacement) PostPlacementCheck(s interfaces.StateInterface) error { + return nil +} + +// DbUpdate records the group membership, storing the SMBSpec JSON verbatim +// as the group config (single source of truth; no parallel schema). +func (smb *SMBServicePlacement) DbUpdate(ctx context.Context, s interfaces.StateInterface) error { + return database.GroupedServicesQuery.AddNew(ctx, s, "smb", smb.Spec.ClusterID, + json.RawMessage(smb.rawSpec), database.SMBServiceInfo{}) +} diff --git a/microceph/ceph/service_placement_smb_test.go b/microceph/ceph/service_placement_smb_test.go new file mode 100644 index 00000000..ee8883df --- /dev/null +++ b/microceph/ceph/service_placement_smb_test.go @@ -0,0 +1,200 @@ +package ceph + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/mocks" + "github.com/canonical/microceph/microceph/tests" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// validSMBPayload is the reference mgr/smb SMBSpec JSON from the design. +const validSMBPayload = `{ + "service_type": "smb", + "service_id": "dev", + "placement": {"hosts": ["smbdev-1", "smbdev-2", "smbdev-3"]}, + "cluster_id": "dev", + "features": ["clustered"], + "config_uri": "rados://.smb/dev/scc.dev.json", + "user_sources": ["rados://.smb/dev/users.dev.json"], + "cluster_meta_uri": "rados://.smb/dev/cluster.meta.json", + "cluster_lock_uri": "rados://.smb/dev/cluster.meta.lock", + "cluster_public_addrs": [ + {"address": "10.105.154.245/24", "destination": null} + ] +}` + +type servicePlacementSMBSuite struct { + tests.BaseSuite + TestStateInterface *mocks.StateInterface +} + +func TestServicesPlacementSMB(t *testing.T) { + suite.Run(t, new(servicePlacementSMBSuite)) +} + +// Set up test suite +func (s *servicePlacementSMBSuite) SetupTest() { + s.BaseSuite.SetupTest() + s.TestStateInterface = mocks.NewStateInterface(s.T()) +} + +// populated returns an SMBServicePlacement loaded from a payload that must +// parse successfully. +func (s *servicePlacementSMBSuite) populated(payload string) *SMBServicePlacement { + smb := &SMBServicePlacement{} + err := smb.PopulateParams(s.TestStateInterface, payload) + assert.NoError(s.T(), err) + return smb +} + +func (s *servicePlacementSMBSuite) TestHandlerWiring() { + payload := types.EnableService{ + Name: "smb", + Wait: true, + Payload: `{"cluster_id":""}`, + } + + // Proves the "smb" placement table entry exists and PopulateParams + // errors propagate through the handler. + err := ServicePlacementHandler(context.Background(), s.TestStateInterface, payload) + assert.ErrorContains(s.T(), err, "not a valid ID") +} + +func (s *servicePlacementSMBSuite) TestInvalidClusterID() { + smb := &SMBServicePlacement{} + + for _, id := range []string{"", "-foo", "foo-", "foo_bar", strings.Repeat("a", 19)} { + err := smb.PopulateParams(s.TestStateInterface, fmt.Sprintf(`{"cluster_id":"%s"}`, id)) + assert.ErrorContains(s.T(), err, "not a valid ID", "cluster_id: %q", id) + } + + // Boundary: 18 alphanumeric chars is the upstream maximum. + err := smb.PopulateParams(s.TestStateInterface, fmt.Sprintf(`{"cluster_id":"%s"}`, strings.Repeat("a", 18))) + assert.NoError(s.T(), err) +} + +func (s *servicePlacementSMBSuite) TestDomainFeatureRejected() { + smb := &SMBServicePlacement{} + err := smb.PopulateParams(s.TestStateInterface, `{"cluster_id":"dev","features":["domain"]}`) + assert.ErrorContains(s.T(), err, "domain") +} + +func (s *servicePlacementSMBSuite) TestUnknownFeatureRejected() { + smb := &SMBServicePlacement{} + err := smb.PopulateParams(s.TestStateInterface, `{"cluster_id":"dev","features":["wormholes"]}`) + assert.ErrorContains(s.T(), err, "wormholes") +} + +func (s *servicePlacementSMBSuite) TestUnsupportedFieldsRejected() { + smb := &SMBServicePlacement{} + + for _, field := range []string{"bind_addrs", "custom_ports", "custom_dns", "remote_control_uri"} { + payload := fmt.Sprintf(`{"cluster_id":"dev","%s":["x"]}`, field) + err := smb.PopulateParams(s.TestStateInterface, payload) + assert.ErrorContains(s.T(), err, field) + } +} + +func (s *servicePlacementSMBSuite) TestUnsetUnsupportedFieldsTolerated() { + // mgr serialization may emit unset fields as null or empty; those must + // not be rejected. + smb := &SMBServicePlacement{} + err := smb.PopulateParams(s.TestStateInterface, + `{"cluster_id":"dev","bind_addrs":null,"custom_dns":[],"custom_ports":{}}`) + assert.NoError(s.T(), err) +} + +func (s *servicePlacementSMBSuite) TestValidSpecAccepted() { + smb := s.populated(validSMBPayload) + assert.Equal(s.T(), "dev", smb.Spec.ClusterID) + assert.Equal(s.T(), []string{"smbdev-1", "smbdev-2", "smbdev-3"}, smb.Spec.Placement.Hosts) +} + +// withSMBGroupMembership patches the grouped-services query to report the +// given rows for this host, and the port check to report available. +func (s *servicePlacementSMBSuite) withHospitalityEnv(rows []database.GroupedService, portFree bool) func() { + db := mocks.NewGroupedServiceQueryIntf(s.T()) + if portFree { + db.On("GetGroupedServicesOnHost", context.Background(), s.TestStateInterface).Return(rows, nil).Maybe() + } + + originalDB := database.GroupedServicesQuery + originalPortCheck := isAddressAvailableFunc + database.GroupedServicesQuery = db + isAddressAvailableFunc = func(address string) (bool, error) { return portFree, nil } + + return func() { + database.GroupedServicesQuery = originalDB + isAddressAvailableFunc = originalPortCheck + } +} + +func (s *servicePlacementSMBSuite) TestHospitalityFreeNode() { + restore := s.withHospitalityEnv([]database.GroupedService{}, true) + defer restore() + + smb := s.populated(validSMBPayload) + assert.NoError(s.T(), smb.HospitalityCheck(s.TestStateInterface)) +} + +func (s *servicePlacementSMBSuite) TestHospitalityPortBusy() { + restore := s.withHospitalityEnv(nil, false) + defer restore() + + smb := s.populated(validSMBPayload) + assert.ErrorContains(s.T(), smb.HospitalityCheck(s.TestStateInterface), "445") +} + +func (s *servicePlacementSMBSuite) TestHospitalityNodeInOtherGroup() { + restore := s.withHospitalityEnv([]database.GroupedService{ + {Member: "smbdev-1", Service: "smb", GroupID: "other"}, + }, true) + defer restore() + + smb := s.populated(validSMBPayload) + assert.ErrorContains(s.T(), smb.HospitalityCheck(s.TestStateInterface), "node-disjoint") +} + +func (s *servicePlacementSMBSuite) TestHospitalityReApplySameGroup() { + restore := s.withHospitalityEnv([]database.GroupedService{ + {Member: "smbdev-1", Service: "smb", GroupID: "dev"}, + }, true) + defer restore() + + smb := s.populated(validSMBPayload) + assert.ErrorContains(s.T(), smb.HospitalityCheck(s.TestStateInterface), "already a member of smb cluster 'dev'") +} + +func (s *servicePlacementSMBSuite) TestHospitalityIgnoresOtherServices() { + restore := s.withHospitalityEnv([]database.GroupedService{ + {Member: "smbdev-1", Service: "nfs", GroupID: "dev"}, + }, true) + defer restore() + + smb := s.populated(validSMBPayload) + assert.NoError(s.T(), smb.HospitalityCheck(s.TestStateInterface)) +} + +func (s *servicePlacementSMBSuite) TestDBUpdate() { + smb := s.populated(validSMBPayload) + + db := mocks.NewGroupedServiceQueryIntf(s.T()) + ctx := context.Background() + db.On("AddNew", []interface{}{ctx, s.TestStateInterface, "smb", "dev", + json.RawMessage(validSMBPayload), database.SMBServiceInfo{}}...).Return(nil).Once() + + originalDB := database.GroupedServicesQuery + defer func() { database.GroupedServicesQuery = originalDB }() + database.GroupedServicesQuery = db + + assert.NoError(s.T(), smb.DbUpdate(ctx, s.TestStateInterface)) +} diff --git a/microceph/ceph/services_placement.go b/microceph/ceph/services_placement.go index b7fb4e64..f619085b 100644 --- a/microceph/ceph/services_placement.go +++ b/microceph/ceph/services_placement.go @@ -30,6 +30,7 @@ func GetServicePlacementTable() map[string](PlacementIntf) { "mgr": &GenericServicePlacement{"mgr"}, "mds": &GenericServicePlacement{"mds"}, "nfs": &NFSServicePlacement{}, + "smb": &SMBServicePlacement{}, "rgw": &RgwServicePlacement{}, "rbd-mirror": &ClientServicePlacement{"rbd-mirror"}, "cephfs-mirror": &ClientServicePlacement{"cephfs-mirror"}, diff --git a/microceph/database/grouped_service.go b/microceph/database/grouped_service.go index 81a91b71..9ca8d86a 100644 --- a/microceph/database/grouped_service.go +++ b/microceph/database/grouped_service.go @@ -37,6 +37,11 @@ type GroupedServiceFilter struct { Member *string } +// SMBServiceInfo is a struct containing per-node GroupedService information +// for SMB. Empty in Phase 1: the SMBSpec stored as group config carries all +// state, and nothing is node-specific yet. +type SMBServiceInfo struct{} + // NFSServiceInfo is a struct containing GroupedService information. type NFSServiceInfo struct { BindAddress string `json:"bind_address"` From 1df92767a1fc9782164cdf9734586e338052845e Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 16:00:46 +0530 Subject: [PATCH 04/31] daemon: add smb cluster endpoints and orchestration Add the cluster-scoped /1.0/services/smb endpoint (PUT SMBSpec apply, DELETE cluster removal, GET status) and the node-scoped /1.0/services/smb/node endpoint used by the per-node fan-out. ApplySMB resolves spec placement (hosts/count) against cluster members, diffs it with recorded membership and converges idempotently; specs are canonicalized (json.Compact) so stored group configs compare stably. Extend GroupedServicesQuery with group member/config accessors. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/api/servers.go | 2 + microceph/api/services_smb.go | 105 +++++++ microceph/ceph/smb.go | 284 +++++++++++++++++++ microceph/ceph/smb_test.go | 279 ++++++++++++++++++ microceph/client/services.go | 33 +++ microceph/database/grouped_service_extras.go | 75 +++++ microceph/mocks/GroupedServiceQueryIntf.go | 76 +++++ 7 files changed, 854 insertions(+) create mode 100644 microceph/api/services_smb.go create mode 100644 microceph/ceph/smb.go create mode 100644 microceph/ceph/smb_test.go diff --git a/microceph/api/servers.go b/microceph/api/servers.go index 6da50809..d9e9e654 100644 --- a/microceph/api/servers.go +++ b/microceph/api/servers.go @@ -25,6 +25,8 @@ var Servers = map[string]mcTypes.Server{ mgrServiceCmd, monServiceCmd, nfsServiceCmd, + smbServiceCmd, + smbNodeServiceCmd, poolsOpCmd, rgwServiceCmd, rbdMirroServiceCmd, diff --git a/microceph/api/services_smb.go b/microceph/api/services_smb.go new file mode 100644 index 00000000..d380df05 --- /dev/null +++ b/microceph/api/services_smb.go @@ -0,0 +1,105 @@ +package api + +import ( + "encoding/json" + "errors" + "io" + "net/http" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microceph/microceph/interfaces" + "github.com/canonical/microceph/microceph/logger" + mcTypes "github.com/canonical/microcluster/v3/microcluster/types" +) + +// /1.0/services/smb endpoint: cluster-scoped SMBSpec operations, the +// mgr/smb -> microceph-orch contract (apply/remove/status). +var smbServiceCmd = mcTypes.Endpoint{ + Path: "services/smb", + Get: mcTypes.EndpointAction{Handler: cmdSMBServiceGet, ProxyTarget: true}, + Put: mcTypes.EndpointAction{Handler: cmdSMBServicePut, ProxyTarget: true}, + Delete: mcTypes.EndpointAction{Handler: cmdSMBServiceDelete, ProxyTarget: true}, +} + +// /1.0/services/smb/node endpoint: node-scoped enable/disable used by the +// cluster-level fan-out (invoked with UseTarget per placed node). +var smbNodeServiceCmd = mcTypes.Endpoint{ + Path: "services/smb/node", + Put: mcTypes.EndpointAction{Handler: cmdEnableServicePut, ProxyTarget: true}, + Delete: mcTypes.EndpointAction{Handler: cmdSMBNodeDelete, ProxyTarget: true}, +} + +// cmdSMBServiceGet lists every smb cluster with its spec and placement. +func cmdSMBServiceGet(s mcTypes.State, r *http.Request) mcTypes.Response { + statuses, err := ceph.ListSMB(r.Context(), interfaces.CephState{State: s}) + if err != nil { + return mcTypes.InternalError(err) + } + + return mcTypes.SyncResponse(true, statuses) +} + +// cmdSMBServicePut applies an SMBSpec (JSON body) to the cluster. +func cmdSMBServicePut(s mcTypes.State, r *http.Request) mcTypes.Response { + // SMBSpecs are small; the limit only guards against runaway bodies. + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + logger.Errorf("failed reading smb spec body: %v", err) + return mcTypes.InternalError(err) + } + + err = ceph.ApplySMB(r.Context(), interfaces.CephState{State: s}, string(body)) + if err != nil { + logger.Errorf("failed applying smb spec: %v", err) + if errors.Is(err, ceph.ErrInvalidSMBSpec) { + return mcTypes.BadRequest(err) + } + return mcTypes.SmartError(err) + } + + return mcTypes.EmptySyncResponse +} + +// cmdSMBServiceDelete removes an smb cluster from all its member nodes. +func cmdSMBServiceDelete(s mcTypes.State, r *http.Request) mcTypes.Response { + var svc types.SMBService + + err := json.NewDecoder(r.Body).Decode(&svc) + if err != nil { + logger.Errorf("failed decoding smb delete request: %v", err) + return mcTypes.InternalError(err) + } + + if !types.SMBClusterIDRegex.MatchString(svc.ClusterID) { + err := errors.New("expected cluster_id to be valid (regex: '" + types.SMBClusterIDRegex.String() + "')") + return mcTypes.BadRequest(err) + } + + err = ceph.RemoveSMB(r.Context(), interfaces.CephState{State: s}, svc.ClusterID) + if err != nil { + logger.Errorf("failed removing smb cluster '%s': %v", svc.ClusterID, err) + return mcTypes.SmartError(err) + } + + return mcTypes.EmptySyncResponse +} + +// cmdSMBNodeDelete tears down smb cluster membership on this node. +func cmdSMBNodeDelete(s mcTypes.State, r *http.Request) mcTypes.Response { + var svc types.SMBService + + err := json.NewDecoder(r.Body).Decode(&svc) + if err != nil { + logger.Errorf("failed decoding smb node delete request: %v", err) + return mcTypes.InternalError(err) + } + + err = ceph.DisableSMB(r.Context(), interfaces.CephState{State: s}, svc.ClusterID) + if err != nil { + logger.Errorf("failed disabling smb on node: %v", err) + return mcTypes.SmartError(err) + } + + return mcTypes.EmptySyncResponse +} diff --git a/microceph/ceph/smb.go b/microceph/ceph/smb.go new file mode 100644 index 00000000..89fb6319 --- /dev/null +++ b/microceph/ceph/smb.go @@ -0,0 +1,284 @@ +package ceph + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "sort" + + "github.com/canonical/lxd/shared/api" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/interfaces" + "github.com/canonical/microceph/microceph/logger" +) + +// ErrInvalidSMBSpec marks SMBSpec validation failures so the API layer can +// map them to HTTP 400 instead of 500. +var ErrInvalidSMBSpec = errors.New("invalid smb spec") + +// Injectable seams for unit tests. +var ( + smbClusterMembersFunc = smbClusterMembers + smbEnableNodeFunc = smbEnableNode + smbDisableNodeFunc = smbDisableNode +) + +// ResolveSMBPlacement resolves the spec placement to a sorted set of +// cluster member names. Phase 1 honors hosts and count (count picks the +// first N sorted candidates, matching cephadm's count-of-hosts semantics); +// label, host_pattern and count_per_host are rejected. +func ResolveSMBPlacement(spec *types.SMBSpec, members []string) ([]string, error) { + placement := spec.Placement + + if placement.Label != "" { + return nil, fmt.Errorf("placement: 'label' is not supported (microceph has no host labels)") + } + if placement.CountPerHost != 0 { + return nil, fmt.Errorf("placement: 'count_per_host' is not supported") + } + if len(placement.HostPattern) > 0 && string(placement.HostPattern) != "null" { + return nil, fmt.Errorf("placement: 'host_pattern' is not supported") + } + + var candidates []string + if len(placement.Hosts) > 0 { + memberSet := make(map[string]bool, len(members)) + for _, member := range members { + memberSet[member] = true + } + + seen := make(map[string]bool, len(placement.Hosts)) + for _, host := range placement.Hosts { + if !memberSet[host] { + return nil, fmt.Errorf("placement: host '%s' is not a cluster member", host) + } + if !seen[host] { + seen[host] = true + candidates = append(candidates, host) + } + } + } else if placement.Count > 0 { + candidates = append(candidates, members...) + } else { + return nil, fmt.Errorf("placement requires 'hosts' or 'count'") + } + + sort.Strings(candidates) + + if placement.Count > 0 { + if placement.Count > len(candidates) { + return nil, fmt.Errorf("placement: count %d exceeds the %d available hosts", placement.Count, len(candidates)) + } + candidates = candidates[:placement.Count] + } + + return candidates, nil +} + +// DiffSMBPlacement returns the node sets to enable and disable to converge +// from the current membership to the desired one. +func DiffSMBPlacement(desired, current []string) ([]string, []string) { + desiredSet := make(map[string]bool, len(desired)) + for _, node := range desired { + desiredSet[node] = true + } + currentSet := make(map[string]bool, len(current)) + for _, node := range current { + currentSet[node] = true + } + + var toEnable, toDisable []string + for _, node := range desired { + if !currentSet[node] { + toEnable = append(toEnable, node) + } + } + for _, node := range current { + if !desiredSet[node] { + toDisable = append(toDisable, node) + } + } + + sort.Strings(toEnable) + sort.Strings(toDisable) + return toEnable, toDisable +} + +// ApplySMB validates an SMBSpec payload, computes the placement diff +// against the recorded membership and drives per-node enable/disable +// across the cluster. Fan-out is fail-fast: a partial apply is converged +// by re-applying (the flow is idempotent). +func ApplySMB(ctx context.Context, s interfaces.StateInterface, payload string) error { + // Canonicalize so stored configs compare stably: AddNew compacts the + // raw spec on write, so every comparison must use compacted bytes too. + var buf bytes.Buffer + err := json.Compact(&buf, []byte(payload)) + if err != nil { + return fmt.Errorf("%w: %v", ErrInvalidSMBSpec, err) + } + canonical := buf.String() + + sp := &SMBServicePlacement{} + err = sp.PopulateParams(s, canonical) + if err != nil { + return fmt.Errorf("%w: %v", ErrInvalidSMBSpec, err) + } + + members, err := smbClusterMembersFunc(s) + if err != nil { + return fmt.Errorf("failed to list cluster members: %w", err) + } + + desired, err := ResolveSMBPlacement(&sp.Spec, members) + if err != nil { + return fmt.Errorf("%w: %v", ErrInvalidSMBSpec, err) + } + + current, err := database.GroupedServicesQuery.GetGroupMembers(ctx, s, "smb", sp.Spec.ClusterID) + if err != nil { + return fmt.Errorf("failed to fetch smb cluster membership: %w", err) + } + + // Refresh the stored spec on re-apply so joining nodes render from the + // latest config. Rolling regeneration of already placed members lands + // with M3 lifecycle. + if len(current) > 0 { + existing, err := database.GroupedServicesQuery.GetGroupConfig(ctx, s, "smb", sp.Spec.ClusterID) + if err != nil { + return fmt.Errorf("failed to fetch smb cluster config: %w", err) + } + if existing != canonical { + err = database.GroupedServicesQuery.UpdateGroupConfig(ctx, s, "smb", sp.Spec.ClusterID, canonical) + if err != nil { + return fmt.Errorf("failed to update smb cluster config: %w", err) + } + } + } + + toEnable, toDisable := DiffSMBPlacement(desired, current) + logger.Infof("smb apply %s: enable %v, disable %v", sp.Spec.ClusterID, toEnable, toDisable) + + for _, node := range toEnable { + err = smbEnableNodeFunc(ctx, s, node, canonical) + if err != nil { + return fmt.Errorf("failed to enable smb cluster '%s' on node '%s': %w", sp.Spec.ClusterID, node, err) + } + } + + for _, node := range toDisable { + err = smbDisableNodeFunc(ctx, s, node, sp.Spec.ClusterID) + if err != nil { + return fmt.Errorf("failed to disable smb cluster '%s' on node '%s': %w", sp.Spec.ClusterID, node, err) + } + } + + return nil +} + +// RemoveSMB drives removal of an smb cluster from all its member nodes. +// The RADOS objects referenced by the spec belong to mgr/smb and are left +// untouched. +func RemoveSMB(ctx context.Context, s interfaces.StateInterface, clusterID string) error { + current, err := database.GroupedServicesQuery.GetGroupMembers(ctx, s, "smb", clusterID) + if err != nil { + return fmt.Errorf("failed to fetch smb cluster membership: %w", err) + } + + if len(current) == 0 { + return api.StatusErrorf(http.StatusNotFound, "no smb cluster '%s'", clusterID) + } + + for _, node := range current { + err = smbDisableNodeFunc(ctx, s, node, clusterID) + if err != nil { + return fmt.Errorf("failed to disable smb cluster '%s' on node '%s': %w", clusterID, node, err) + } + } + + return nil +} + +// ListSMB reports every smb cluster with its stored spec and current +// placement. +func ListSMB(ctx context.Context, s interfaces.StateInterface) ([]types.SMBServiceStatus, error) { + rows, err := database.GroupedServicesQuery.GetGroupedServices(ctx, s) + if err != nil { + return nil, fmt.Errorf("failed to fetch grouped services: %w", err) + } + + membersByCluster := map[string][]string{} + for _, row := range rows { + if row.Service != "smb" { + continue + } + membersByCluster[row.GroupID] = append(membersByCluster[row.GroupID], row.Member) + } + + statuses := make([]types.SMBServiceStatus, 0, len(membersByCluster)) + for clusterID, members := range membersByCluster { + config, err := database.GroupedServicesQuery.GetGroupConfig(ctx, s, "smb", clusterID) + if err != nil { + return nil, fmt.Errorf("failed to fetch config for smb cluster '%s': %w", clusterID, err) + } + + sort.Strings(members) + statuses = append(statuses, types.SMBServiceStatus{ + ClusterID: clusterID, + Spec: json.RawMessage(config), + PlacedOn: members, + }) + } + + sort.Slice(statuses, func(i, j int) bool { return statuses[i].ClusterID < statuses[j].ClusterID }) + return statuses, nil +} + +// DisableSMB removes this node from the smb cluster's records. Service +// teardown (ctdbd stop, config cleanup) lands with M3 lifecycle. +func DisableSMB(ctx context.Context, s interfaces.StateInterface, clusterID string) error { + return database.GroupedServicesQuery.RemoveForHost(ctx, s, "smb", clusterID) +} + +// smbClusterMembers lists the cluster member names. +func smbClusterMembers(s interfaces.StateInterface) ([]string, error) { + cli, err := s.ClusterState().Connect().Leader(false) + if err != nil { + return nil, err + } + return client.MClient.GetClusterMembers(cli) +} + +// smbEnableNode runs the smb placement flow on the given node, locally or +// via the node-scoped endpoint. +func smbEnableNode(ctx context.Context, s interfaces.StateInterface, node, payload string) error { + data := types.EnableService{Name: "smb", Wait: true, Payload: payload} + if node == s.ClusterState().Name() { + return ServicePlacementHandler(ctx, s, data) + } + + cli, err := s.ClusterState().Connect().Leader(false) + if err != nil { + return err + } + return client.EnableSMBNodeService(ctx, cli, node, &data) +} + +// smbDisableNode tears down smb membership on the given node, locally or +// via the node-scoped endpoint. +func smbDisableNode(ctx context.Context, s interfaces.StateInterface, node, clusterID string) error { + if node == s.ClusterState().Name() { + return DisableSMB(ctx, s, clusterID) + } + + cli, err := s.ClusterState().Connect().Leader(false) + if err != nil { + return err + } + return client.DeleteSMBNodeService(ctx, cli, node, &types.SMBService{ClusterID: clusterID}) +} diff --git a/microceph/ceph/smb_test.go b/microceph/ceph/smb_test.go new file mode 100644 index 00000000..ee62976c --- /dev/null +++ b/microceph/ceph/smb_test.go @@ -0,0 +1,279 @@ +package ceph + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/interfaces" + "github.com/canonical/microceph/microceph/mocks" + "github.com/canonical/microceph/microceph/tests" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// mustCompactJSON compacts a JSON document, panicking on invalid input. +func mustCompactJSON(in string) string { + var buf bytes.Buffer + err := json.Compact(&buf, []byte(in)) + if err != nil { + panic(err) + } + return buf.String() +} + +type smbSuite struct { + tests.BaseSuite + TestStateInterface *mocks.StateInterface + + enabled []string + disabled []string +} + +func TestSMBSuite(t *testing.T) { + suite.Run(t, new(smbSuite)) +} + +// SetupTest wires recorder seams so orchestration tests observe exact +// per-node enable/disable sets without running the placement flow. +func (s *smbSuite) SetupTest() { + s.BaseSuite.SetupTest() + s.TestStateInterface = mocks.NewStateInterface(s.T()) + + s.enabled = nil + s.disabled = nil + + originalMembers := smbClusterMembersFunc + originalEnable := smbEnableNodeFunc + originalDisable := smbDisableNodeFunc + s.T().Cleanup(func() { + smbClusterMembersFunc = originalMembers + smbEnableNodeFunc = originalEnable + smbDisableNodeFunc = originalDisable + }) + + smbClusterMembersFunc = func(s interfaces.StateInterface) ([]string, error) { + return []string{"m1", "m2", "m3"}, nil + } + smbEnableNodeFunc = func(ctx context.Context, st interfaces.StateInterface, node, payload string) error { + s.enabled = append(s.enabled, node) + return nil + } + smbDisableNodeFunc = func(ctx context.Context, st interfaces.StateInterface, node, clusterID string) error { + s.disabled = append(s.disabled, node) + return nil + } +} + +// withDB patches the grouped-services query with a fresh mock and restores +// the original on test cleanup. +func (s *smbSuite) withDB() *mocks.GroupedServiceQueryIntf { + db := mocks.NewGroupedServiceQueryIntf(s.T()) + originalDB := database.GroupedServicesQuery + s.T().Cleanup(func() { database.GroupedServicesQuery = originalDB }) + database.GroupedServicesQuery = db + return db +} + +func smbPayload(hosts string, count int) string { + placement := fmt.Sprintf(`{"hosts": [%s]}`, hosts) + if count > 0 { + placement = fmt.Sprintf(`{"hosts": [%s], "count": %d}`, hosts, count) + } + return fmt.Sprintf(`{"service_type": "smb", "service_id": "dev", "cluster_id": "dev", "placement": %s}`, placement) +} + +// --- ResolveSMBPlacement --- + +func (s *smbSuite) resolve(placementJSON string, members ...string) ([]string, error) { + var spec types.SMBSpec + err := json.Unmarshal([]byte(fmt.Sprintf(`{"cluster_id": "dev", "placement": %s}`, placementJSON)), &spec) + assert.NoError(s.T(), err) + return ResolveSMBPlacement(&spec, members) +} + +func (s *smbSuite) TestResolveHosts() { + nodes, err := s.resolve(`{"hosts": ["m2", "m1"]}`, "m1", "m2", "m3") + assert.NoError(s.T(), err) + assert.Equal(s.T(), []string{"m1", "m2"}, nodes) +} + +func (s *smbSuite) TestResolveHostsUnknown() { + _, err := s.resolve(`{"hosts": ["m1", "ghost"]}`, "m1", "m2") + assert.ErrorContains(s.T(), err, "ghost") +} + +func (s *smbSuite) TestResolveHostsDeduped() { + nodes, err := s.resolve(`{"hosts": ["m1", "m1", "m2"]}`, "m1", "m2") + assert.NoError(s.T(), err) + assert.Equal(s.T(), []string{"m1", "m2"}, nodes) +} + +func (s *smbSuite) TestResolveCount() { + nodes, err := s.resolve(`{"count": 2}`, "m3", "m1", "m2") + assert.NoError(s.T(), err) + assert.Equal(s.T(), []string{"m1", "m2"}, nodes) +} + +func (s *smbSuite) TestResolveCountTooLarge() { + _, err := s.resolve(`{"count": 4}`, "m1", "m2", "m3") + assert.ErrorContains(s.T(), err, "count") +} + +func (s *smbSuite) TestResolveHostsWithCount() { + nodes, err := s.resolve(`{"hosts": ["m3", "m1", "m2"], "count": 2}`, "m1", "m2", "m3") + assert.NoError(s.T(), err) + assert.Equal(s.T(), []string{"m1", "m2"}, nodes) +} + +func (s *smbSuite) TestResolveLabelUnsupported() { + _, err := s.resolve(`{"label": "smb"}`, "m1") + assert.ErrorContains(s.T(), err, "label") +} + +func (s *smbSuite) TestResolveCountPerHostUnsupported() { + _, err := s.resolve(`{"hosts": ["m1"], "count_per_host": 2}`, "m1") + assert.ErrorContains(s.T(), err, "count_per_host") +} + +func (s *smbSuite) TestResolveHostPatternUnsupported() { + _, err := s.resolve(`{"host_pattern": "m*"}`, "m1") + assert.ErrorContains(s.T(), err, "host_pattern") +} + +func (s *smbSuite) TestResolveEmptyPlacement() { + _, err := s.resolve(`{}`, "m1") + assert.ErrorContains(s.T(), err, "placement") +} + +// --- DiffSMBPlacement --- + +func (s *smbSuite) TestDiffIdempotent() { + toEnable, toDisable := DiffSMBPlacement([]string{"m1", "m2"}, []string{"m1", "m2"}) + assert.Empty(s.T(), toEnable) + assert.Empty(s.T(), toDisable) +} + +func (s *smbSuite) TestDiffFresh() { + toEnable, toDisable := DiffSMBPlacement([]string{"m1", "m2"}, nil) + assert.Equal(s.T(), []string{"m1", "m2"}, toEnable) + assert.Empty(s.T(), toDisable) +} + +func (s *smbSuite) TestDiffMemberChange() { + toEnable, toDisable := DiffSMBPlacement([]string{"m1", "m2"}, []string{"m2", "m3"}) + assert.Equal(s.T(), []string{"m1"}, toEnable) + assert.Equal(s.T(), []string{"m3"}, toDisable) +} + +// --- ApplySMB --- + +func (s *smbSuite) TestApplyFresh() { + db := s.withDB() + db.On("GetGroupMembers", context.Background(), s.TestStateInterface, "smb", "dev").Return([]string{}, nil).Once() + + err := ApplySMB(context.Background(), s.TestStateInterface, smbPayload(`"m1", "m2", "m3"`, 0)) + assert.NoError(s.T(), err) + assert.Equal(s.T(), []string{"m1", "m2", "m3"}, s.enabled) + assert.Empty(s.T(), s.disabled) +} + +func (s *smbSuite) TestApplyIdempotent() { + payload := smbPayload(`"m1", "m2", "m3"`, 0) + canonical := mustCompactJSON(payload) + + db := s.withDB() + db.On("GetGroupMembers", context.Background(), s.TestStateInterface, "smb", "dev").Return([]string{"m1", "m2", "m3"}, nil).Once() + db.On("GetGroupConfig", context.Background(), s.TestStateInterface, "smb", "dev").Return(canonical, nil).Once() + + err := ApplySMB(context.Background(), s.TestStateInterface, payload) + assert.NoError(s.T(), err) + assert.Empty(s.T(), s.enabled) + assert.Empty(s.T(), s.disabled) +} + +func (s *smbSuite) TestApplyMemberChange() { + payload := smbPayload(`"m1", "m2"`, 0) + canonical := mustCompactJSON(payload) + + db := s.withDB() + db.On("GetGroupMembers", context.Background(), s.TestStateInterface, "smb", "dev").Return([]string{"m2", "m3"}, nil).Once() + db.On("GetGroupConfig", context.Background(), s.TestStateInterface, "smb", "dev").Return(canonical, nil).Once() + + err := ApplySMB(context.Background(), s.TestStateInterface, payload) + assert.NoError(s.T(), err) + assert.Equal(s.T(), []string{"m1"}, s.enabled) + assert.Equal(s.T(), []string{"m3"}, s.disabled) +} + +func (s *smbSuite) TestApplyConfigChange() { + payload := smbPayload(`"m1", "m2"`, 0) + canonical := mustCompactJSON(payload) + + db := s.withDB() + db.On("GetGroupMembers", context.Background(), s.TestStateInterface, "smb", "dev").Return([]string{"m1", "m2"}, nil).Once() + db.On("GetGroupConfig", context.Background(), s.TestStateInterface, "smb", "dev").Return(`{"stale": true}`, nil).Once() + db.On("UpdateGroupConfig", context.Background(), s.TestStateInterface, "smb", "dev", canonical).Return(nil).Once() + + err := ApplySMB(context.Background(), s.TestStateInterface, payload) + assert.NoError(s.T(), err) + assert.Empty(s.T(), s.enabled) + assert.Empty(s.T(), s.disabled) +} + +func (s *smbSuite) TestApplyInvalidSpec() { + err := ApplySMB(context.Background(), s.TestStateInterface, `{"cluster_id": "-bad-"}`) + assert.ErrorIs(s.T(), err, ErrInvalidSMBSpec) + assert.Empty(s.T(), s.enabled) + assert.Empty(s.T(), s.disabled) +} + +func (s *smbSuite) TestApplyUnknownHost() { + err := ApplySMB(context.Background(), s.TestStateInterface, smbPayload(`"m1", "ghost"`, 0)) + assert.ErrorIs(s.T(), err, ErrInvalidSMBSpec) + assert.Empty(s.T(), s.enabled) +} + +// --- RemoveSMB --- + +func (s *smbSuite) TestRemoveSMB() { + db := s.withDB() + db.On("GetGroupMembers", context.Background(), s.TestStateInterface, "smb", "dev").Return([]string{"m1", "m2"}, nil).Once() + + err := RemoveSMB(context.Background(), s.TestStateInterface, "dev") + assert.NoError(s.T(), err) + assert.Equal(s.T(), []string{"m1", "m2"}, s.disabled) +} + +func (s *smbSuite) TestRemoveSMBUnknown() { + db := s.withDB() + db.On("GetGroupMembers", context.Background(), s.TestStateInterface, "smb", "ghost").Return([]string{}, nil).Once() + + err := RemoveSMB(context.Background(), s.TestStateInterface, "ghost") + assert.ErrorContains(s.T(), err, "no smb cluster") + assert.Empty(s.T(), s.disabled) +} + +// --- ListSMB --- + +func (s *smbSuite) TestListSMB() { + db := s.withDB() + db.On("GetGroupedServices", context.Background(), s.TestStateInterface).Return([]database.GroupedService{ + {Service: "smb", GroupID: "dev", Member: "m2"}, + {Service: "smb", GroupID: "dev", Member: "m1"}, + {Service: "nfs", GroupID: "other", Member: "m1"}, + }, nil).Once() + db.On("GetGroupConfig", context.Background(), s.TestStateInterface, "smb", "dev").Return(`{"cluster_id":"dev"}`, nil).Once() + + statuses, err := ListSMB(context.Background(), s.TestStateInterface) + assert.NoError(s.T(), err) + assert.Len(s.T(), statuses, 1) + assert.Equal(s.T(), "dev", statuses[0].ClusterID) + assert.Equal(s.T(), []string{"m1", "m2"}, statuses[0].PlacedOn) + assert.JSONEq(s.T(), `{"cluster_id":"dev"}`, string(statuses[0].Spec)) +} diff --git a/microceph/client/services.go b/microceph/client/services.go index 0b1c724a..7aa71b44 100644 --- a/microceph/client/services.go +++ b/microceph/client/services.go @@ -76,6 +76,39 @@ func SendServicePlacementReq(ctx context.Context, c mcTypes.Client, data *types. return nil } +// EnableSMBNodeService requests the target node run the smb placement flow. +func EnableSMBNodeService(ctx context.Context, c mcTypes.Client, target string, data *types.EnableService) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + defer cancel() + + // Send this request to target. + c = c.UseTarget(target) + + err := c.Query(queryCtx, "PUT", types.ExtendedPathPrefix, &api.NewURL().Path("services", "smb", "node").URL, data, nil) + if err != nil { + return fmt.Errorf("failed placing smb service on %s: %w", target, err) + } + + return nil +} + +// DeleteSMBNodeService requests the target node tear down its smb cluster +// membership. +func DeleteSMBNodeService(ctx context.Context, c mcTypes.Client, target string, svc *types.SMBService) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + defer cancel() + + // Send this request to target. + c = c.UseTarget(target) + + err := c.Query(queryCtx, "DELETE", types.ExtendedPathPrefix, &api.NewURL().Path("services", "smb", "node").URL, svc, nil) + if err != nil { + return fmt.Errorf("failed deleting smb service on %s: %w", target, err) + } + + return nil +} + // Sends a request to the host to restart the provided service. func RestartService(ctx context.Context, c mcTypes.Client, data *types.Services) error { // 120 second timeout for waiting. diff --git a/microceph/database/grouped_service_extras.go b/microceph/database/grouped_service_extras.go index 75fc15fa..c640a01b 100644 --- a/microceph/database/grouped_service_extras.go +++ b/microceph/database/grouped_service_extras.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "net/http" + "sort" "github.com/canonical/microceph/microceph/interfaces" @@ -26,6 +27,11 @@ type GroupedServiceQueryIntf interface { // Exists Methods ExistsOnHost(ctx context.Context, s interfaces.StateInterface, service, groupID string) (bool, error) + // Group Methods + GetGroupMembers(ctx context.Context, s interfaces.StateInterface, service, groupID string) ([]string, error) + GetGroupConfig(ctx context.Context, s interfaces.StateInterface, service, groupID string) (string, error) + UpdateGroupConfig(ctx context.Context, s interfaces.StateInterface, service, groupID, config string) error + // Delete Methods RemoveForHost(ctx context.Context, s interfaces.StateInterface, service, groupID string) error } @@ -151,6 +157,75 @@ func (g GroupedServiceQueryImpl) ExistsOnHost(ctx context.Context, s interfaces. return exists, err } +// GetGroupMembers returns the sorted member names of a service group. +func (g GroupedServiceQueryImpl) GetGroupMembers(ctx context.Context, s interfaces.StateInterface, service, groupID string) ([]string, error) { + if s.ClusterState().ServerCert() == nil { + return nil, fmt.Errorf("no server certificate") + } + + var members []string + + err := s.ClusterState().Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + filter := GroupedServiceFilter{ + Service: &service, + GroupID: &groupID, + } + + services, err := GetGroupedServices(ctx, tx, filter) + if err != nil { + return fmt.Errorf("failed to get grouped services records: %w", err) + } + + for _, service := range services { + members = append(members, service.Member) + } + + return nil + }) + if err != nil { + return nil, err + } + + sort.Strings(members) + return members, nil +} + +// GetGroupConfig returns the stored config of a service group. +func (g GroupedServiceQueryImpl) GetGroupConfig(ctx context.Context, s interfaces.StateInterface, service, groupID string) (string, error) { + if s.ClusterState().ServerCert() == nil { + return "", fmt.Errorf("no server certificate") + } + + var config string + + err := s.ClusterState().Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + serviceGroup, err := GetServiceGroup(ctx, tx, service, groupID) + if err != nil { + return err + } + + config = serviceGroup.Config + return nil + }) + + return config, err +} + +// UpdateGroupConfig replaces the stored config of a service group. +func (g GroupedServiceQueryImpl) UpdateGroupConfig(ctx context.Context, s interfaces.StateInterface, service, groupID, config string) error { + if s.ClusterState().ServerCert() == nil { + return fmt.Errorf("no server certificate") + } + + return s.ClusterState().Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + return UpdateServiceGroup(ctx, tx, service, groupID, ServiceGroup{ + Service: service, + GroupID: groupID, + Config: config, + }) + }) +} + // RemoveForHost deletes the given service record in the grouped_service database, and deletes the // service record from the service_groups database if there is no grouped_service referencing it. func (g GroupedServiceQueryImpl) RemoveForHost(ctx context.Context, s interfaces.StateInterface, service, groupID string) error { diff --git a/microceph/mocks/GroupedServiceQueryIntf.go b/microceph/mocks/GroupedServiceQueryIntf.go index 4b6cd321..31ffcfc2 100644 --- a/microceph/mocks/GroupedServiceQueryIntf.go +++ b/microceph/mocks/GroupedServiceQueryIntf.go @@ -140,6 +140,82 @@ func (_m *GroupedServiceQueryIntf) RemoveForHost(ctx context.Context, s interfac return r0 } +// GetGroupMembers provides a mock function with given fields: ctx, s, service, groupID +func (_m *GroupedServiceQueryIntf) GetGroupMembers(ctx context.Context, s interfaces.StateInterface, service string, groupID string) ([]string, error) { + ret := _m.Called(ctx, s, service, groupID) + + if len(ret) == 0 { + panic("no return value specified for GetGroupMembers") + } + + var r0 []string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, interfaces.StateInterface, string, string) ([]string, error)); ok { + return rf(ctx, s, service, groupID) + } + if rf, ok := ret.Get(0).(func(context.Context, interfaces.StateInterface, string, string) []string); ok { + r0 = rf(ctx, s, service, groupID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, interfaces.StateInterface, string, string) error); ok { + r1 = rf(ctx, s, service, groupID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetGroupConfig provides a mock function with given fields: ctx, s, service, groupID +func (_m *GroupedServiceQueryIntf) GetGroupConfig(ctx context.Context, s interfaces.StateInterface, service string, groupID string) (string, error) { + ret := _m.Called(ctx, s, service, groupID) + + if len(ret) == 0 { + panic("no return value specified for GetGroupConfig") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, interfaces.StateInterface, string, string) (string, error)); ok { + return rf(ctx, s, service, groupID) + } + if rf, ok := ret.Get(0).(func(context.Context, interfaces.StateInterface, string, string) string); ok { + r0 = rf(ctx, s, service, groupID) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, interfaces.StateInterface, string, string) error); ok { + r1 = rf(ctx, s, service, groupID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// UpdateGroupConfig provides a mock function with given fields: ctx, s, service, groupID, config +func (_m *GroupedServiceQueryIntf) UpdateGroupConfig(ctx context.Context, s interfaces.StateInterface, service string, groupID string, config string) error { + ret := _m.Called(ctx, s, service, groupID, config) + + if len(ret) == 0 { + panic("no return value specified for UpdateGroupConfig") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, interfaces.StateInterface, string, string, string) error); ok { + r0 = rf(ctx, s, service, groupID, config) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // NewGroupedServiceQueryIntf creates a new instance of GroupedServiceQueryIntf. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewGroupedServiceQueryIntf(t interface { From 20ece8cd0055b584960bfb9295cff398fefafbda Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 16:04:40 +0530 Subject: [PATCH 05/31] snap: productize samba/ctdb packaging for native SMB Tighten the samba part to a whitelist prime: the smbd/ctdb daemons and tools, ctdb helpers (incl. ctdb_mutex_ceph_rados_helper), samba module dir and the exact ELF NEEDED closure of those binaries, dropping docs, man pages, AD client tools and python bindings. Add the ctdbd app (snapd-managed, install-mode disable) with its start wrapper pointing CTDB_BASE at SNAP_DATA, layouts for /etc/ctdb and /usr/share/ctdb, a snapctl-based replacement 50.samba event script (CTDB manages smbd via snapd, not systemd), and a sambacc part pinned to 0.9 for config translation. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- snap/snapcraft.yaml | 76 ++++++++++++++++++-- snapcraft/commands/ctdbd.start | 20 ++++++ snapcraft/ctdb/events/legacy/50.samba.script | 19 +++++ snapcraft/ctdb/script.options | 5 ++ 4 files changed, 116 insertions(+), 4 deletions(-) create mode 100755 snapcraft/commands/ctdbd.start create mode 100755 snapcraft/ctdb/events/legacy/50.samba.script create mode 100644 snapcraft/ctdb/script.options diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 3656351c..446308ed 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -47,6 +47,10 @@ layout: symlink: $SNAP/lib/ganesha /usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/samba: symlink: $SNAP/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/samba + /usr/share/ctdb: + symlink: $SNAP/usr/share/ctdb + /etc/ctdb: + bind: $SNAP_DATA/conf/ctdb /usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/rados-classes: symlink: $SNAP/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/rados-classes /etc/ceph: @@ -130,6 +134,17 @@ apps: - network-bind - network-control - process-control + ctdbd: + command: commands/ctdbd.start + daemon: simple + install-mode: disable + after: + - daemon + plugs: + - network + - network-bind + - network-control + - process-control osd: command: commands/osd.start reload-command: commands/osd.reload @@ -490,10 +505,63 @@ parts: usr/lib/: lib/ usr/libexec/: libexec/ prime: - - -lib/systemd - - -lib/tmpfiles.d - - -lib/sysusers.d - - -lib/*/avahi + # daemons and tools + - bin/smbd + - bin/smbpasswd + - bin/pdbedit + - bin/testparm + - bin/ctdb + - bin/ctdbd + - bin/ltdbtool + - bin/onnode + # ctdb helpers, incl. ctdb_mutex_ceph_rados_helper + - libexec/ctdb + # smbd's on-demand DCE-RPC helpers (samba-dcerpcd, rpcd_*) + - libexec/samba + # samba private libs + vfs/ldb/auth/gensec modules + - lib/*/samba + # ELF NEEDED closure resolved inside this part (absent from core24 + # and from every other part's prime list) + - lib/*/libavahi-client.so* + - lib/*/libavahi-common.so* + - lib/*/libcups.so* + - lib/*/libdcerpc-binding.so* + - lib/*/libdcerpc-server-core.so* + - lib/*/libdcerpc-server.so* + - lib/*/libdcerpc.so* + - lib/*/libjansson.so* + - lib/*/libldb.so* + - lib/*/libndr-krb5pac.so* + - lib/*/libndr-nbt.so* + - lib/*/libndr-standard.so* + - lib/*/libndr.so* + - lib/*/libnetapi.so* + - lib/*/libpyldb-util*.so* + - lib/*/libpytalloc-util*.so* + - lib/*/libsamba-credentials.so* + - lib/*/libsamba-errors.so* + - lib/*/libsamba-hostconfig.so* + - lib/*/libsamba-passdb.so* + - lib/*/libsamba-util.so* + - lib/*/libsamdb.so* + - lib/*/libsmbconf.so* + - lib/*/libsmbldap.so* + - lib/*/libtalloc.so* + - lib/*/libtdb.so* + - lib/*/libtevent-util.so* + - lib/*/libtevent.so* + - lib/*/liburing.so* + # ctdb runtime data (event scripts + config templates) + - etc/ctdb + - usr/share/ctdb + + sambacc: + plugin: nil + build-packages: + - python3-pip + override-build: | + craftctl default + pip3 install --target="${CRAFT_PART_INSTALL}/lib/python3/dist-packages" sambacc==0.9 logrotate: plugin: nil diff --git a/snapcraft/commands/ctdbd.start b/snapcraft/commands/ctdbd.start new file mode 100755 index 00000000..db263264 --- /dev/null +++ b/snapcraft/commands/ctdbd.start @@ -0,0 +1,20 @@ +#!/bin/bash + +. "${SNAP}/commands/common" + +limits + +wait_for_config + +# ctdbd reads ctdb.conf, script.options and events/ from CTDB_BASE; the +# deployment engine renders per-cluster configs there before starting us. +export CTDB_BASE="${SNAP_DATA}/conf/ctdb" +export CTDB_SOCKET="${SNAP_DATA}/run/ctdb/ctdbd.socket" + +mkdir -p \ + "${CTDB_BASE}" \ + "${SNAP_DATA}/run/ctdb" \ + "${SNAP_COMMON}/logs/ctdb" \ + "${SNAP_COMMON}/data/ctdb" + +exec ctdbd --interactive diff --git a/snapcraft/ctdb/events/legacy/50.samba.script b/snapcraft/ctdb/events/legacy/50.samba.script new file mode 100755 index 00000000..2534bb71 --- /dev/null +++ b/snapcraft/ctdb/events/legacy/50.samba.script @@ -0,0 +1,19 @@ +#!/bin/sh +# Replacement for CTDB's stock 50.samba event script: inside the snap, +# smbd is managed through snapd, not systemd. The deployment engine links +# this into $CTDB_BASE/events/legacy/ when rendering node configs. + +case "$1" in +startup) + snapctl start microceph.smbd + ;; +shutdown) + snapctl stop microceph.smbd + ;; +monitor) + # Stock share checks assume /etc/samba paths and real directories; + # vfs-backed shares are virtual, so smbd health is left to snapd. + ;; +esac + +exit 0 diff --git a/snapcraft/ctdb/script.options b/snapcraft/ctdb/script.options new file mode 100644 index 00000000..f13c5212 --- /dev/null +++ b/snapcraft/ctdb/script.options @@ -0,0 +1,5 @@ +# Options for CTDB event scripts in the microceph snap. +# The stock 50.samba share check fails on vfs-backed (virtual) share +# paths; the snapctl replacement script ignores it, but keep the knob set +# for anything that still consults it. +CTDB_SAMBA_SKIP_SHARE_CHECK=yes From 80465643005671d5e932c487e05146be4c1b8f59 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 16:21:20 +0530 Subject: [PATCH 06/31] snap: ship real mgr/smb module and fix samba part priming Replace the samba part's prime whitelist with negative-list pruning: snapcraft resolves shared staged files against every staging part's filter, so whitelisting inside the samba part silently dropped libs other parts prime (ceph-mgr's libpython3.12, dashboard's markdown and dnspython, libldap/libicu symlink targets). Exclude only samba-owned paths: docs, man pages, GPO templates, and AD/DC/NetBIOS tooling. Retire stub patch 0003 and add an mgr-smb part that sparse-clones the ceph tag matching the staged PPA debs (asserted against apt-cache policy at build time) and installs src/pybind/mgr/smb into the snap's mgr module path. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- patches/0003-add-stub-smb-mgr-module.patch | 95 ------------------ snap/snapcraft.yaml | 108 +++++++++++---------- 2 files changed, 59 insertions(+), 144 deletions(-) delete mode 100644 patches/0003-add-stub-smb-mgr-module.patch diff --git a/patches/0003-add-stub-smb-mgr-module.patch b/patches/0003-add-stub-smb-mgr-module.patch deleted file mode 100644 index 0830cabf..00000000 --- a/patches/0003-add-stub-smb-mgr-module.patch +++ /dev/null @@ -1,95 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Utkarsh Bhatt -Date: Wed, 9 Apr 2026 14:00:00 +0530 -Subject: [PATCH] Add stub smb mgr module for dashboard compatibility - -The ceph-mgr-smb package is not available in the Ubuntu distribution. -The Ceph tentacle dashboard imports from the smb mgr module -unconditionally (controllers/smb.py), causing the dashboard to fail -to load without it. - -This adds a minimal stub that satisfies the dashboard's imports -(Intent, Simplified, Cluster, JoinAuth, Share, UsersAndGroups) -without providing actual SMB functionality. The dashboard's SMB -status endpoint will report SMB as unavailable. - -Signed-off-by: Utkarsh Bhatt ---- - share/ceph/mgr/smb/__init__.py | 1 + - share/ceph/mgr/smb/enums.py | 17 +++++++++++++++++ - share/ceph/mgr/smb/proto.py | 5 +++++ - share/ceph/mgr/smb/resources.py | 20 ++++++++++++++++++++ - 4 files changed, 43 insertions(+) - create mode 100644 share/ceph/mgr/smb/__init__.py - create mode 100644 share/ceph/mgr/smb/enums.py - create mode 100644 share/ceph/mgr/smb/proto.py - create mode 100644 share/ceph/mgr/smb/resources.py - -diff --git a/share/ceph/mgr/smb/__init__.py b/share/ceph/mgr/smb/__init__.py -new file mode 100644 -index 0000000..b357543 ---- /dev/null -+++ b/share/ceph/mgr/smb/__init__.py -@@ -0,0 +1 @@ -+# Stub smb mgr module — ceph-mgr-smb is not available in the distribution. -diff --git a/share/ceph/mgr/smb/enums.py b/share/ceph/mgr/smb/enums.py -new file mode 100644 -index 0000000..a1c2e3f ---- /dev/null -+++ b/share/ceph/mgr/smb/enums.py -@@ -0,0 +1,17 @@ -+"""Stub enums for the smb mgr module.""" -+ -+import sys -+ -+if sys.version_info >= (3, 11): -+ from enum import StrEnum as _StrEnum -+else: -+ import enum -+ -+ class _StrEnum(str, enum.Enum): -+ def __str__(self) -> str: -+ return self.value -+ -+ -+class Intent(_StrEnum): -+ PRESENT = 'present' -+ REMOVED = 'removed' -diff --git a/share/ceph/mgr/smb/proto.py b/share/ceph/mgr/smb/proto.py -new file mode 100644 -index 0000000..d3c5a7e ---- /dev/null -+++ b/share/ceph/mgr/smb/proto.py -@@ -0,0 +1,5 @@ -+"""Stub proto types for the smb mgr module.""" -+ -+from typing import Any, Dict -+ -+Simplified = Dict[str, Any] -diff --git a/share/ceph/mgr/smb/resources.py b/share/ceph/mgr/smb/resources.py -new file mode 100644 -index 0000000..f8e1d2c ---- /dev/null -+++ b/share/ceph/mgr/smb/resources.py -@@ -0,0 +1,19 @@ -+"""Stub resource types for the smb mgr module.""" -+ -+from typing import Any, Dict -+ -+ -+class Cluster(Dict[str, Any]): -+ pass -+ -+ -+class JoinAuth(Dict[str, Any]): -+ pass -+ -+ -+class Share(Dict[str, Any]): -+ pass -+ -+ -+class UsersAndGroups(Dict[str, Any]): -+ pass --- -2.43.0 diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 446308ed..d5176249 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -504,56 +504,66 @@ parts: usr/sbin/: bin/ usr/lib/: lib/ usr/libexec/: libexec/ + # Negative-list pruning only: a whitelist here silently drops files + # this part stages that OTHER parts also stage and prime (snapcraft + # resolves shared staged files against every owner's filter), which + # broke ceph-mgr's libpython and the dashboard's python deps. Only + # samba-owned paths are excluded. prime: - # daemons and tools - - bin/smbd - - bin/smbpasswd - - bin/pdbedit - - bin/testparm - - bin/ctdb - - bin/ctdbd - - bin/ltdbtool - - bin/onnode - # ctdb helpers, incl. ctdb_mutex_ceph_rados_helper - - libexec/ctdb - # smbd's on-demand DCE-RPC helpers (samba-dcerpcd, rpcd_*) - - libexec/samba - # samba private libs + vfs/ldb/auth/gensec modules - - lib/*/samba - # ELF NEEDED closure resolved inside this part (absent from core24 - # and from every other part's prime list) - - lib/*/libavahi-client.so* - - lib/*/libavahi-common.so* - - lib/*/libcups.so* - - lib/*/libdcerpc-binding.so* - - lib/*/libdcerpc-server-core.so* - - lib/*/libdcerpc-server.so* - - lib/*/libdcerpc.so* - - lib/*/libjansson.so* - - lib/*/libldb.so* - - lib/*/libndr-krb5pac.so* - - lib/*/libndr-nbt.so* - - lib/*/libndr-standard.so* - - lib/*/libndr.so* - - lib/*/libnetapi.so* - - lib/*/libpyldb-util*.so* - - lib/*/libpytalloc-util*.so* - - lib/*/libsamba-credentials.so* - - lib/*/libsamba-errors.so* - - lib/*/libsamba-hostconfig.so* - - lib/*/libsamba-passdb.so* - - lib/*/libsamba-util.so* - - lib/*/libsamdb.so* - - lib/*/libsmbconf.so* - - lib/*/libsmbldap.so* - - lib/*/libtalloc.so* - - lib/*/libtdb.so* - - lib/*/libtevent-util.so* - - lib/*/libtevent.so* - - lib/*/liburing.so* - # ctdb runtime data (event scripts + config templates) - - etc/ctdb - - usr/share/ctdb + - -lib/systemd + - -lib/tmpfiles.d + - -lib/sysusers.d + - -lib/*/avahi + - -usr/share/doc + - -usr/share/man + - -usr/share/samba + # AD/DC, NetBIOS and registry tooling not used by Phase 1 + - -bin/samba + - -bin/samba-gpupdate + - -bin/samba-tool + - -bin/samba_dnsupdate + - -bin/samba_downgrade_db + - -bin/samba_kcc + - -bin/samba_spnupdate + - -bin/samba_upgradedns + - -bin/samba-regedit + - -bin/samba-log-parser + - -bin/nmbd + - -bin/nmblookup + - -bin/net + - -bin/oLschema2ldif + - -bin/dumpmscat + - -bin/eventlogadm + - -bin/profiles + - -bin/sharesec + - -bin/mvxattr + + # Real upstream mgr/smb module, built from the ceph tag matching the + # staged PPA debs (asserted at build time). Replaces the retired stub + # patch 0003. + mgr-smb: + plugin: nil + build-packages: + - git + override-pull: | + craftctl default + git clone --branch v20.2.1 --depth 1 --filter=blob:none --sparse https://github.com/ceph/ceph.git ceph-src + git -C ceph-src sparse-checkout set src/pybind/mgr/smb + override-build: | + craftctl default + pkg_version=$(apt-cache policy ceph-common | awk '/Candidate:/{ print $2 }') + src_tag=$(git -C ceph-src describe --tags --exact-match) + case "${pkg_version}" in + "${src_tag#v}"-*|"${src_tag#v}"~*|"${src_tag#v}"+*) + ;; + *) + echo "mgr/smb source ${src_tag} does not match staged ceph ${pkg_version}" >&2 + exit 1 + ;; + esac + mkdir -p "${CRAFT_PART_INSTALL}/share/ceph/mgr" + cp -r ceph-src/src/pybind/mgr/smb "${CRAFT_PART_INSTALL}/share/ceph/mgr/" + rm -rf "${CRAFT_PART_INSTALL}/share/ceph/mgr/smb/tests" sambacc: plugin: nil From 1822d309a0162d4419ecda56d7b787a29dbab0fc Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 16:25:30 +0530 Subject: [PATCH 07/31] cli: add hidden smb debug commands Add 'microceph smb apply-spec/rm/list' hitting the services/smb endpoints directly. Hidden: the supported control plane is mgr/smb via the orchestrator; these keep E2E honest before microceph-orch gains smb support, and remain for support tooling. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/client/services.go | 42 +++++++++ microceph/cmd/microceph/main.go | 3 + microceph/cmd/microceph/smb.go | 149 ++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 microceph/cmd/microceph/smb.go diff --git a/microceph/client/services.go b/microceph/client/services.go index 7aa71b44..751f40a8 100644 --- a/microceph/client/services.go +++ b/microceph/client/services.go @@ -3,6 +3,7 @@ package client import ( "context" + "encoding/json" "fmt" "time" @@ -76,6 +77,47 @@ func SendServicePlacementReq(ctx context.Context, c mcTypes.Client, data *types. return nil } +// ApplySMBSpec submits an SMBSpec JSON document for cluster-wide apply. +func ApplySMBSpec(ctx context.Context, c mcTypes.Client, spec []byte) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + defer cancel() + + err := c.Query(queryCtx, "PUT", types.ExtendedPathPrefix, &api.NewURL().Path("services", "smb").URL, json.RawMessage(spec), nil) + if err != nil { + return fmt.Errorf("failed applying smb spec: %w", err) + } + + return nil +} + +// RemoveSMBService removes an smb cluster from all its member nodes. +func RemoveSMBService(ctx context.Context, c mcTypes.Client, svc *types.SMBService) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + defer cancel() + + err := c.Query(queryCtx, "DELETE", types.ExtendedPathPrefix, &api.NewURL().Path("services", "smb").URL, svc, nil) + if err != nil { + return fmt.Errorf("failed removing smb cluster: %w", err) + } + + return nil +} + +// GetSMBServices lists every smb cluster with its spec and placement. +func GetSMBServices(ctx context.Context, c mcTypes.Client) ([]types.SMBServiceStatus, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + + statuses := []types.SMBServiceStatus{} + + err := c.Query(queryCtx, "GET", types.ExtendedPathPrefix, &api.NewURL().Path("services", "smb").URL, nil, &statuses) + if err != nil { + return nil, fmt.Errorf("failed listing smb services: %w", err) + } + + return statuses, nil +} + // EnableSMBNodeService requests the target node run the smb placement flow. func EnableSMBNodeService(ctx context.Context, c mcTypes.Client, target string, data *types.EnableService) error { queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) diff --git a/microceph/cmd/microceph/main.go b/microceph/cmd/microceph/main.go index 016ef73c..f8ecbed2 100644 --- a/microceph/cmd/microceph/main.go +++ b/microceph/cmd/microceph/main.go @@ -57,6 +57,9 @@ func main() { cmdDisable := cmdDisable{common: &commonCmd} app.AddCommand(cmdDisable.Command()) + cmdSMBTop := cmdSMB{common: &commonCmd} + app.AddCommand(cmdSMBTop.Command()) + cmdInit := cmdInit{common: &commonCmd} app.AddCommand(cmdInit.Command()) diff --git a/microceph/cmd/microceph/smb.go b/microceph/cmd/microceph/smb.go new file mode 100644 index 00000000..50f5f01a --- /dev/null +++ b/microceph/cmd/microceph/smb.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/canonical/microcluster/v3/microcluster" + "github.com/spf13/cobra" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" +) + +// cmdSMB is a hidden debug command family: the supported control plane is +// mgr/smb (ceph smb ...) via the orchestrator. These commands drive the +// microcephd endpoints directly for development and support. +type cmdSMB struct { + common *CmdControl +} + +func (c *cmdSMB) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "smb", + Short: "Debug commands for the SMB deployment backend", + Hidden: true, + } + + smbApplyCmd := cmdSMBApply{common: c.common} + smbRmCmd := cmdSMBRm{common: c.common} + smbListCmd := cmdSMBList{common: c.common} + cmd.AddCommand(smbApplyCmd.Command()) + cmd.AddCommand(smbRmCmd.Command()) + cmd.AddCommand(smbListCmd.Command()) + return cmd +} + +// localClient returns a client to the local microcephd socket. +func smbLocalClient(common *CmdControl) (*microcluster.MicroCluster, error) { + return microcluster.App(microcluster.Args{StateDir: common.FlagStateDir}) +} + +type cmdSMBApply struct { + common *CmdControl +} + +func (c *cmdSMBApply) Command() *cobra.Command { + return &cobra.Command{ + Use: "apply-spec ", + Short: "Apply an SMBSpec JSON file to the cluster", + Args: cobra.ExactArgs(1), + RunE: c.Run, + } +} + +// Run handles the smb apply-spec command. +func (c *cmdSMBApply) Run(cmd *cobra.Command, args []string) error { + spec, err := os.ReadFile(args[0]) + if err != nil { + return err + } + + if !json.Valid(spec) { + return fmt.Errorf("%s is not valid JSON", args[0]) + } + + m, err := smbLocalClient(c.common) + if err != nil { + return err + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + return client.ApplySMBSpec(context.Background(), cli, spec) +} + +type cmdSMBRm struct { + common *CmdControl +} + +func (c *cmdSMBRm) Command() *cobra.Command { + return &cobra.Command{ + Use: "rm ", + Short: "Remove an SMB cluster from all its member nodes", + Args: cobra.ExactArgs(1), + RunE: c.Run, + } +} + +// Run handles the smb rm command. +func (c *cmdSMBRm) Run(cmd *cobra.Command, args []string) error { + if !types.SMBClusterIDRegex.MatchString(args[0]) { + return fmt.Errorf("'%s' is not a valid cluster id (regex: '%s')", args[0], types.SMBClusterIDRegex.String()) + } + + m, err := smbLocalClient(c.common) + if err != nil { + return err + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + return client.RemoveSMBService(context.Background(), cli, &types.SMBService{ClusterID: args[0]}) +} + +type cmdSMBList struct { + common *CmdControl +} + +func (c *cmdSMBList) Command() *cobra.Command { + return &cobra.Command{ + Use: "list", + Short: "List SMB clusters with their specs and placement", + RunE: c.Run, + } +} + +// Run handles the smb list command. +func (c *cmdSMBList) Run(cmd *cobra.Command, args []string) error { + m, err := smbLocalClient(c.common) + if err != nil { + return err + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + statuses, err := client.GetSMBServices(context.Background(), cli) + if err != nil { + return err + } + + out, err := json.MarshalIndent(statuses, "", " ") + if err != nil { + return err + } + + fmt.Println(string(out)) + return nil +} From 760b9e0063da625ebea781224c37a6a1415ce071 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 16:37:00 +0530 Subject: [PATCH 08/31] daemon: pass request context to HospitalityCheck The smb hospitality check reads group membership from the cluster DB, and microcluster transactions require the request context (its logger); context.Background() fails with 'Logger does not exist on context', caught live on the first end-to-end spec apply. Thread ctx through PlacementIntf.HospitalityCheck for all services. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/service_placement_client.go | 2 +- microceph/ceph/service_placement_mon.go | 2 +- microceph/ceph/service_placement_nfs.go | 2 +- microceph/ceph/service_placement_smb.go | 4 ++-- microceph/ceph/service_placement_smb_test.go | 10 +++++----- microceph/ceph/services_placement.go | 4 ++-- microceph/ceph/services_placement_generic.go | 2 +- microceph/ceph/services_placement_rgw.go | 2 +- microceph/ceph/services_placement_test.go | 7 ++++--- microceph/mocks/PlacementIntf.go | 10 +++++----- 10 files changed, 23 insertions(+), 22 deletions(-) diff --git a/microceph/ceph/service_placement_client.go b/microceph/ceph/service_placement_client.go index 13568b58..e4259bb9 100644 --- a/microceph/ceph/service_placement_client.go +++ b/microceph/ceph/service_placement_client.go @@ -22,7 +22,7 @@ func (gsp *ClientServicePlacement) PopulateParams(s interfaces.StateInterface, p return nil } -func (gsp *ClientServicePlacement) HospitalityCheck(s interfaces.StateInterface) error { +func (gsp *ClientServicePlacement) HospitalityCheck(ctx context.Context, s interfaces.StateInterface) error { return genericHospitalityCheck(gsp.Name) } diff --git a/microceph/ceph/service_placement_mon.go b/microceph/ceph/service_placement_mon.go index 3eed9c41..3936e891 100644 --- a/microceph/ceph/service_placement_mon.go +++ b/microceph/ceph/service_placement_mon.go @@ -20,7 +20,7 @@ func (msp *MonServicePlacement) PopulateParams(s interfaces.StateInterface, payl } // Check if host is hospitable to the new service to be enabled. -func (msp *MonServicePlacement) HospitalityCheck(s interfaces.StateInterface) error { +func (msp *MonServicePlacement) HospitalityCheck(ctx context.Context, s interfaces.StateInterface) error { return genericHospitalityCheck(msp.Name) } diff --git a/microceph/ceph/service_placement_nfs.go b/microceph/ceph/service_placement_nfs.go index 19206f47..b20c6b6e 100644 --- a/microceph/ceph/service_placement_nfs.go +++ b/microceph/ceph/service_placement_nfs.go @@ -52,7 +52,7 @@ func (nfs *NFSServicePlacement) PopulateParams(s interfaces.StateInterface, payl return nil } -func (nfs *NFSServicePlacement) HospitalityCheck(s interfaces.StateInterface) error { +func (nfs *NFSServicePlacement) HospitalityCheck(ctx context.Context, s interfaces.StateInterface) error { address := fmt.Sprintf("%s:%d", nfs.BindAddress, nfs.BindPort) available, err := isAddressAvailable(address) if err != nil { diff --git a/microceph/ceph/service_placement_smb.go b/microceph/ceph/service_placement_smb.go index 145572bb..24f97ca2 100644 --- a/microceph/ceph/service_placement_smb.go +++ b/microceph/ceph/service_placement_smb.go @@ -93,7 +93,7 @@ func checkSMBUnsupportedFields(payload []byte) error { // HospitalityCheck verifies the SMB port is free and the node is not // already part of an smb cluster (smb clusters are node-disjoint: one // ctdb/smbd instance per node). -func (smb *SMBServicePlacement) HospitalityCheck(s interfaces.StateInterface) error { +func (smb *SMBServicePlacement) HospitalityCheck(ctx context.Context, s interfaces.StateInterface) error { address := fmt.Sprintf("0.0.0.0:%d", smbPort) available, err := isAddressAvailableFunc(address) if err != nil { @@ -102,7 +102,7 @@ func (smb *SMBServicePlacement) HospitalityCheck(s interfaces.StateInterface) er return fmt.Errorf("address '%s' is currently in use.", address) } - services, err := database.GroupedServicesQuery.GetGroupedServicesOnHost(context.Background(), s) + services, err := database.GroupedServicesQuery.GetGroupedServicesOnHost(ctx, s) if err != nil { return fmt.Errorf("failed to fetch smb group membership: %w", err) } diff --git a/microceph/ceph/service_placement_smb_test.go b/microceph/ceph/service_placement_smb_test.go index ee8883df..5260e74d 100644 --- a/microceph/ceph/service_placement_smb_test.go +++ b/microceph/ceph/service_placement_smb_test.go @@ -143,7 +143,7 @@ func (s *servicePlacementSMBSuite) TestHospitalityFreeNode() { defer restore() smb := s.populated(validSMBPayload) - assert.NoError(s.T(), smb.HospitalityCheck(s.TestStateInterface)) + assert.NoError(s.T(), smb.HospitalityCheck(context.Background(), s.TestStateInterface)) } func (s *servicePlacementSMBSuite) TestHospitalityPortBusy() { @@ -151,7 +151,7 @@ func (s *servicePlacementSMBSuite) TestHospitalityPortBusy() { defer restore() smb := s.populated(validSMBPayload) - assert.ErrorContains(s.T(), smb.HospitalityCheck(s.TestStateInterface), "445") + assert.ErrorContains(s.T(), smb.HospitalityCheck(context.Background(), s.TestStateInterface), "445") } func (s *servicePlacementSMBSuite) TestHospitalityNodeInOtherGroup() { @@ -161,7 +161,7 @@ func (s *servicePlacementSMBSuite) TestHospitalityNodeInOtherGroup() { defer restore() smb := s.populated(validSMBPayload) - assert.ErrorContains(s.T(), smb.HospitalityCheck(s.TestStateInterface), "node-disjoint") + assert.ErrorContains(s.T(), smb.HospitalityCheck(context.Background(), s.TestStateInterface), "node-disjoint") } func (s *servicePlacementSMBSuite) TestHospitalityReApplySameGroup() { @@ -171,7 +171,7 @@ func (s *servicePlacementSMBSuite) TestHospitalityReApplySameGroup() { defer restore() smb := s.populated(validSMBPayload) - assert.ErrorContains(s.T(), smb.HospitalityCheck(s.TestStateInterface), "already a member of smb cluster 'dev'") + assert.ErrorContains(s.T(), smb.HospitalityCheck(context.Background(), s.TestStateInterface), "already a member of smb cluster 'dev'") } func (s *servicePlacementSMBSuite) TestHospitalityIgnoresOtherServices() { @@ -181,7 +181,7 @@ func (s *servicePlacementSMBSuite) TestHospitalityIgnoresOtherServices() { defer restore() smb := s.populated(validSMBPayload) - assert.NoError(s.T(), smb.HospitalityCheck(s.TestStateInterface)) + assert.NoError(s.T(), smb.HospitalityCheck(context.Background(), s.TestStateInterface)) } func (s *servicePlacementSMBSuite) TestDBUpdate() { diff --git a/microceph/ceph/services_placement.go b/microceph/ceph/services_placement.go index f619085b..7e5f11b7 100644 --- a/microceph/ceph/services_placement.go +++ b/microceph/ceph/services_placement.go @@ -15,7 +15,7 @@ type PlacementIntf interface { // Populate json payload data to the service object. PopulateParams(interfaces.StateInterface, string) error // Check if host is hospitable to the new service to be enabled. - HospitalityCheck(interfaces.StateInterface) error + HospitalityCheck(context.Context, interfaces.StateInterface) error // Initialise the new service. ServiceInit(context.Context, interfaces.StateInterface) error // Perform Post Placement checks for the service @@ -100,7 +100,7 @@ func EnableService(ctx context.Context, s interfaces.StateInterface, payload typ } // Check if host is hospitable to the new service to be enabled. - err = item.HospitalityCheck(s) + err = item.HospitalityCheck(ctx, s) if err != nil { retErr := fmt.Errorf("host failed hospitality check for %s enablement: %v", payload.Name, err) logger.Error(retErr.Error()) diff --git a/microceph/ceph/services_placement_generic.go b/microceph/ceph/services_placement_generic.go index 968dea80..6cf862ac 100644 --- a/microceph/ceph/services_placement_generic.go +++ b/microceph/ceph/services_placement_generic.go @@ -37,7 +37,7 @@ func (gsp *GenericServicePlacement) PopulateParams(s interfaces.StateInterface, return nil } -func (gsp *GenericServicePlacement) HospitalityCheck(s interfaces.StateInterface) error { +func (gsp *GenericServicePlacement) HospitalityCheck(ctx context.Context, s interfaces.StateInterface) error { return genericHospitalityCheck(gsp.Name) } diff --git a/microceph/ceph/services_placement_rgw.go b/microceph/ceph/services_placement_rgw.go index 50cc3cfb..1e1014d5 100644 --- a/microceph/ceph/services_placement_rgw.go +++ b/microceph/ceph/services_placement_rgw.go @@ -25,7 +25,7 @@ func (rgw *RgwServicePlacement) PopulateParams(s interfaces.StateInterface, payl return nil } -func (rgw *RgwServicePlacement) HospitalityCheck(s interfaces.StateInterface) error { +func (rgw *RgwServicePlacement) HospitalityCheck(ctx context.Context, s interfaces.StateInterface) error { return genericHospitalityCheck("rgw") } diff --git a/microceph/ceph/services_placement_test.go b/microceph/ceph/services_placement_test.go index e10e1b99..8833e3df 100644 --- a/microceph/ceph/services_placement_test.go +++ b/microceph/ceph/services_placement_test.go @@ -12,6 +12,7 @@ import ( "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/microceph/microceph/mocks" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -45,20 +46,20 @@ func addSnapServiceActiveExpectations(r *mocks.Runner, service string, retStr st func addPlacementServiceInitFailExpectation(sp *mocks.PlacementIntf, s *mocks.StateInterface, payload types.EnableService) { sp.On("PopulateParams", s, payload.Payload).Return(nil).Once() - sp.On("HospitalityCheck", s).Return(nil).Once() + sp.On("HospitalityCheck", mock.Anything, s).Return(nil).Once() sp.On("ServiceInit", s).Return(fmt.Errorf("ERROR")).Once() } func addPostPlacementCheckFailExpectation(sp *mocks.PlacementIntf, s *mocks.StateInterface, payload types.EnableService) { sp.On("PopulateParams", s, payload.Payload).Return(nil).Once() - sp.On("HospitalityCheck", s).Return(nil).Once() + sp.On("HospitalityCheck", mock.Anything, s).Return(nil).Once() sp.On("ServiceInit", s).Return(nil).Once() sp.On("PostPlacementCheck", s).Return(fmt.Errorf("ERROR")).Once() } func addDbUpdateFailExpectation(sp *mocks.PlacementIntf, s *mocks.StateInterface, payload types.EnableService) { sp.On("PopulateParams", s, payload.Payload).Return(nil).Once() - sp.On("HospitalityCheck", s).Return(nil).Once() + sp.On("HospitalityCheck", mock.Anything, s).Return(nil).Once() sp.On("ServiceInit", s).Return(nil).Once() sp.On("PostPlacementCheck", s).Return(nil).Once() sp.On("DbUpdate", s).Return(fmt.Errorf("ERROR")).Once() diff --git a/microceph/mocks/PlacementIntf.go b/microceph/mocks/PlacementIntf.go index a180fa27..e621e4fa 100644 --- a/microceph/mocks/PlacementIntf.go +++ b/microceph/mocks/PlacementIntf.go @@ -28,13 +28,13 @@ func (_m *PlacementIntf) DbUpdate(ctx context.Context, _a0 interfaces.StateInter return r0 } -// HospitalityCheck provides a mock function with given fields: _a0 -func (_m *PlacementIntf) HospitalityCheck(_a0 interfaces.StateInterface) error { - ret := _m.Called(_a0) +// HospitalityCheck provides a mock function with given fields: ctx, _a0 +func (_m *PlacementIntf) HospitalityCheck(ctx context.Context, _a0 interfaces.StateInterface) error { + ret := _m.Called(ctx, _a0) var r0 error - if rf, ok := ret.Get(0).(func(interfaces.StateInterface) error); ok { - r0 = rf(_a0) + if rf, ok := ret.Get(0).(func(context.Context, interfaces.StateInterface) error); ok { + r0 = rf(ctx, _a0) } else { r0 = ret.Error(0) } From 7668c22b8f6c947450a99ab6d688ad47cf56e064 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 16:47:33 +0530 Subject: [PATCH 09/31] daemon: add smb cephx keyring management Add per-node daemon key handling mirroring cephadm's SMBService key model: mon caps allow reads plus this cluster's smb/config/ config-key prefix; osd caps derive from the spec's RADOS URIs with enhanced rwx on the smb pool's cluster.meta. object prefix (CTDB reclock). Caps converge on re-apply via auth caps rather than get-or-create. The include_ceph_users data-path keys (owned by mgr/smb) are fetched into standard-named keyring files under the conf dir, where the default /etc/ceph search path finds them; entity names are validated before touching disk. Wiring into the enable flow lands with M3 lifecycle. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/smb_keyring.go | 196 +++++++++++++++++++++++++++++ microceph/ceph/smb_keyring_test.go | 146 +++++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 microceph/ceph/smb_keyring.go create mode 100644 microceph/ceph/smb_keyring_test.go diff --git a/microceph/ceph/smb_keyring.go b/microceph/ceph/smb_keyring.go new file mode 100644 index 00000000..090c73f7 --- /dev/null +++ b/microceph/ceph/smb_keyring.go @@ -0,0 +1,196 @@ +package ceph + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/logger" +) + +// smbRADOSPool is the pool mgr/smb keeps its objects in; URIs pointing at +// it get the enhanced caps CTDB needs to lock the cluster meta object. +const smbRADOSPool = ".smb" + +// smbEntityRegex guards cephx entity names used to build keyring file +// paths; the spec is external input. +var smbEntityRegex = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) + +// SMBDaemonEntity returns the per-node cephx entity for an smb cluster. +func SMBDaemonEntity(clusterID, hostname string) string { + return fmt.Sprintf("client.smb.%s.%s", clusterID, hostname) +} + +// smbPoolCapsFromURI mirrors cephadm's SMBService._pool_caps_from_uri: +// read access for foreign pools, and for the smb pool additionally rwx on +// the cluster.meta. object prefix (the x perm locks the CTDB reclock +// object). +func smbPoolCapsFromURI(uri string) []string { + if !strings.HasPrefix(uri, "rados://") { + logger.Debugf("ignoring unexpected uri scheme: %s", uri) + return nil + } + + part := strings.TrimRight(strings.TrimPrefix(uri, "rados://"), "/") + pool, rest, found := strings.Cut(part, "/") + if !found { + logger.Debugf("ignoring poolless uri: %s", uri) + return nil + } + + namespace := "" + if strings.Contains(rest, "/") { + namespace, _, _ = strings.Cut(rest, "/") + } + + if pool != smbRADOSPool { + return []string{fmt.Sprintf("allow r pool=%s", pool)} + } + + return []string{ + fmt.Sprintf("allow r pool=%s", pool), + fmt.Sprintf("allow rwx pool=%s namespace=%s object_prefix cluster.meta.", pool, namespace), + } +} + +// smbOSDCaps unions the pool caps over every RADOS URI in the spec, +// deduplicated and sorted for stable comparisons. +func smbOSDCaps(spec *types.SMBSpec) string { + uris := []string{spec.ConfigURI} + uris = append(uris, spec.UserSources...) + + capSet := map[string]bool{} + for _, uri := range uris { + for _, cap := range smbPoolCapsFromURI(uri) { + capSet[cap] = true + } + } + + caps := make([]string, 0, len(capSet)) + for cap := range capSet { + caps = append(caps, cap) + } + sort.Strings(caps) + + return strings.Join(caps, ", ") +} + +// smbMonCaps allows mon reads plus fetching this smb cluster's config +// keys from the mon config-key store (mirrors cephadm). +func smbMonCaps(clusterID string) string { + return fmt.Sprintf(`allow r, allow command "config-key get" with "key" prefix "smb/config/%s/"`, clusterID) +} + +// smbKeyringPath returns the standard-named keyring file for an entity, +// discovered by the default /etc/ceph search path (bound to confDir in +// the snap). +func smbKeyringPath(confDir, entity string) string { + return filepath.Join(confDir, fmt.Sprintf("ceph.%s.keyring", entity)) +} + +// fetchSMBKeyring writes the entity's keyring file under confDir, +// atomically and readable by root only. +func fetchSMBKeyring(confDir, entity string) error { + path := smbKeyringPath(confDir, entity) + tmpFile := path + ".tmp" + + _, err := cephRun("auth", "get", entity, "-o", tmpFile) + if err != nil { + os.Remove(tmpFile) + return fmt.Errorf("failed to fetch keyring for '%s': %w", entity, err) + } + + err = os.Chmod(tmpFile, 0600) + if err != nil { + os.Remove(tmpFile) + return err + } + + err = os.Rename(tmpFile, path) + if err != nil { + os.Remove(tmpFile) + return err + } + + return nil +} + +// checkSMBEntities validates every entity name the spec makes us touch on +// disk or pass to ceph. +func checkSMBEntities(entities []string) error { + for _, entity := range entities { + if !smbEntityRegex.MatchString(entity) { + return fmt.Errorf("'%s' is not a valid cephx entity name", entity) + } + } + return nil +} + +// EnsureSMBKeyrings creates or updates the per-node smb daemon key (caps +// converge on re-apply) and fetches the spec's include_ceph_users keys, +// writing standard-named keyring files under confDir. +func EnsureSMBKeyrings(spec *types.SMBSpec, hostname, confDir string) error { + entity := SMBDaemonEntity(spec.ClusterID, hostname) + + err := checkSMBEntities(append([]string{entity}, spec.IncludeCephUsers...)) + if err != nil { + return err + } + + // get-or-create without caps, then converge caps separately: a plain + // get-or-create fails when the entity exists with different caps, and + // re-applies may legitimately change the URI-derived caps. + _, err = cephRun("auth", "get-or-create", entity) + if err != nil { + return fmt.Errorf("failed to ensure cephx entity '%s': %w", entity, err) + } + + _, err = cephRun("auth", "caps", entity, "mon", smbMonCaps(spec.ClusterID), "osd", smbOSDCaps(spec)) + if err != nil { + return fmt.Errorf("failed to set caps for '%s': %w", entity, err) + } + + err = fetchSMBKeyring(confDir, entity) + if err != nil { + return err + } + + for _, user := range spec.IncludeCephUsers { + err = fetchSMBKeyring(confDir, user) + if err != nil { + return err + } + } + + return nil +} + +// RemoveSMBKeyrings deletes the per-node daemon key and every keyring +// file this node fetched for the cluster. The include_ceph_users +// entities themselves belong to mgr/smb and are left in place. +func RemoveSMBKeyrings(spec *types.SMBSpec, hostname, confDir string) error { + entity := SMBDaemonEntity(spec.ClusterID, hostname) + + err := checkSMBEntities(append([]string{entity}, spec.IncludeCephUsers...)) + if err != nil { + return err + } + + _, err = cephRun("auth", "del", entity) + if err != nil { + return fmt.Errorf("failed to delete cephx entity '%s': %w", entity, err) + } + + for _, name := range append([]string{entity}, spec.IncludeCephUsers...) { + err = os.Remove(smbKeyringPath(confDir, name)) + if err != nil && !os.IsNotExist(err) { + return err + } + } + + return nil +} diff --git a/microceph/ceph/smb_keyring_test.go b/microceph/ceph/smb_keyring_test.go new file mode 100644 index 00000000..36bc12a5 --- /dev/null +++ b/microceph/ceph/smb_keyring_test.go @@ -0,0 +1,146 @@ +package ceph + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microceph/microceph/mocks" + "github.com/canonical/microceph/microceph/tests" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type smbKeyringSuite struct { + tests.BaseSuite +} + +func TestSMBKeyringSuite(t *testing.T) { + suite.Run(t, new(smbKeyringSuite)) +} + +func (s *smbKeyringSuite) spec() *types.SMBSpec { + var spec types.SMBSpec + err := json.Unmarshal([]byte(validSMBPayload), &spec) + assert.NoError(s.T(), err) + return &spec +} + +func (s *smbKeyringSuite) TestSMBDaemonEntity() { + assert.Equal(s.T(), "client.smb.dev.smbdev-1", SMBDaemonEntity("dev", "smbdev-1")) +} + +func (s *smbKeyringSuite) TestPoolCapsSMBPool() { + caps := smbPoolCapsFromURI("rados://.smb/dev/scc.dev.json") + assert.Equal(s.T(), []string{ + "allow r pool=.smb", + "allow rwx pool=.smb namespace=dev object_prefix cluster.meta.", + }, caps) +} + +func (s *smbKeyringSuite) TestPoolCapsSMBPoolNoNamespace() { + caps := smbPoolCapsFromURI("rados://.smb/scc.json") + assert.Equal(s.T(), []string{ + "allow r pool=.smb", + "allow rwx pool=.smb namespace= object_prefix cluster.meta.", + }, caps) +} + +func (s *smbKeyringSuite) TestPoolCapsForeignPool() { + assert.Equal(s.T(), []string{"allow r pool=users"}, smbPoolCapsFromURI("rados://users/x/y.json")) +} + +func (s *smbKeyringSuite) TestPoolCapsNonRADOS() { + assert.Empty(s.T(), smbPoolCapsFromURI("http://example.com/x.json")) +} + +func (s *smbKeyringSuite) TestOSDCaps() { + spec := s.spec() + spec.UserSources = append(spec.UserSources, "rados://users/x.json") + + caps := smbOSDCaps(spec) + assert.Equal(s.T(), + "allow r pool=.smb, allow r pool=users, allow rwx pool=.smb namespace=dev object_prefix cluster.meta.", + caps) +} + +func (s *smbKeyringSuite) TestMonCaps() { + assert.Equal(s.T(), + `allow r, allow command "config-key get" with "key" prefix "smb/config/dev/"`, + smbMonCaps("dev")) +} + +// writeKeyringOnGet makes an "auth get ... -o " expectation create the +// output file, as the real ceph CLI would. +func writeKeyringOnGet(s *smbKeyringSuite, r *mocks.Runner, entity string) { + r.On("RunCommand", "ceph", "auth", "get", entity, "-o", mock.Anything).Run(func(args mock.Arguments) { + path := args.Get(5).(string) + err := os.WriteFile(path, []byte("["+entity+"]\n\tkey = secret\n"), 0600) + assert.NoError(s.T(), err) + }).Return("", nil).Once() +} + +func (s *smbKeyringSuite) TestEnsureSMBKeyrings() { + confDir := s.T().TempDir() + spec := s.spec() + spec.IncludeCephUsers = []string{"client.data1"} + + entity := "client.smb.dev.host1" + r := mocks.NewRunner(s.T()) + r.On("RunCommand", "ceph", "auth", "get-or-create", entity).Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "caps", entity, + "mon", smbMonCaps("dev"), "osd", smbOSDCaps(spec)).Return("", nil).Once() + writeKeyringOnGet(s, r, entity) + writeKeyringOnGet(s, r, "client.data1") + common.ProcessExec = r + + err := EnsureSMBKeyrings(spec, "host1", confDir) + assert.NoError(s.T(), err) + + for _, name := range []string{"ceph.client.smb.dev.host1.keyring", "ceph.client.data1.keyring"} { + info, err := os.Stat(filepath.Join(confDir, name)) + assert.NoError(s.T(), err, name) + assert.Equal(s.T(), os.FileMode(0600), info.Mode().Perm(), name) + } +} + +func (s *smbKeyringSuite) TestEnsureSMBKeyringsRejectsBadEntity() { + confDir := s.T().TempDir() + spec := s.spec() + spec.IncludeCephUsers = []string{"client.foo/../../etc"} + + // No Runner expectations: validation must fail before any ceph call. + common.ProcessExec = mocks.NewRunner(s.T()) + + err := EnsureSMBKeyrings(spec, "host1", confDir) + assert.ErrorContains(s.T(), err, "not a valid cephx entity") +} + +func (s *smbKeyringSuite) TestRemoveSMBKeyrings() { + confDir := s.T().TempDir() + spec := s.spec() + spec.IncludeCephUsers = []string{"client.data1"} + + for _, name := range []string{"ceph.client.smb.dev.host1.keyring", "ceph.client.data1.keyring"} { + err := os.WriteFile(filepath.Join(confDir, name), []byte("k"), 0600) + assert.NoError(s.T(), err) + } + + r := mocks.NewRunner(s.T()) + // The daemon key is deleted from ceph; include_ceph_users keys belong + // to mgr/smb and only their fetched files are removed. + r.On("RunCommand", "ceph", "auth", "del", "client.smb.dev.host1").Return("", nil).Once() + common.ProcessExec = r + + err := RemoveSMBKeyrings(spec, "host1", confDir) + assert.NoError(s.T(), err) + + entries, err := os.ReadDir(confDir) + assert.NoError(s.T(), err) + assert.Empty(s.T(), entries) +} From bf10e6394f10ccc82e42559933fa53adb585b3e1 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 17:06:20 +0530 Subject: [PATCH 10/31] daemon: add smb config rendering Render the per-node SMB config set: fetch the mgr/smb sambacc document from config_uri, translate it for this backend (classic ceph vfs module, since the snap's samba 4.19 has no ceph_new; inject a microceph globals section carrying the proven snap paths, netbios name and CTDB wiring) and emit smb.conf through the bundled sambacc print-config (pure-python path). CTDB files render natively: ctdb.conf points the 4.19 'cluster lock' at the bundled rados mutex helper with a per-cluster object in the lock pool's default namespace (the 4.19 helper has no namespace support, so the namespaced cluster_lock_uri object stays with mgr/smb untouched), nodes carries caller-ordered private IPs, public_addresses resolves each VIP's interface on-node. Clustered specs additionally get an osd cap for the reclock prefix. All writes are atomic tmp+rename; golden-file tests cover each rendered file. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/smb_config.go | 350 ++++++++++++++++++ microceph/ceph/smb_config_test.go | 182 +++++++++ microceph/ceph/smb_keyring.go | 15 +- microceph/ceph/smb_keyring_test.go | 12 +- microceph/ceph/testdata/smb/config.smb.json | 43 +++ microceph/ceph/testdata/smb/ctdb.conf.golden | 5 + microceph/ceph/testdata/smb/nodes.golden | 3 + .../ceph/testdata/smb/public_addresses.golden | 2 + .../ceph/testdata/smb/translated.json.golden | 65 ++++ 9 files changed, 675 insertions(+), 2 deletions(-) create mode 100644 microceph/ceph/smb_config.go create mode 100644 microceph/ceph/smb_config_test.go create mode 100644 microceph/ceph/testdata/smb/config.smb.json create mode 100644 microceph/ceph/testdata/smb/ctdb.conf.golden create mode 100644 microceph/ceph/testdata/smb/nodes.golden create mode 100644 microceph/ceph/testdata/smb/public_addresses.golden create mode 100644 microceph/ceph/testdata/smb/translated.json.golden diff --git a/microceph/ceph/smb_config.go b/microceph/ceph/smb_config.go new file mode 100644 index 00000000..3ec6ef61 --- /dev/null +++ b/microceph/ceph/smb_config.go @@ -0,0 +1,350 @@ +package ceph + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microceph/microceph/constants" +) + +// SMBPaths carries the snap path roots the renderers embed in configs. +type SMBPaths struct { + Conf string + Run string + Data string + Log string + Snap string +} + +// SMBRenderParams carries per-node, per-cluster rendering inputs. +type SMBRenderParams struct { + ClusterID string + // Entity is the node's daemon cephx entity (client. prefixed). + Entity string + Clustered bool + Paths SMBPaths +} + +// NewSMBRenderParams builds render params from the snap environment. +func NewSMBRenderParams(clusterID, hostname string, clustered bool) SMBRenderParams { + pathConsts := constants.GetPathConst() + return SMBRenderParams{ + ClusterID: clusterID, + Entity: SMBDaemonEntity(clusterID, hostname), + Clustered: clustered, + Paths: SMBPaths{ + Conf: pathConsts.ConfPath, + Run: pathConsts.RunPath, + Data: pathConsts.DataPath, + Log: pathConsts.LogPath, + Snap: strings.TrimRight(pathConsts.SnapPath, "/"), + }, + } +} + +// parseRADOSURI splits rados:///[/]. +func parseRADOSURI(uri string) (string, string, string, error) { + if !strings.HasPrefix(uri, "rados://") { + return "", "", "", fmt.Errorf("'%s' is not a rados:// URI", uri) + } + + part := strings.TrimRight(strings.TrimPrefix(uri, "rados://"), "/") + fields := strings.Split(part, "/") + switch len(fields) { + case 2: + return fields[0], "", fields[1], nil + case 3: + return fields[0], fields[1], fields[2], nil + } + return "", "", "", fmt.Errorf("cannot parse rados URI '%s'", uri) +} + +// microcephGlobalOptions returns the snap-specific smb.conf globals the +// generated config must carry: the known-working paths from the live +// experiment, plus clustering wiring when CTDB is on. +func microcephGlobalOptions(p SMBRenderParams) map[string]any { + dataDir := filepath.Join(p.Paths.Data, "samba", p.ClusterID) + runDir := filepath.Join(p.Paths.Run, "samba", p.ClusterID) + + options := map[string]any{ + "security": "user", + "netbios name": strings.ToUpper(p.ClusterID), + "private dir": filepath.Join(dataDir, "private"), + "lock directory": filepath.Join(dataDir, "lock"), + "state directory": filepath.Join(dataDir, "state"), + "cache directory": filepath.Join(dataDir, "cache"), + "pid directory": runDir, + "ncalrpc dir": filepath.Join(runDir, "ncalrpc"), + "log file": filepath.Join(p.Paths.Log, "samba", p.ClusterID, "log.%m"), + } + + if p.Clustered { + options["clustering"] = "yes" + options["ctdbd socket"] = filepath.Join(p.Paths.Run, "ctdb", "ctdbd.socket") + } + + return options +} + +// translateShareOptions rewrites ceph_new vfs options to the classic ceph +// module: the snap's samba 4.19 ships only ceph.so, while mgr/smb's +// default share provider emits the proxied ceph_new form. Drop this when +// the snap moves to samba >= 4.20. +func translateShareOptions(options map[string]any) { + if vfs, ok := options["vfs objects"].(string); ok { + fields := strings.Fields(vfs) + for i, field := range fields { + if field == "ceph_new" { + fields[i] = "ceph" + } + } + options["vfs objects"] = strings.Join(fields, " ") + } + + delete(options, "ceph_new:proxy") + for key, value := range options { + if strings.HasPrefix(key, "ceph_new:") { + options["ceph:"+strings.TrimPrefix(key, "ceph_new:")] = value + delete(options, key) + } + } +} + +// TranslateSMBConfig rewrites the mgr/smb sambacc config document for +// this backend and reports whether the instance is CTDB-clustered. The +// output feeds sambacc print-config. +func TranslateSMBConfig(raw []byte, p SMBRenderParams) ([]byte, bool, error) { + var doc map[string]any + err := json.Unmarshal(raw, &doc) + if err != nil { + return nil, false, fmt.Errorf("cannot parse smb config document: %w", err) + } + + version, _ := doc["samba-container-config"].(string) + if version != "v0" { + return nil, false, fmt.Errorf("unsupported samba-container-config version '%s'", version) + } + + configs, _ := doc["configs"].(map[string]any) + instance, _ := configs[p.ClusterID].(map[string]any) + if instance == nil { + return nil, false, fmt.Errorf("config document has no instance for cluster '%s'", p.ClusterID) + } + + clustered := false + if features, ok := instance["instance_features"].([]any); ok { + for _, feature := range features { + if feature == "ctdb" { + clustered = true + } + } + } + p.Clustered = clustered + + // Inject the microceph globals section and reference it last so its + // options win over the mgr-emitted ones. + globals, ok := doc["globals"].(map[string]any) + if !ok { + globals = map[string]any{} + doc["globals"] = globals + } + globals["microceph"] = map[string]any{"options": microcephGlobalOptions(p)} + + globalRefs, _ := instance["globals"].([]any) + instance["globals"] = append(globalRefs, "microceph") + + if shares, ok := doc["shares"].(map[string]any); ok { + for _, share := range shares { + shareMap, ok := share.(map[string]any) + if !ok { + continue + } + if options, ok := shareMap["options"].(map[string]any); ok { + translateShareOptions(options) + } + } + } + + translated, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return nil, false, err + } + + return append(translated, '\n'), clustered, nil +} + +// smbReclockObject is the CTDB cluster lock object prefix; the samba 4.19 +// rados mutex helper has no namespace support, so the lock lives in the +// lock pool's default namespace under a per-cluster name instead of at +// the (namespaced) cluster_lock_uri object, which stays owned by mgr/smb. +const smbReclockObject = "microceph.reclock." + +// RenderCTDBConf renders ctdb.conf with the cluster lock held via the +// bundled rados mutex helper (4.19 syntax: 'cluster lock'). +func RenderCTDBConf(p SMBRenderParams, lockURI string) (string, error) { + pool, _, _, err := parseRADOSURI(lockURI) + if err != nil { + return "", fmt.Errorf("cannot derive cluster lock from cluster_lock_uri: %w", err) + } + + helper := filepath.Join(p.Paths.Snap, "libexec", "ctdb", "ctdb_mutex_ceph_rados_helper") + object := smbReclockObject + p.ClusterID + + return fmt.Sprintf(`[logging] + log level = NOTICE + +[cluster] + cluster lock = !%s ceph %s %s %s +`, helper, p.Entity, pool, object), nil +} + +// RenderCTDBNodes renders the nodes file: one private address per line. +// Callers must pass a stable, append-only ordering (CTDB node numbers +// are line indices); ordering by DB row id provides that. +func RenderCTDBNodes(ips []string) string { + var b strings.Builder + for _, ip := range ips { + b.WriteString(ip) + b.WriteByte('\n') + } + return b.String() +} + +// atomicWriteFile writes via a .tmp sibling and rename so a failed write +// cannot leave partial config state on disk. +func atomicWriteFile(path string, data []byte, mode os.FileMode) error { + tmpFile := path + ".tmp" + err := os.WriteFile(tmpFile, data, mode) + if err != nil { + return err + } + err = os.Rename(tmpFile, path) + if err != nil { + os.Remove(tmpFile) + return err + } + return nil +} + +// fetchSMBConfigObject reads the object behind a rados:// URI. +func fetchSMBConfigObject(uri string) ([]byte, error) { + pool, namespace, object, err := parseRADOSURI(uri) + if err != nil { + return nil, err + } + + args := []string{"get", "--pool", pool} + if namespace != "" { + args = append(args, "-N", namespace) + } + args = append(args, object, "-") + + out, err := radosRun(args...) + if err != nil { + return nil, fmt.Errorf("failed to fetch '%s': %w", uri, err) + } + return []byte(out), nil +} + +// renderSMBConfText runs the bundled sambacc to turn a translated config +// document into smb.conf text (pure-python path, no samba binaries). +func renderSMBConfText(configPath, identity string) (string, error) { + out, err := common.ProcessExec.RunCommand("python3", + "-m", "sambacc.commands.main", + "--config", configPath, + "--identity", identity, + "print-config") + if err != nil { + return "", fmt.Errorf("sambacc print-config failed: %w", err) + } + return out, nil +} + +// WriteSMBNodeConfigs fetches the cluster's sambacc config document, +// renders smb.conf through sambacc, and writes the CTDB config set for +// this node. nodeIPs must already carry the stable CTDB ordering. +func WriteSMBNodeConfigs(spec *types.SMBSpec, p SMBRenderParams, nodeIPs []string, resolveIface func(cidr string) (string, error)) error { + raw, err := fetchSMBConfigObject(spec.ConfigURI) + if err != nil { + return err + } + + translated, clustered, err := TranslateSMBConfig(raw, p) + if err != nil { + return err + } + p.Clustered = clustered + + sambaDir := filepath.Join(p.Paths.Conf, "samba") + err = os.MkdirAll(sambaDir, 0755) + if err != nil { + return err + } + + configPath := filepath.Join(sambaDir, "config.json") + err = atomicWriteFile(configPath, translated, 0644) + if err != nil { + return err + } + + smbConf, err := renderSMBConfText(configPath, p.ClusterID) + if err != nil { + return err + } + err = atomicWriteFile(filepath.Join(sambaDir, "smb.conf"), []byte(smbConf), 0644) + if err != nil { + return err + } + + if !clustered { + return nil + } + + ctdbDir := filepath.Join(p.Paths.Conf, "ctdb") + err = os.MkdirAll(ctdbDir, 0755) + if err != nil { + return err + } + + ctdbConf, err := RenderCTDBConf(p, spec.ClusterLockURI) + if err != nil { + return err + } + err = atomicWriteFile(filepath.Join(ctdbDir, "ctdb.conf"), []byte(ctdbConf), 0644) + if err != nil { + return err + } + + err = atomicWriteFile(filepath.Join(ctdbDir, "nodes"), []byte(RenderCTDBNodes(nodeIPs)), 0644) + if err != nil { + return err + } + + publicAddresses, err := RenderCTDBPublicAddresses(spec.ClusterPublicAddrs, resolveIface) + if err != nil { + return err + } + return atomicWriteFile(filepath.Join(ctdbDir, "public_addresses"), []byte(publicAddresses), 0644) +} + +// RenderCTDBPublicAddresses renders public_addresses: ' ' +// per line, with the interface resolved on this node for each VIP. +func RenderCTDBPublicAddresses(addrs []types.SMBPublicAddrSpec, resolveIface func(cidr string) (string, error)) (string, error) { + var b strings.Builder + for _, addr := range addrs { + iface, err := resolveIface(addr.Address) + if err != nil { + return "", fmt.Errorf("cannot resolve interface for public address '%s': %w", addr.Address, err) + } + b.WriteString(addr.Address) + b.WriteByte(' ') + b.WriteString(iface) + b.WriteByte('\n') + } + return b.String(), nil +} diff --git a/microceph/ceph/smb_config_test.go b/microceph/ceph/smb_config_test.go new file mode 100644 index 00000000..f542974f --- /dev/null +++ b/microceph/ceph/smb_config_test.go @@ -0,0 +1,182 @@ +package ceph + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microceph/microceph/mocks" + "github.com/canonical/microceph/microceph/tests" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// Golden files live in testdata/smb. Regenerate by running the tests with +// UPDATE_GOLDEN=1 and reviewing the diff. +func (s *smbConfigSuite) golden(name string, got []byte) { + path := filepath.Join("testdata", "smb", name) + if os.Getenv("UPDATE_GOLDEN") == "1" { + assert.NoError(s.T(), os.WriteFile(path, got, 0644)) + return + } + + want, err := os.ReadFile(path) + assert.NoError(s.T(), err) + assert.Equal(s.T(), string(want), string(got), name) +} + +type smbConfigSuite struct { + tests.BaseSuite +} + +func TestSMBConfigSuite(t *testing.T) { + suite.Run(t, new(smbConfigSuite)) +} + +func testRenderParams() SMBRenderParams { + return SMBRenderParams{ + ClusterID: "dev", + Entity: "client.smb.dev.host1", + Clustered: true, + Paths: SMBPaths{ + Conf: "/var/snap/microceph/current/conf", + Run: "/var/snap/microceph/current/run", + Data: "/var/snap/microceph/common/data", + Log: "/var/snap/microceph/common/logs", + Snap: "/snap/microceph/current", + }, + } +} + +func (s *smbConfigSuite) TestParseRADOSURI() { + pool, ns, object, err := parseRADOSURI("rados://.smb/dev/cluster.meta.lock") + assert.NoError(s.T(), err) + assert.Equal(s.T(), ".smb", pool) + assert.Equal(s.T(), "dev", ns) + assert.Equal(s.T(), "cluster.meta.lock", object) + + pool, ns, object, err = parseRADOSURI("rados://pool/obj.json") + assert.NoError(s.T(), err) + assert.Equal(s.T(), "pool", pool) + assert.Empty(s.T(), ns) + assert.Equal(s.T(), "obj.json", object) + + _, _, _, err = parseRADOSURI("http://x/y") + assert.Error(s.T(), err) + + _, _, _, err = parseRADOSURI("rados://poolonly") + assert.Error(s.T(), err) +} + +func (s *smbConfigSuite) TestTranslateSMBConfig() { + raw, err := os.ReadFile(filepath.Join("testdata", "smb", "config.smb.json")) + assert.NoError(s.T(), err) + + translated, clustered, err := TranslateSMBConfig(raw, testRenderParams()) + assert.NoError(s.T(), err) + assert.True(s.T(), clustered) + + s.golden("translated.json.golden", translated) +} + +func (s *smbConfigSuite) TestTranslateSMBConfigRejectsWrongVersion() { + _, _, err := TranslateSMBConfig([]byte(`{"samba-container-config": "v9"}`), testRenderParams()) + assert.ErrorContains(s.T(), err, "samba-container-config") +} + +func (s *smbConfigSuite) TestTranslateSMBConfigRejectsMissingIdentity() { + _, _, err := TranslateSMBConfig([]byte(`{"samba-container-config": "v0", "configs": {"other": {}}}`), testRenderParams()) + assert.ErrorContains(s.T(), err, "dev") +} + +func (s *smbConfigSuite) TestRenderCTDBConf() { + got, err := RenderCTDBConf(testRenderParams(), "rados://.smb/dev/cluster.meta.lock") + assert.NoError(s.T(), err) + s.golden("ctdb.conf.golden", []byte(got)) +} + +func (s *smbConfigSuite) TestRenderCTDBNodes() { + got := RenderCTDBNodes([]string{"10.0.0.1", "10.0.0.2", "10.0.0.3"}) + s.golden("nodes.golden", []byte(got)) +} + +func (s *smbConfigSuite) TestRenderCTDBPublicAddresses() { + addrs := []types.SMBPublicAddrSpec{ + {Address: "10.105.154.245/24"}, + {Address: "10.105.155.1/24", Destination: types.SMBDestination{"10.105.155.0/24"}}, + } + + resolver := func(cidr string) (string, error) { return "enp5s0", nil } + + got, err := RenderCTDBPublicAddresses(addrs, resolver) + assert.NoError(s.T(), err) + s.golden("public_addresses.golden", []byte(got)) +} + +func (s *smbConfigSuite) TestRenderCTDBPublicAddressesResolverError() { + addrs := []types.SMBPublicAddrSpec{{Address: "10.0.0.1/24"}} + resolver := func(cidr string) (string, error) { return "", assert.AnError } + + _, err := RenderCTDBPublicAddresses(addrs, resolver) + assert.Error(s.T(), err) +} + +func (s *smbConfigSuite) TestWriteSMBNodeConfigs() { + confDir := s.T().TempDir() + p := testRenderParams() + p.Paths.Conf = confDir + + raw, err := os.ReadFile(filepath.Join("testdata", "smb", "config.smb.json")) + assert.NoError(s.T(), err) + + var spec types.SMBSpec + assert.NoError(s.T(), json.Unmarshal([]byte(validSMBPayload), &spec)) + + r := mocks.NewRunner(s.T()) + r.On("RunCommand", "rados", "get", "--pool", ".smb", "-N", "dev", "scc.dev.json", "-"). + Return(string(raw), nil).Once() + r.On("RunCommand", "python3", "-m", "sambacc.commands.main", + "--config", filepath.Join(confDir, "samba", "config.json"), + "--identity", "dev", "print-config"). + Return("[global]\n\tfake = conf\n", nil).Once() + common.ProcessExec = r + + resolver := func(cidr string) (string, error) { return "enp5s0", nil } + err = WriteSMBNodeConfigs(&spec, p, []string{"10.0.0.1", "10.0.0.2"}, resolver) + assert.NoError(s.T(), err) + + for _, f := range []struct { + path string + want string + }{ + {"samba/smb.conf", "[global]\n\tfake = conf\n"}, + {"ctdb/nodes", "10.0.0.1\n10.0.0.2\n"}, + {"ctdb/public_addresses", "10.105.154.245/24 enp5s0\n"}, + } { + got, err := os.ReadFile(filepath.Join(confDir, f.path)) + assert.NoError(s.T(), err, f.path) + assert.Equal(s.T(), f.want, string(got), f.path) + + info, err := os.Stat(filepath.Join(confDir, f.path)) + assert.NoError(s.T(), err) + assert.Equal(s.T(), os.FileMode(0644), info.Mode().Perm(), f.path) + } + + ctdbConf, err := os.ReadFile(filepath.Join(confDir, "ctdb", "ctdb.conf")) + assert.NoError(s.T(), err) + assert.Contains(s.T(), string(ctdbConf), "cluster lock = !") + assert.Contains(s.T(), string(ctdbConf), "microceph.reclock.dev") + + // No stray .tmp files left behind. + for _, dir := range []string{"samba", "ctdb"} { + entries, err := os.ReadDir(filepath.Join(confDir, dir)) + assert.NoError(s.T(), err) + for _, entry := range entries { + assert.NotContains(s.T(), entry.Name(), ".tmp") + } + } +} diff --git a/microceph/ceph/smb_keyring.go b/microceph/ceph/smb_keyring.go index 090c73f7..a274f14e 100644 --- a/microceph/ceph/smb_keyring.go +++ b/microceph/ceph/smb_keyring.go @@ -58,7 +58,10 @@ func smbPoolCapsFromURI(uri string) []string { } // smbOSDCaps unions the pool caps over every RADOS URI in the spec, -// deduplicated and sorted for stable comparisons. +// deduplicated and sorted for stable comparisons. Clustered specs also +// get access to the per-cluster CTDB reclock object (see RenderCTDBConf: +// the 4.19 rados mutex helper is namespace-blind, so the lock lives in +// the lock pool's default namespace under our own prefix). func smbOSDCaps(spec *types.SMBSpec) string { uris := []string{spec.ConfigURI} uris = append(uris, spec.UserSources...) @@ -70,6 +73,16 @@ func smbOSDCaps(spec *types.SMBSpec) string { } } + for _, feature := range spec.Features { + if feature != "clustered" { + continue + } + lockPool, _, _, err := parseRADOSURI(spec.ClusterLockURI) + if err == nil { + capSet[fmt.Sprintf("allow rwx pool=%s object_prefix %s", lockPool, smbReclockObject)] = true + } + } + caps := make([]string, 0, len(capSet)) for cap := range capSet { caps = append(caps, cap) diff --git a/microceph/ceph/smb_keyring_test.go b/microceph/ceph/smb_keyring_test.go index 36bc12a5..0f053192 100644 --- a/microceph/ceph/smb_keyring_test.go +++ b/microceph/ceph/smb_keyring_test.go @@ -65,10 +65,20 @@ func (s *smbKeyringSuite) TestOSDCaps() { caps := smbOSDCaps(spec) assert.Equal(s.T(), - "allow r pool=.smb, allow r pool=users, allow rwx pool=.smb namespace=dev object_prefix cluster.meta.", + "allow r pool=.smb, allow r pool=users, "+ + "allow rwx pool=.smb namespace=dev object_prefix cluster.meta., "+ + "allow rwx pool=.smb object_prefix microceph.reclock.", caps) } +func (s *smbKeyringSuite) TestOSDCapsUnclustered() { + spec := s.spec() + spec.Features = nil + + caps := smbOSDCaps(spec) + assert.NotContains(s.T(), caps, "microceph.reclock.") +} + func (s *smbKeyringSuite) TestMonCaps() { assert.Equal(s.T(), `allow r, allow command "config-key get" with "key" prefix "smb/config/dev/"`, diff --git a/microceph/ceph/testdata/smb/config.smb.json b/microceph/ceph/testdata/smb/config.smb.json new file mode 100644 index 00000000..9334316f --- /dev/null +++ b/microceph/ceph/testdata/smb/config.smb.json @@ -0,0 +1,43 @@ +{ + "samba-container-config": "v0", + "configs": { + "dev": { + "instance_name": "dev", + "instance_features": ["ctdb"], + "globals": ["default", "dev"], + "shares": ["share1"] + } + }, + "globals": { + "default": { + "options": { + "load printers": "No", + "printing": "bsd", + "printcap name": "/dev/null", + "disable spoolss": "Yes", + "smbd profiling level": "on" + } + }, + "dev": { + "options": {} + } + }, + "shares": { + "share1": { + "options": { + "path": "/volumes/_nogroup/s1/uuid", + "vfs objects": "acl_xattr ceph_snapshots ceph_new", + "acl_xattr:security_acl_name": "user.NTACL", + "ceph_new:config_file": "/etc/ceph/ceph.conf", + "ceph_new:filesystem": "newfs", + "ceph_new:user_id": "smb.fs.cluster.dev", + "read only": "No", + "browseable": "Yes", + "kernel share modes": "no", + "x:ceph:id": "dev.share1", + "smbd profiling share": "yes", + "ceph_new:proxy": "yes" + } + } + } +} diff --git a/microceph/ceph/testdata/smb/ctdb.conf.golden b/microceph/ceph/testdata/smb/ctdb.conf.golden new file mode 100644 index 00000000..89c46a1f --- /dev/null +++ b/microceph/ceph/testdata/smb/ctdb.conf.golden @@ -0,0 +1,5 @@ +[logging] + log level = NOTICE + +[cluster] + cluster lock = !/snap/microceph/current/libexec/ctdb/ctdb_mutex_ceph_rados_helper ceph client.smb.dev.host1 .smb microceph.reclock.dev diff --git a/microceph/ceph/testdata/smb/nodes.golden b/microceph/ceph/testdata/smb/nodes.golden new file mode 100644 index 00000000..bf791c1f --- /dev/null +++ b/microceph/ceph/testdata/smb/nodes.golden @@ -0,0 +1,3 @@ +10.0.0.1 +10.0.0.2 +10.0.0.3 diff --git a/microceph/ceph/testdata/smb/public_addresses.golden b/microceph/ceph/testdata/smb/public_addresses.golden new file mode 100644 index 00000000..bc3d4c28 --- /dev/null +++ b/microceph/ceph/testdata/smb/public_addresses.golden @@ -0,0 +1,2 @@ +10.105.154.245/24 enp5s0 +10.105.155.1/24 enp5s0 diff --git a/microceph/ceph/testdata/smb/translated.json.golden b/microceph/ceph/testdata/smb/translated.json.golden new file mode 100644 index 00000000..c6f3d25e --- /dev/null +++ b/microceph/ceph/testdata/smb/translated.json.golden @@ -0,0 +1,65 @@ +{ + "configs": { + "dev": { + "globals": [ + "default", + "dev", + "microceph" + ], + "instance_features": [ + "ctdb" + ], + "instance_name": "dev", + "shares": [ + "share1" + ] + } + }, + "globals": { + "default": { + "options": { + "disable spoolss": "Yes", + "load printers": "No", + "printcap name": "/dev/null", + "printing": "bsd", + "smbd profiling level": "on" + } + }, + "dev": { + "options": {} + }, + "microceph": { + "options": { + "cache directory": "/var/snap/microceph/common/data/samba/dev/cache", + "clustering": "yes", + "ctdbd socket": "/var/snap/microceph/current/run/ctdb/ctdbd.socket", + "lock directory": "/var/snap/microceph/common/data/samba/dev/lock", + "log file": "/var/snap/microceph/common/logs/samba/dev/log.%m", + "ncalrpc dir": "/var/snap/microceph/current/run/samba/dev/ncalrpc", + "netbios name": "DEV", + "pid directory": "/var/snap/microceph/current/run/samba/dev", + "private dir": "/var/snap/microceph/common/data/samba/dev/private", + "security": "user", + "state directory": "/var/snap/microceph/common/data/samba/dev/state" + } + } + }, + "samba-container-config": "v0", + "shares": { + "share1": { + "options": { + "acl_xattr:security_acl_name": "user.NTACL", + "browseable": "Yes", + "ceph:config_file": "/etc/ceph/ceph.conf", + "ceph:filesystem": "newfs", + "ceph:user_id": "smb.fs.cluster.dev", + "kernel share modes": "no", + "path": "/volumes/_nogroup/s1/uuid", + "read only": "No", + "smbd profiling share": "yes", + "vfs objects": "acl_xattr ceph_snapshots ceph", + "x:ceph:id": "dev.share1" + } + } + } +} From f593b82e14d16871204d3d135b7dc53506914609 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 17:16:21 +0530 Subject: [PATCH 11/31] daemon: add smb node lifecycle Wire the smb placement flow end to end: ServiceInit now creates the per-cluster directories (root-owned; confined daemons have no dac_override), ensures keyrings, renders configs with row-id-ordered node IPs (CTDB node numbers are line indices, so ordering must be stable and append-only), populates CTDB_BASE (stock deb-enabled event script links, the snapctl 50.samba script, functions, script.options) and starts ctdbd; PostPlacementCheck verifies ctdbd stays up. Disable stops ctdbd/smbd, removes keyrings, configs, runtime and per-cluster data dirs, then the DB record. Membership or spec changes now trigger a serialized regenerate (re-render + ctdbd restart) across all desired members via a node-scoped POST, so every nodes file converges to the same content. VIP interfaces resolve against local subnets. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/api/services_smb.go | 21 ++ microceph/ceph/service_placement_smb.go | 11 +- microceph/ceph/smb.go | 43 ++- microceph/ceph/smb_config.go | 2 + microceph/ceph/smb_lifecycle.go | 316 +++++++++++++++++++ microceph/ceph/smb_lifecycle_test.go | 235 ++++++++++++++ microceph/ceph/smb_test.go | 19 +- microceph/client/services.go | 17 + microceph/client/wrap.go | 18 ++ microceph/database/grouped_service_extras.go | 33 ++ microceph/mocks/ClientInterface.go | 26 ++ microceph/mocks/GroupedServiceQueryIntf.go | 30 ++ 12 files changed, 758 insertions(+), 13 deletions(-) create mode 100644 microceph/ceph/smb_lifecycle.go create mode 100644 microceph/ceph/smb_lifecycle_test.go diff --git a/microceph/api/services_smb.go b/microceph/api/services_smb.go index d380df05..dcbd7d67 100644 --- a/microceph/api/services_smb.go +++ b/microceph/api/services_smb.go @@ -27,9 +27,30 @@ var smbServiceCmd = mcTypes.Endpoint{ var smbNodeServiceCmd = mcTypes.Endpoint{ Path: "services/smb/node", Put: mcTypes.EndpointAction{Handler: cmdEnableServicePut, ProxyTarget: true}, + Post: mcTypes.EndpointAction{Handler: cmdSMBNodePost, ProxyTarget: true}, Delete: mcTypes.EndpointAction{Handler: cmdSMBNodeDelete, ProxyTarget: true}, } +// cmdSMBNodePost regenerates this node's smb configs from the stored +// spec and restarts ctdbd. +func cmdSMBNodePost(s mcTypes.State, r *http.Request) mcTypes.Response { + var svc types.SMBService + + err := json.NewDecoder(r.Body).Decode(&svc) + if err != nil { + logger.Errorf("failed decoding smb node regenerate request: %v", err) + return mcTypes.InternalError(err) + } + + err = ceph.RegenerateSMBNode(r.Context(), interfaces.CephState{State: s}, svc.ClusterID) + if err != nil { + logger.Errorf("failed regenerating smb on node: %v", err) + return mcTypes.SmartError(err) + } + + return mcTypes.EmptySyncResponse +} + // cmdSMBServiceGet lists every smb cluster with its spec and placement. func cmdSMBServiceGet(s mcTypes.State, r *http.Request) mcTypes.Response { statuses, err := ceph.ListSMB(r.Context(), interfaces.CephState{State: s}) diff --git a/microceph/ceph/service_placement_smb.go b/microceph/ceph/service_placement_smb.go index 24f97ca2..11edacaf 100644 --- a/microceph/ceph/service_placement_smb.go +++ b/microceph/ceph/service_placement_smb.go @@ -120,16 +120,15 @@ func (smb *SMBServicePlacement) HospitalityCheck(ctx context.Context, s interfac return nil } -// ServiceInit is a no-op in Phase 1 milestone M2: config rendering and -// ctdbd lifecycle land with M3. +// ServiceInit brings the node into the smb cluster: keyrings, rendered +// configs, CTDB_BASE and the ctdbd service. func (smb *SMBServicePlacement) ServiceInit(ctx context.Context, s interfaces.StateInterface) error { - return nil + return EnableSMB(ctx, s, &smb.Spec) } -// PostPlacementCheck is a no-op until ServiceInit starts services (M3), -// after which it will verify ctdbd health. +// PostPlacementCheck verifies ctdbd stays up after placement. func (smb *SMBServicePlacement) PostPlacementCheck(s interfaces.StateInterface) error { - return nil + return genericPostPlacementCheck("ctdbd") } // DbUpdate records the group membership, storing the SMBSpec JSON verbatim diff --git a/microceph/ceph/smb.go b/microceph/ceph/smb.go index 89fb6319..c274c2fc 100644 --- a/microceph/ceph/smb.go +++ b/microceph/ceph/smb.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "net/http" + "os" "sort" "github.com/canonical/lxd/shared/api" @@ -27,6 +28,7 @@ var ( smbClusterMembersFunc = smbClusterMembers smbEnableNodeFunc = smbEnableNode smbDisableNodeFunc = smbDisableNode + smbRegenerateNodeFunc = smbRegenerateNode ) // ResolveSMBPlacement resolves the spec placement to a sorted set of @@ -146,8 +148,8 @@ func ApplySMB(ctx context.Context, s interfaces.StateInterface, payload string) } // Refresh the stored spec on re-apply so joining nodes render from the - // latest config. Rolling regeneration of already placed members lands - // with M3 lifecycle. + // latest config. + configChanged := false if len(current) > 0 { existing, err := database.GroupedServicesQuery.GetGroupConfig(ctx, s, "smb", sp.Spec.ClusterID) if err != nil { @@ -158,6 +160,7 @@ func ApplySMB(ctx context.Context, s interfaces.StateInterface, payload string) if err != nil { return fmt.Errorf("failed to update smb cluster config: %w", err) } + configChanged = true } } @@ -178,6 +181,18 @@ func ApplySMB(ctx context.Context, s interfaces.StateInterface, payload string) } } + // Membership or spec changes invalidate every member's rendered + // configs (the nodes file must be identical cluster-wide), so + // regenerate all desired members, one node at a time. + if len(toEnable) > 0 || len(toDisable) > 0 || configChanged { + for _, node := range desired { + err = smbRegenerateNodeFunc(ctx, s, node, sp.Spec.ClusterID) + if err != nil { + return fmt.Errorf("failed to regenerate smb cluster '%s' on node '%s': %w", sp.Spec.ClusterID, node, err) + } + } + } + return nil } @@ -239,10 +254,14 @@ func ListSMB(ctx context.Context, s interfaces.StateInterface) ([]types.SMBServi return statuses, nil } -// DisableSMB removes this node from the smb cluster's records. Service -// teardown (ctdbd stop, config cleanup) lands with M3 lifecycle. +// DisableSMB tears this node out of an smb cluster and removes its +// records. func DisableSMB(ctx context.Context, s interfaces.StateInterface, clusterID string) error { - return database.GroupedServicesQuery.RemoveForHost(ctx, s, "smb", clusterID) + hostname, err := os.Hostname() + if err != nil { + return err + } + return disableSMBLocal(ctx, s, clusterID, NewSMBRenderParams(clusterID, hostname, true)) } // smbClusterMembers lists the cluster member names. @@ -269,6 +288,20 @@ func smbEnableNode(ctx context.Context, s interfaces.StateInterface, node, paylo return client.EnableSMBNodeService(ctx, cli, node, &data) } +// smbRegenerateNode re-renders configs and restarts ctdbd on the given +// node, locally or via the node-scoped endpoint. +func smbRegenerateNode(ctx context.Context, s interfaces.StateInterface, node, clusterID string) error { + if node == s.ClusterState().Name() { + return RegenerateSMBNode(ctx, s, clusterID) + } + + cli, err := s.ClusterState().Connect().Leader(false) + if err != nil { + return err + } + return client.RegenerateSMBNodeService(ctx, cli, node, &types.SMBService{ClusterID: clusterID}) +} + // smbDisableNode tears down smb membership on the given node, locally or // via the node-scoped endpoint. func smbDisableNode(ctx context.Context, s interfaces.StateInterface, node, clusterID string) error { diff --git a/microceph/ceph/smb_config.go b/microceph/ceph/smb_config.go index 3ec6ef61..c741cb72 100644 --- a/microceph/ceph/smb_config.go +++ b/microceph/ceph/smb_config.go @@ -24,6 +24,7 @@ type SMBPaths struct { // SMBRenderParams carries per-node, per-cluster rendering inputs. type SMBRenderParams struct { ClusterID string + Hostname string // Entity is the node's daemon cephx entity (client. prefixed). Entity string Clustered bool @@ -35,6 +36,7 @@ func NewSMBRenderParams(clusterID, hostname string, clustered bool) SMBRenderPar pathConsts := constants.GetPathConst() return SMBRenderParams{ ClusterID: clusterID, + Hostname: hostname, Entity: SMBDaemonEntity(clusterID, hostname), Clustered: clustered, Paths: SMBPaths{ diff --git a/microceph/ceph/smb_lifecycle.go b/microceph/ceph/smb_lifecycle.go new file mode 100644 index 00000000..cc387ea4 --- /dev/null +++ b/microceph/ceph/smb_lifecycle.go @@ -0,0 +1,316 @@ +package ceph + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "path/filepath" + "sort" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/interfaces" + "github.com/canonical/microceph/microceph/logger" +) + +// Injectable seams for unit tests. +var ( + smbMemberAddressesFunc = smbMemberAddresses + resolveSMBIfaceFunc = resolveSMBIface +) + +// smbStockCTDBScripts are the legacy event scripts the ctdb deb enables +// by default (shipped read-only at $SNAP/etc/ctdb); 10.interface is what +// assigns public addresses. +var smbStockCTDBScripts = []string{ + "00.ctdb.script", + "01.reclock.script", + "05.system.script", + "10.interface.script", +} + +// smbMemberAddresses maps cluster member names to their host addresses. +func smbMemberAddresses(s interfaces.StateInterface) (map[string]string, error) { + cli, err := s.ClusterState().Connect().Leader(false) + if err != nil { + return nil, err + } + return client.MClient.GetClusterMemberAddresses(cli) +} + +// resolveSMBIface returns the local interface whose subnet contains the +// given VIP address. +func resolveSMBIface(cidr string) (string, error) { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + return "", err + } + + ifaces, err := net.Interfaces() + if err != nil { + return "", err + } + + for _, iface := range ifaces { + addrs, err := iface.Addrs() + if err != nil { + continue + } + for _, addr := range addrs { + _, ifaceNet, err := net.ParseCIDR(addr.String()) + if err != nil { + continue + } + if ifaceNet.Contains(ip) { + return iface.Name, nil + } + } + } + + return "", fmt.Errorf("no local interface covers '%s'", cidr) +} + +// smbOrderedNodeIPs returns the group members' host addresses in row-id +// order: stable and append-only, as the CTDB nodes file requires. +func smbOrderedNodeIPs(ctx context.Context, s interfaces.StateInterface, clusterID string) ([]string, error) { + records, err := database.GroupedServicesQuery.GetGroupMemberRecords(ctx, s, "smb", clusterID) + if err != nil { + return nil, err + } + + // CTDB node numbers are nodes-file line indices: the order must be + // stable and append-only, which row ids provide. + sort.Slice(records, func(i, j int) bool { return records[i].ID < records[j].ID }) + + addresses, err := smbMemberAddressesFunc(s) + if err != nil { + return nil, fmt.Errorf("failed to fetch cluster member addresses: %w", err) + } + + ips := make([]string, 0, len(records)) + for _, record := range records { + ip, ok := addresses[record.Member] + if !ok { + return nil, fmt.Errorf("no address known for cluster member '%s'", record.Member) + } + ips = append(ips, ip) + } + + return ips, nil +} + +// populateCTDBBase fills CTDB_BASE with the stock deb-enabled event +// script links, our snapctl-based 50.samba, the functions library and +// script.options. +func populateCTDBBase(ctdbDir, snapPath string) error { + legacyDir := filepath.Join(ctdbDir, "events", "legacy") + err := os.MkdirAll(legacyDir, 0755) + if err != nil { + return err + } + + relink := func(target, link string) error { + err := os.Remove(link) + if err != nil && !os.IsNotExist(err) { + return err + } + return os.Symlink(target, link) + } + + for _, script := range smbStockCTDBScripts { + err = relink(filepath.Join(snapPath, "etc", "ctdb", "events", "legacy", script), filepath.Join(legacyDir, script)) + if err != nil { + return err + } + } + + err = relink(filepath.Join(snapPath, "ctdb", "events", "legacy", "50.samba.script"), filepath.Join(legacyDir, "50.samba.script")) + if err != nil { + return err + } + + err = relink(filepath.Join(snapPath, "etc", "ctdb", "functions"), filepath.Join(ctdbDir, "functions")) + if err != nil { + return err + } + + options, err := os.ReadFile(filepath.Join(snapPath, "ctdb", "script.options")) + if err != nil { + return err + } + return atomicWriteFile(filepath.Join(ctdbDir, "script.options"), options, 0644) +} + +// smbNodeDirs returns the per-cluster directories the daemons need, with +// their modes. Everything is root-owned: confined daemons have no +// dac_override, so a foreign-owned dir would be unreadable. +func smbNodeDirs(p SMBRenderParams) map[string]os.FileMode { + dataDir := filepath.Join(p.Paths.Data, "samba", p.ClusterID) + runDir := filepath.Join(p.Paths.Run, "samba", p.ClusterID) + + return map[string]os.FileMode{ + p.Paths.Conf: 0755, + filepath.Join(p.Paths.Conf, "samba"): 0755, + filepath.Join(dataDir, "private"): 0700, + filepath.Join(dataDir, "lock"): 0755, + filepath.Join(dataDir, "state"): 0755, + filepath.Join(dataDir, "cache"): 0755, + filepath.Join(runDir, "ncalrpc"): 0755, + filepath.Join(p.Paths.Run, "ctdb"): 0755, + filepath.Join(p.Paths.Log, "samba", p.ClusterID): 0755, + } +} + +// enableSMBNodeLocal brings this node into an smb cluster: directories, +// keyrings, rendered configs, CTDB_BASE and the ctdbd service (which +// starts smbd through the 50.samba event script). +func enableSMBNodeLocal(ctx context.Context, s interfaces.StateInterface, spec *types.SMBSpec, p SMBRenderParams) error { + for dir, mode := range smbNodeDirs(p) { + err := os.MkdirAll(dir, mode) + if err != nil { + return err + } + } + + err := EnsureSMBKeyrings(spec, p.Hostname, p.Paths.Conf) + if err != nil { + return err + } + + ips, err := smbOrderedNodeIPs(ctx, s, spec.ClusterID) + if err != nil { + return err + } + + err = WriteSMBNodeConfigs(spec, p, ips, resolveSMBIfaceFunc) + if err != nil { + return err + } + + err = populateCTDBBase(filepath.Join(p.Paths.Conf, "ctdb"), p.Paths.Snap) + if err != nil { + return err + } + + err = snapStart("ctdbd", true) + if err != nil { + return fmt.Errorf("failed to start ctdbd: %w", err) + } + + logger.Infof("enabled smb cluster '%s' on this node", spec.ClusterID) + return nil +} + +// EnableSMB is the env-wired entry point used by the placement flow. +func EnableSMB(ctx context.Context, s interfaces.StateInterface, spec *types.SMBSpec) error { + hostname, err := os.Hostname() + if err != nil { + return err + } + + clustered := false + for _, feature := range spec.Features { + if feature == "clustered" { + clustered = true + } + } + + return enableSMBNodeLocal(ctx, s, spec, NewSMBRenderParams(spec.ClusterID, hostname, clustered)) +} + +// disableSMBLocal tears this node out of an smb cluster: services, +// keyrings, configs, runtime dirs and per-cluster data, then the DB +// record. Missing group config downgrades to best-effort file cleanup. +func disableSMBLocal(ctx context.Context, s interfaces.StateInterface, clusterID string, p SMBRenderParams) error { + err := snapStop("ctdbd", true) + if err != nil { + logger.Warnf("failed to stop ctdbd while disabling smb '%s': %v", clusterID, err) + } + err = snapStop("smbd", true) + if err != nil { + logger.Warnf("failed to stop smbd while disabling smb '%s': %v", clusterID, err) + } + + config, err := database.GroupedServicesQuery.GetGroupConfig(ctx, s, "smb", clusterID) + if err == nil { + var spec types.SMBSpec + err = json.Unmarshal([]byte(config), &spec) + if err == nil { + err = RemoveSMBKeyrings(&spec, p.Hostname, p.Paths.Conf) + if err != nil { + logger.Warnf("failed to remove smb keyrings for '%s': %v", clusterID, err) + } + } + } else { + logger.Warnf("no stored config for smb cluster '%s'; skipping keyring cleanup: %v", clusterID, err) + } + + for _, path := range []string{ + filepath.Join(p.Paths.Conf, "samba", "smb.conf"), + filepath.Join(p.Paths.Conf, "samba", "config.json"), + } { + err = os.Remove(path) + if err != nil && !os.IsNotExist(err) { + return err + } + } + + for _, dir := range []string{ + filepath.Join(p.Paths.Conf, "ctdb"), + filepath.Join(p.Paths.Run, "samba", clusterID), + filepath.Join(p.Paths.Run, "ctdb"), + filepath.Join(p.Paths.Data, "samba", clusterID), + } { + err = os.RemoveAll(dir) + if err != nil { + return err + } + } + + return database.GroupedServicesQuery.RemoveForHost(ctx, s, "smb", clusterID) +} + +// regenerateSMBNodeLocal re-renders this node's configs from the stored +// spec and restarts ctdbd; used when membership or the spec changes. +func regenerateSMBNodeLocal(ctx context.Context, s interfaces.StateInterface, clusterID string, p SMBRenderParams) error { + config, err := database.GroupedServicesQuery.GetGroupConfig(ctx, s, "smb", clusterID) + if err != nil { + return fmt.Errorf("failed to fetch config for smb cluster '%s': %w", clusterID, err) + } + + var spec types.SMBSpec + err = json.Unmarshal([]byte(config), &spec) + if err != nil { + return fmt.Errorf("cannot parse stored spec for smb cluster '%s': %w", clusterID, err) + } + + ips, err := smbOrderedNodeIPs(ctx, s, clusterID) + if err != nil { + return err + } + + err = WriteSMBNodeConfigs(&spec, p, ips, resolveSMBIfaceFunc) + if err != nil { + return err + } + + err = snapRestart("ctdbd", false) + if err != nil { + return fmt.Errorf("failed to restart ctdbd: %w", err) + } + + logger.Infof("regenerated smb cluster '%s' configs on this node", clusterID) + return nil +} + +// RegenerateSMBNode is the env-wired entry point for config regeneration. +func RegenerateSMBNode(ctx context.Context, s interfaces.StateInterface, clusterID string) error { + hostname, err := os.Hostname() + if err != nil { + return err + } + return regenerateSMBNodeLocal(ctx, s, clusterID, NewSMBRenderParams(clusterID, hostname, true)) +} diff --git a/microceph/ceph/smb_lifecycle_test.go b/microceph/ceph/smb_lifecycle_test.go new file mode 100644 index 00000000..e4c3372a --- /dev/null +++ b/microceph/ceph/smb_lifecycle_test.go @@ -0,0 +1,235 @@ +package ceph + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/interfaces" + "github.com/canonical/microceph/microceph/mocks" + "github.com/canonical/microceph/microceph/tests" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type smbLifecycleSuite struct { + tests.BaseSuite + TestStateInterface *mocks.StateInterface +} + +func TestSMBLifecycleSuite(t *testing.T) { + suite.Run(t, new(smbLifecycleSuite)) +} + +func (s *smbLifecycleSuite) SetupTest() { + s.BaseSuite.SetupTest() + s.TestStateInterface = mocks.NewStateInterface(s.T()) + + originalAddresses := smbMemberAddressesFunc + originalIface := resolveSMBIfaceFunc + s.T().Cleanup(func() { + smbMemberAddressesFunc = originalAddresses + resolveSMBIfaceFunc = originalIface + }) + + smbMemberAddressesFunc = func(st interfaces.StateInterface) (map[string]string, error) { + return map[string]string{"host1": "10.0.0.1", "host2": "10.0.0.2"}, nil + } + resolveSMBIfaceFunc = func(cidr string) (string, error) { return "enp5s0", nil } +} + +// lifecycleEnv builds a fake snap tree (stock ctdb files) and render +// params rooted in temp dirs. +func (s *smbLifecycleSuite) lifecycleEnv() SMBRenderParams { + root := s.T().TempDir() + snapDir := filepath.Join(root, "snap") + + for _, dir := range []string{ + filepath.Join(snapDir, "etc", "ctdb", "events", "legacy"), + filepath.Join(snapDir, "ctdb", "events", "legacy"), + } { + assert.NoError(s.T(), os.MkdirAll(dir, 0755)) + } + for _, script := range smbStockCTDBScripts { + assert.NoError(s.T(), os.WriteFile(filepath.Join(snapDir, "etc", "ctdb", "events", "legacy", script), []byte("#!/bin/sh\n"), 0755)) + } + assert.NoError(s.T(), os.WriteFile(filepath.Join(snapDir, "etc", "ctdb", "functions"), []byte("# functions\n"), 0644)) + assert.NoError(s.T(), os.WriteFile(filepath.Join(snapDir, "ctdb", "events", "legacy", "50.samba.script"), []byte("#!/bin/sh\n"), 0755)) + assert.NoError(s.T(), os.WriteFile(filepath.Join(snapDir, "ctdb", "script.options"), []byte("CTDB_SAMBA_SKIP_SHARE_CHECK=yes\n"), 0644)) + + return SMBRenderParams{ + ClusterID: "dev", + Hostname: "host1", + Entity: "client.smb.dev.host1", + Clustered: true, + Paths: SMBPaths{ + Conf: filepath.Join(root, "conf"), + Run: filepath.Join(root, "run"), + Data: filepath.Join(root, "data"), + Log: filepath.Join(root, "logs"), + Snap: snapDir, + }, + } +} + +func (s *smbLifecycleSuite) withDB() *mocks.GroupedServiceQueryIntf { + db := mocks.NewGroupedServiceQueryIntf(s.T()) + originalDB := database.GroupedServicesQuery + s.T().Cleanup(func() { database.GroupedServicesQuery = originalDB }) + database.GroupedServicesQuery = db + return db +} + +func (s *smbLifecycleSuite) memberRecords() []database.GroupedService { + return []database.GroupedService{ + {ID: 1, Service: "smb", GroupID: "dev", Member: "host1"}, + {ID: 2, Service: "smb", GroupID: "dev", Member: "host2"}, + } +} + +func (s *smbLifecycleSuite) TestEnableSMBNodeLocal() { + p := s.lifecycleEnv() + ctx := context.Background() + + var spec types.SMBSpec + assert.NoError(s.T(), json.Unmarshal([]byte(validSMBPayload), &spec)) + + configJSON, err := os.ReadFile(filepath.Join("testdata", "smb", "config.smb.json")) + assert.NoError(s.T(), err) + + db := s.withDB() + db.On("GetGroupMemberRecords", ctx, s.TestStateInterface, "smb", "dev").Return(s.memberRecords(), nil).Once() + + r := mocks.NewRunner(s.T()) + r.On("RunCommand", "ceph", "auth", "get-or-create", "client.smb.dev.host1").Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "caps", "client.smb.dev.host1", + "mon", smbMonCaps("dev"), "osd", smbOSDCaps(&spec)).Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "get", "client.smb.dev.host1", "-o", mock.Anything).Run(func(args mock.Arguments) { + assert.NoError(s.T(), os.WriteFile(args.Get(5).(string), []byte("[client]\nkey=x\n"), 0600)) + }).Return("", nil).Once() + r.On("RunCommand", "rados", "get", "--pool", ".smb", "-N", "dev", "scc.dev.json", "-"). + Return(string(configJSON), nil).Once() + r.On("RunCommand", "python3", "-m", "sambacc.commands.main", + "--config", filepath.Join(p.Paths.Conf, "samba", "config.json"), + "--identity", "dev", "print-config").Return("[global]\nrendered\n", nil).Once() + r.On("RunCommand", "snapctl", "start", "microceph.ctdbd", "--enable").Return("", nil).Once() + common.ProcessExec = r + + err = enableSMBNodeLocal(ctx, s.TestStateInterface, &spec, p) + assert.NoError(s.T(), err) + + // Directories. + info, err := os.Stat(filepath.Join(p.Paths.Data, "samba", "dev", "private")) + assert.NoError(s.T(), err) + assert.Equal(s.T(), os.FileMode(0700), info.Mode().Perm()) + + // Rendered configs. + smbConf, err := os.ReadFile(filepath.Join(p.Paths.Conf, "samba", "smb.conf")) + assert.NoError(s.T(), err) + assert.Equal(s.T(), "[global]\nrendered\n", string(smbConf)) + + nodes, err := os.ReadFile(filepath.Join(p.Paths.Conf, "ctdb", "nodes")) + assert.NoError(s.T(), err) + assert.Equal(s.T(), "10.0.0.1\n10.0.0.2\n", string(nodes)) + + // CTDB_BASE population. + for _, script := range append(smbStockCTDBScripts, "50.samba.script") { + link := filepath.Join(p.Paths.Conf, "ctdb", "events", "legacy", script) + target, err := os.Readlink(link) + assert.NoError(s.T(), err, script) + assert.FileExists(s.T(), target, script) + } + options, err := os.ReadFile(filepath.Join(p.Paths.Conf, "ctdb", "script.options")) + assert.NoError(s.T(), err) + assert.Contains(s.T(), string(options), "CTDB_SAMBA_SKIP_SHARE_CHECK=yes") +} + +func (s *smbLifecycleSuite) TestDisableSMBLocal() { + p := s.lifecycleEnv() + ctx := context.Background() + + // Seed on-disk state to tear down. + for _, dir := range []string{ + filepath.Join(p.Paths.Conf, "samba"), + filepath.Join(p.Paths.Conf, "ctdb"), + filepath.Join(p.Paths.Data, "samba", "dev"), + filepath.Join(p.Paths.Run, "samba", "dev"), + } { + assert.NoError(s.T(), os.MkdirAll(dir, 0755)) + } + assert.NoError(s.T(), os.WriteFile(filepath.Join(p.Paths.Conf, "samba", "smb.conf"), []byte("x"), 0644)) + assert.NoError(s.T(), os.WriteFile(filepath.Join(p.Paths.Conf, "samba", "config.json"), []byte("x"), 0644)) + assert.NoError(s.T(), os.WriteFile(filepath.Join(p.Paths.Conf, "ceph.client.smb.dev.host1.keyring"), []byte("k"), 0600)) + + canonical := mustCompactJSON(validSMBPayload) + + db := s.withDB() + db.On("GetGroupConfig", ctx, s.TestStateInterface, "smb", "dev").Return(canonical, nil).Once() + db.On("RemoveForHost", ctx, s.TestStateInterface, "smb", "dev").Return(nil).Once() + + r := mocks.NewRunner(s.T()) + r.On("RunCommand", "snapctl", "stop", "microceph.ctdbd", "--disable").Return("", nil).Once() + r.On("RunCommand", "snapctl", "stop", "microceph.smbd", "--disable").Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "del", "client.smb.dev.host1").Return("", nil).Once() + common.ProcessExec = r + + err := disableSMBLocal(ctx, s.TestStateInterface, "dev", p) + assert.NoError(s.T(), err) + + assert.NoFileExists(s.T(), filepath.Join(p.Paths.Conf, "samba", "smb.conf")) + assert.NoFileExists(s.T(), filepath.Join(p.Paths.Conf, "ceph.client.smb.dev.host1.keyring")) + assert.NoDirExists(s.T(), filepath.Join(p.Paths.Conf, "ctdb")) + assert.NoDirExists(s.T(), filepath.Join(p.Paths.Data, "samba", "dev")) +} + +func (s *smbLifecycleSuite) TestRegenerateSMBNodeLocal() { + p := s.lifecycleEnv() + ctx := context.Background() + + configJSON, err := os.ReadFile(filepath.Join("testdata", "smb", "config.smb.json")) + assert.NoError(s.T(), err) + + canonical := mustCompactJSON(validSMBPayload) + + db := s.withDB() + db.On("GetGroupConfig", ctx, s.TestStateInterface, "smb", "dev").Return(canonical, nil).Once() + db.On("GetGroupMemberRecords", ctx, s.TestStateInterface, "smb", "dev").Return(s.memberRecords(), nil).Once() + + r := mocks.NewRunner(s.T()) + r.On("RunCommand", "rados", "get", "--pool", ".smb", "-N", "dev", "scc.dev.json", "-"). + Return(string(configJSON), nil).Once() + r.On("RunCommand", "python3", "-m", "sambacc.commands.main", + "--config", filepath.Join(p.Paths.Conf, "samba", "config.json"), + "--identity", "dev", "print-config").Return("[global]\nregen\n", nil).Once() + r.On("RunCommand", "snapctl", "restart", "microceph.ctdbd").Return("", nil).Once() + common.ProcessExec = r + + err = regenerateSMBNodeLocal(ctx, s.TestStateInterface, "dev", p) + assert.NoError(s.T(), err) + + smbConf, err := os.ReadFile(filepath.Join(p.Paths.Conf, "samba", "smb.conf")) + assert.NoError(s.T(), err) + assert.Equal(s.T(), "[global]\nregen\n", string(smbConf)) +} + +func (s *smbLifecycleSuite) TestOrderedNodeIPsFollowsRowIDs() { + ctx := context.Background() + + db := s.withDB() + // Rows deliberately out of row-id order from the mapper. + db.On("GetGroupMemberRecords", ctx, s.TestStateInterface, "smb", "dev").Return([]database.GroupedService{ + {ID: 2, Member: "host2"}, + {ID: 1, Member: "host1"}, + }, nil).Once() + + ips, err := smbOrderedNodeIPs(ctx, s.TestStateInterface, "dev") + assert.NoError(s.T(), err) + assert.Equal(s.T(), []string{"10.0.0.1", "10.0.0.2"}, ips) +} diff --git a/microceph/ceph/smb_test.go b/microceph/ceph/smb_test.go index ee62976c..bfb8cf61 100644 --- a/microceph/ceph/smb_test.go +++ b/microceph/ceph/smb_test.go @@ -31,8 +31,9 @@ type smbSuite struct { tests.BaseSuite TestStateInterface *mocks.StateInterface - enabled []string - disabled []string + enabled []string + disabled []string + regenerated []string } func TestSMBSuite(t *testing.T) { @@ -47,14 +48,17 @@ func (s *smbSuite) SetupTest() { s.enabled = nil s.disabled = nil + s.regenerated = nil originalMembers := smbClusterMembersFunc originalEnable := smbEnableNodeFunc originalDisable := smbDisableNodeFunc + originalRegenerate := smbRegenerateNodeFunc s.T().Cleanup(func() { smbClusterMembersFunc = originalMembers smbEnableNodeFunc = originalEnable smbDisableNodeFunc = originalDisable + smbRegenerateNodeFunc = originalRegenerate }) smbClusterMembersFunc = func(s interfaces.StateInterface) ([]string, error) { @@ -68,6 +72,10 @@ func (s *smbSuite) SetupTest() { s.disabled = append(s.disabled, node) return nil } + smbRegenerateNodeFunc = func(ctx context.Context, st interfaces.StateInterface, node, clusterID string) error { + s.regenerated = append(s.regenerated, node) + return nil + } } // withDB patches the grouped-services query with a fresh mock and restores @@ -181,6 +189,9 @@ func (s *smbSuite) TestApplyFresh() { assert.NoError(s.T(), err) assert.Equal(s.T(), []string{"m1", "m2", "m3"}, s.enabled) assert.Empty(s.T(), s.disabled) + // A fresh apply regenerates every member so all nodes files carry the + // complete membership (early joiners rendered before later rows). + assert.Equal(s.T(), []string{"m1", "m2", "m3"}, s.regenerated) } func (s *smbSuite) TestApplyIdempotent() { @@ -195,6 +206,7 @@ func (s *smbSuite) TestApplyIdempotent() { assert.NoError(s.T(), err) assert.Empty(s.T(), s.enabled) assert.Empty(s.T(), s.disabled) + assert.Empty(s.T(), s.regenerated) } func (s *smbSuite) TestApplyMemberChange() { @@ -209,6 +221,7 @@ func (s *smbSuite) TestApplyMemberChange() { assert.NoError(s.T(), err) assert.Equal(s.T(), []string{"m1"}, s.enabled) assert.Equal(s.T(), []string{"m3"}, s.disabled) + assert.Equal(s.T(), []string{"m1", "m2"}, s.regenerated) } func (s *smbSuite) TestApplyConfigChange() { @@ -224,6 +237,8 @@ func (s *smbSuite) TestApplyConfigChange() { assert.NoError(s.T(), err) assert.Empty(s.T(), s.enabled) assert.Empty(s.T(), s.disabled) + // Spec content changed with steady membership: every member re-renders. + assert.Equal(s.T(), []string{"m1", "m2"}, s.regenerated) } func (s *smbSuite) TestApplyInvalidSpec() { diff --git a/microceph/client/services.go b/microceph/client/services.go index 751f40a8..92be9280 100644 --- a/microceph/client/services.go +++ b/microceph/client/services.go @@ -134,6 +134,23 @@ func EnableSMBNodeService(ctx context.Context, c mcTypes.Client, target string, return nil } +// RegenerateSMBNodeService requests the target node re-render its smb +// configs and restart ctdbd. +func RegenerateSMBNodeService(ctx context.Context, c mcTypes.Client, target string, svc *types.SMBService) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + defer cancel() + + // Send this request to target. + c = c.UseTarget(target) + + err := c.Query(queryCtx, "POST", types.ExtendedPathPrefix, &api.NewURL().Path("services", "smb", "node").URL, svc, nil) + if err != nil { + return fmt.Errorf("failed regenerating smb service on %s: %w", target, err) + } + + return nil +} + // DeleteSMBNodeService requests the target node tear down its smb cluster // membership. func DeleteSMBNodeService(ctx context.Context, c mcTypes.Client, target string, svc *types.SMBService) error { diff --git a/microceph/client/wrap.go b/microceph/client/wrap.go index e8c7b25c..4220097a 100644 --- a/microceph/client/wrap.go +++ b/microceph/client/wrap.go @@ -13,6 +13,7 @@ import ( // This is useful for mocking in unit tests type ClientInterface interface { GetClusterMembers(mcTypes.Client) ([]string, error) + GetClusterMemberAddresses(mcTypes.Client) (map[string]string, error) GetDisks(mcTypes.Client) (types.Disks, error) GetServices(mcTypes.Client) (types.Services, error) DeleteService(mcTypes.Client, string, string) error @@ -21,6 +22,23 @@ type ClientInterface interface { type ClientImpl struct{} +// GetClusterMemberAddresses gets a member name to host address mapping +// (ports stripped). +func (c ClientImpl) GetClusterMemberAddresses(cli mcTypes.Client) (map[string]string, error) { + var members []mcTypes.ClusterMember + err := cli.Query(context.Background(), "GET", mcTypes.PublicEndpoint, &api.NewURL().Path("cluster").URL, nil, &members) + if err != nil { + return nil, err + } + + addresses := make(map[string]string, len(members)) + for _, member := range members { + addresses[member.Name] = member.Address.Addr().String() + } + + return addresses, nil +} + // GetClusterMembers gets the cluster member names // We return names only here because the Member type is internal to microclient func (c ClientImpl) GetClusterMembers(cli mcTypes.Client) ([]string, error) { diff --git a/microceph/database/grouped_service_extras.go b/microceph/database/grouped_service_extras.go index c640a01b..132d35c2 100644 --- a/microceph/database/grouped_service_extras.go +++ b/microceph/database/grouped_service_extras.go @@ -29,6 +29,7 @@ type GroupedServiceQueryIntf interface { // Group Methods GetGroupMembers(ctx context.Context, s interfaces.StateInterface, service, groupID string) ([]string, error) + GetGroupMemberRecords(ctx context.Context, s interfaces.StateInterface, service, groupID string) ([]GroupedService, error) GetGroupConfig(ctx context.Context, s interfaces.StateInterface, service, groupID string) (string, error) UpdateGroupConfig(ctx context.Context, s interfaces.StateInterface, service, groupID, config string) error @@ -190,6 +191,38 @@ func (g GroupedServiceQueryImpl) GetGroupMembers(ctx context.Context, s interfac return members, nil } +// GetGroupMemberRecords returns the group's rows in row-id (insertion) +// order, for consumers that need a stable, append-only ordering such as +// the CTDB nodes file. +func (g GroupedServiceQueryImpl) GetGroupMemberRecords(ctx context.Context, s interfaces.StateInterface, service, groupID string) ([]GroupedService, error) { + if s.ClusterState().ServerCert() == nil { + return nil, fmt.Errorf("no server certificate") + } + + var services []GroupedService + + err := s.ClusterState().Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + filter := GroupedServiceFilter{ + Service: &service, + GroupID: &groupID, + } + + var err error + services, err = GetGroupedServices(ctx, tx, filter) + if err != nil { + return fmt.Errorf("failed to get grouped services records: %w", err) + } + + return nil + }) + if err != nil { + return nil, err + } + + sort.Slice(services, func(i, j int) bool { return services[i].ID < services[j].ID }) + return services, nil +} + // GetGroupConfig returns the stored config of a service group. func (g GroupedServiceQueryImpl) GetGroupConfig(ctx context.Context, s interfaces.StateInterface, service, groupID string) (string, error) { if s.ClusterState().ServerCert() == nil { diff --git a/microceph/mocks/ClientInterface.go b/microceph/mocks/ClientInterface.go index 5db40142..b5d89f58 100644 --- a/microceph/mocks/ClientInterface.go +++ b/microceph/mocks/ClientInterface.go @@ -43,6 +43,32 @@ func (_m *ClientInterface) DeleteService(_a0 mcTypes.Client, _a1 string, _a2 str return r0 } +// GetClusterMemberAddresses provides a mock function with given fields: _a0 +func (_m *ClientInterface) GetClusterMemberAddresses(_a0 mcTypes.Client) (map[string]string, error) { + ret := _m.Called(_a0) + + var r0 map[string]string + var r1 error + if rf, ok := ret.Get(0).(func(mcTypes.Client) (map[string]string, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(mcTypes.Client) map[string]string); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]string) + } + } + + if rf, ok := ret.Get(1).(func(mcTypes.Client) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetClusterMembers provides a mock function with given fields: _a0 func (_m *ClientInterface) GetClusterMembers(_a0 mcTypes.Client) ([]string, error) { ret := _m.Called(_a0) diff --git a/microceph/mocks/GroupedServiceQueryIntf.go b/microceph/mocks/GroupedServiceQueryIntf.go index 31ffcfc2..0ae9e456 100644 --- a/microceph/mocks/GroupedServiceQueryIntf.go +++ b/microceph/mocks/GroupedServiceQueryIntf.go @@ -170,6 +170,36 @@ func (_m *GroupedServiceQueryIntf) GetGroupMembers(ctx context.Context, s interf return r0, r1 } +// GetGroupMemberRecords provides a mock function with given fields: ctx, s, service, groupID +func (_m *GroupedServiceQueryIntf) GetGroupMemberRecords(ctx context.Context, s interfaces.StateInterface, service string, groupID string) ([]database.GroupedService, error) { + ret := _m.Called(ctx, s, service, groupID) + + if len(ret) == 0 { + panic("no return value specified for GetGroupMemberRecords") + } + + var r0 []database.GroupedService + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, interfaces.StateInterface, string, string) ([]database.GroupedService, error)); ok { + return rf(ctx, s, service, groupID) + } + if rf, ok := ret.Get(0).(func(context.Context, interfaces.StateInterface, string, string) []database.GroupedService); ok { + r0 = rf(ctx, s, service, groupID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]database.GroupedService) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, interfaces.StateInterface, string, string) error); ok { + r1 = rf(ctx, s, service, groupID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetGroupConfig provides a mock function with given fields: ctx, s, service, groupID func (_m *GroupedServiceQueryIntf) GetGroupConfig(ctx context.Context, s interfaces.StateInterface, service string, groupID string) (string, error) { ret := _m.Called(ctx, s, service, groupID) From 6252b6183b0a602867602ea073d0d93bf2067554 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 17:45:22 +0530 Subject: [PATCH 12/31] daemon,snap: fix ctdb runtime paths and nodes-file self-inclusion Live single-node bring-up surfaced four gaps, all fixed at the root: ctdbd's compiled-in paths need overriding inside the snap ([database] directories and the logging location in the rendered ctdb.conf, plus CTDB_EVENTD/CTDB_*_HELPER/CTDB_HELPER_BINDIR exports and a /run/ctdb pid dir in the start wrapper); CTDB_BASE additionally needs notify.sh linked. The nodes file rendered empty on first enable because rendering runs before DbUpdate records the local node; the local node is now appended when its row is missing, preserving row-id order. With these, ctdbd reaches recovery NORMAL and takes the RADOS cluster lock with the per-node daemon key. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/smb_config.go | 15 +++++- microceph/ceph/smb_lifecycle.go | 52 ++++++++++++++------ microceph/ceph/smb_lifecycle_test.go | 17 ++++++- microceph/ceph/testdata/smb/ctdb.conf.golden | 6 +++ snapcraft/commands/ctdbd.start | 13 +++++ 5 files changed, 85 insertions(+), 18 deletions(-) diff --git a/microceph/ceph/smb_config.go b/microceph/ceph/smb_config.go index c741cb72..c4f992cb 100644 --- a/microceph/ceph/smb_config.go +++ b/microceph/ceph/smb_config.go @@ -196,13 +196,26 @@ func RenderCTDBConf(p SMBRenderParams, lockURI string) (string, error) { helper := filepath.Join(p.Paths.Snap, "libexec", "ctdb", "ctdb_mutex_ceph_rados_helper") object := smbReclockObject + p.ClusterID + dbDir := filepath.Join(p.Paths.Data, "ctdb") + // The [database] paths and the logging location override ctdbd's + // compiled-in /var/lib/ctdb and /var/log defaults, which do not exist + // inside the snap (the event daemon dies at init without a usable + // logging location). return fmt.Sprintf(`[logging] + location = file:%s log level = NOTICE +[database] + volatile database directory = %s + persistent database directory = %s + state database directory = %s + [cluster] cluster lock = !%s ceph %s %s %s -`, helper, p.Entity, pool, object), nil +`, filepath.Join(p.Paths.Log, "ctdb", "log.ctdb"), + filepath.Join(dbDir, "volatile"), filepath.Join(dbDir, "persistent"), filepath.Join(dbDir, "state"), + helper, p.Entity, pool, object), nil } // RenderCTDBNodes renders the nodes file: one private address per line. diff --git a/microceph/ceph/smb_lifecycle.go b/microceph/ceph/smb_lifecycle.go index cc387ea4..7b22ba2b 100644 --- a/microceph/ceph/smb_lifecycle.go +++ b/microceph/ceph/smb_lifecycle.go @@ -74,8 +74,11 @@ func resolveSMBIface(cidr string) (string, error) { } // smbOrderedNodeIPs returns the group members' host addresses in row-id -// order: stable and append-only, as the CTDB nodes file requires. -func smbOrderedNodeIPs(ctx context.Context, s interfaces.StateInterface, clusterID string) ([]string, error) { +// order: stable and append-only, as the CTDB nodes file requires. The +// local node is appended when its own row does not exist yet: during +// enable, rendering runs before DbUpdate records this node, and the +// membership row lands next (so appending preserves row-id order). +func smbOrderedNodeIPs(ctx context.Context, s interfaces.StateInterface, clusterID, hostname string) ([]string, error) { records, err := database.GroupedServicesQuery.GetGroupMemberRecords(ctx, s, "smb", clusterID) if err != nil { return nil, err @@ -85,6 +88,16 @@ func smbOrderedNodeIPs(ctx context.Context, s interfaces.StateInterface, cluster // stable and append-only, which row ids provide. sort.Slice(records, func(i, j int) bool { return records[i].ID < records[j].ID }) + selfRecorded := false + for _, record := range records { + if record.Member == hostname { + selfRecorded = true + } + } + if !selfRecorded { + records = append(records, database.GroupedService{Member: hostname}) + } + addresses, err := smbMemberAddressesFunc(s) if err != nil { return nil, fmt.Errorf("failed to fetch cluster member addresses: %w", err) @@ -132,9 +145,11 @@ func populateCTDBBase(ctdbDir, snapPath string) error { return err } - err = relink(filepath.Join(snapPath, "etc", "ctdb", "functions"), filepath.Join(ctdbDir, "functions")) - if err != nil { - return err + for _, file := range []string{"functions", "notify.sh"} { + err = relink(filepath.Join(snapPath, "etc", "ctdb", file), filepath.Join(ctdbDir, file)) + if err != nil { + return err + } } options, err := os.ReadFile(filepath.Join(snapPath, "ctdb", "script.options")) @@ -152,15 +167,19 @@ func smbNodeDirs(p SMBRenderParams) map[string]os.FileMode { runDir := filepath.Join(p.Paths.Run, "samba", p.ClusterID) return map[string]os.FileMode{ - p.Paths.Conf: 0755, - filepath.Join(p.Paths.Conf, "samba"): 0755, - filepath.Join(dataDir, "private"): 0700, - filepath.Join(dataDir, "lock"): 0755, - filepath.Join(dataDir, "state"): 0755, - filepath.Join(dataDir, "cache"): 0755, - filepath.Join(runDir, "ncalrpc"): 0755, - filepath.Join(p.Paths.Run, "ctdb"): 0755, - filepath.Join(p.Paths.Log, "samba", p.ClusterID): 0755, + p.Paths.Conf: 0755, + filepath.Join(p.Paths.Conf, "samba"): 0755, + filepath.Join(dataDir, "private"): 0700, + filepath.Join(dataDir, "lock"): 0755, + filepath.Join(dataDir, "state"): 0755, + filepath.Join(dataDir, "cache"): 0755, + filepath.Join(runDir, "ncalrpc"): 0755, + filepath.Join(p.Paths.Run, "ctdb"): 0755, + filepath.Join(p.Paths.Log, "samba", p.ClusterID): 0755, + filepath.Join(p.Paths.Data, "ctdb", "volatile"): 0700, + filepath.Join(p.Paths.Data, "ctdb", "persistent"): 0700, + filepath.Join(p.Paths.Data, "ctdb", "state"): 0700, + filepath.Join(p.Paths.Log, "ctdb"): 0755, } } @@ -180,7 +199,7 @@ func enableSMBNodeLocal(ctx context.Context, s interfaces.StateInterface, spec * return err } - ips, err := smbOrderedNodeIPs(ctx, s, spec.ClusterID) + ips, err := smbOrderedNodeIPs(ctx, s, spec.ClusterID, p.Hostname) if err != nil { return err } @@ -263,6 +282,7 @@ func disableSMBLocal(ctx context.Context, s interfaces.StateInterface, clusterID filepath.Join(p.Paths.Run, "samba", clusterID), filepath.Join(p.Paths.Run, "ctdb"), filepath.Join(p.Paths.Data, "samba", clusterID), + filepath.Join(p.Paths.Data, "ctdb"), } { err = os.RemoveAll(dir) if err != nil { @@ -287,7 +307,7 @@ func regenerateSMBNodeLocal(ctx context.Context, s interfaces.StateInterface, cl return fmt.Errorf("cannot parse stored spec for smb cluster '%s': %w", clusterID, err) } - ips, err := smbOrderedNodeIPs(ctx, s, clusterID) + ips, err := smbOrderedNodeIPs(ctx, s, clusterID, p.Hostname) if err != nil { return err } diff --git a/microceph/ceph/smb_lifecycle_test.go b/microceph/ceph/smb_lifecycle_test.go index e4c3372a..71d4da36 100644 --- a/microceph/ceph/smb_lifecycle_test.go +++ b/microceph/ceph/smb_lifecycle_test.go @@ -229,7 +229,22 @@ func (s *smbLifecycleSuite) TestOrderedNodeIPsFollowsRowIDs() { {ID: 1, Member: "host1"}, }, nil).Once() - ips, err := smbOrderedNodeIPs(ctx, s.TestStateInterface, "dev") + ips, err := smbOrderedNodeIPs(ctx, s.TestStateInterface, "dev", "host1") + assert.NoError(s.T(), err) + assert.Equal(s.T(), []string{"10.0.0.1", "10.0.0.2"}, ips) +} + +func (s *smbLifecycleSuite) TestOrderedNodeIPsIncludesUnrecordedSelf() { + ctx := context.Background() + + // During enable, rendering happens before DbUpdate records this node: + // the local node must still appear in its own nodes file. + db := s.withDB() + db.On("GetGroupMemberRecords", ctx, s.TestStateInterface, "smb", "dev").Return([]database.GroupedService{ + {ID: 1, Member: "host1"}, + }, nil).Once() + + ips, err := smbOrderedNodeIPs(ctx, s.TestStateInterface, "dev", "host2") assert.NoError(s.T(), err) assert.Equal(s.T(), []string{"10.0.0.1", "10.0.0.2"}, ips) } diff --git a/microceph/ceph/testdata/smb/ctdb.conf.golden b/microceph/ceph/testdata/smb/ctdb.conf.golden index 89c46a1f..003e719b 100644 --- a/microceph/ceph/testdata/smb/ctdb.conf.golden +++ b/microceph/ceph/testdata/smb/ctdb.conf.golden @@ -1,5 +1,11 @@ [logging] + location = file:/var/snap/microceph/common/logs/ctdb/log.ctdb log level = NOTICE +[database] + volatile database directory = /var/snap/microceph/common/data/ctdb/volatile + persistent database directory = /var/snap/microceph/common/data/ctdb/persistent + state database directory = /var/snap/microceph/common/data/ctdb/state + [cluster] cluster lock = !/snap/microceph/current/libexec/ctdb/ctdb_mutex_ceph_rados_helper ceph client.smb.dev.host1 .smb microceph.reclock.dev diff --git a/snapcraft/commands/ctdbd.start b/snapcraft/commands/ctdbd.start index db263264..81176daf 100755 --- a/snapcraft/commands/ctdbd.start +++ b/snapcraft/commands/ctdbd.start @@ -11,10 +11,23 @@ wait_for_config export CTDB_BASE="${SNAP_DATA}/conf/ctdb" export CTDB_SOCKET="${SNAP_DATA}/run/ctdb/ctdbd.socket" +# ctdbd and its event scripts spawn helpers at the compiled-in +# /usr/libexec/ctdb path, which does not exist inside the snap. +export CTDB_EVENTD="${SNAP}/libexec/ctdb/ctdb-eventd" +export CTDB_LOCK_HELPER="${SNAP}/libexec/ctdb/ctdb_lock_helper" +export CTDB_RECOVERY_HELPER="${SNAP}/libexec/ctdb/ctdb_recovery_helper" +export CTDB_TAKEOVER_HELPER="${SNAP}/libexec/ctdb/ctdb_takeover_helper" +export CTDB_HELPER_BINDIR="${SNAP}/libexec/ctdb" + mkdir -p \ "${CTDB_BASE}" \ "${SNAP_DATA}/run/ctdb" \ "${SNAP_COMMON}/logs/ctdb" \ "${SNAP_COMMON}/data/ctdb" +# The PID file path is compiled in and has no override; the run dir is +# writable under devmode and will need the smb-support interface under +# strict confinement. +mkdir -p /run/ctdb + exec ctdbd --interactive From 5318d525803bce87de21fae4e245e03f21ed0bec Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 18:03:14 +0530 Subject: [PATCH 13/31] daemon: keep the ctdb layout bind target on disable Removing conf/ctdb outright leaves the snap namespace's /etc/ctdb layout bind pointing at a dead inode, so services read an empty ghost directory until the namespace is rebuilt (observed live after a disable/re-enable cycle). Clear the directory's contents instead. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/smb_lifecycle.go | 30 +++++++++++++++++++++++++++- microceph/ceph/smb_lifecycle_test.go | 6 +++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/microceph/ceph/smb_lifecycle.go b/microceph/ceph/smb_lifecycle.go index 7b22ba2b..8eac4644 100644 --- a/microceph/ceph/smb_lifecycle.go +++ b/microceph/ceph/smb_lifecycle.go @@ -159,6 +159,26 @@ func populateCTDBBase(ctdbDir, snapPath string) error { return atomicWriteFile(filepath.Join(ctdbDir, "script.options"), options, 0644) } +// removeDirContents deletes everything inside dir but keeps dir itself. +func removeDirContents(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + for _, entry := range entries { + err = os.RemoveAll(filepath.Join(dir, entry.Name())) + if err != nil { + return err + } + } + + return nil +} + // smbNodeDirs returns the per-cluster directories the daemons need, with // their modes. Everything is root-owned: confined daemons have no // dac_override, so a foreign-owned dir would be unreadable. @@ -277,8 +297,16 @@ func disableSMBLocal(ctx context.Context, s interfaces.StateInterface, clusterID } } + // conf/ctdb is the target of the /etc/ctdb layout bind: removing the + // directory itself leaves the snap namespace bound to a dead inode + // (services then read an empty ghost dir until the ns is rebuilt), so + // only its contents are cleared. + err = removeDirContents(filepath.Join(p.Paths.Conf, "ctdb")) + if err != nil { + return err + } + for _, dir := range []string{ - filepath.Join(p.Paths.Conf, "ctdb"), filepath.Join(p.Paths.Run, "samba", clusterID), filepath.Join(p.Paths.Run, "ctdb"), filepath.Join(p.Paths.Data, "samba", clusterID), diff --git a/microceph/ceph/smb_lifecycle_test.go b/microceph/ceph/smb_lifecycle_test.go index 71d4da36..776a4ce6 100644 --- a/microceph/ceph/smb_lifecycle_test.go +++ b/microceph/ceph/smb_lifecycle_test.go @@ -185,8 +185,12 @@ func (s *smbLifecycleSuite) TestDisableSMBLocal() { assert.NoFileExists(s.T(), filepath.Join(p.Paths.Conf, "samba", "smb.conf")) assert.NoFileExists(s.T(), filepath.Join(p.Paths.Conf, "ceph.client.smb.dev.host1.keyring")) - assert.NoDirExists(s.T(), filepath.Join(p.Paths.Conf, "ctdb")) assert.NoDirExists(s.T(), filepath.Join(p.Paths.Data, "samba", "dev")) + + // conf/ctdb is a layout bind target: the dir must survive, emptied. + entries, err := os.ReadDir(filepath.Join(p.Paths.Conf, "ctdb")) + assert.NoError(s.T(), err) + assert.Empty(s.T(), entries) } func (s *smbLifecycleSuite) TestRegenerateSMBNodeLocal() { From 218f2f8c5f2ce58f876f7165387d0a84d0f67fda Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 18:07:38 +0530 Subject: [PATCH 14/31] snap: idle ctdbd until its config is rendered An enabled ctdbd starting before the deployment engine has rendered ctdb.conf (snap refresh, node reboot) crashlooped into systemd's restart rate limit, which then blocked the legitimate start during enable. Wait for the config file instead of exiting. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- snapcraft/commands/ctdbd.start | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/snapcraft/commands/ctdbd.start b/snapcraft/commands/ctdbd.start index 81176daf..8dea463d 100755 --- a/snapcraft/commands/ctdbd.start +++ b/snapcraft/commands/ctdbd.start @@ -25,6 +25,14 @@ mkdir -p \ "${SNAP_COMMON}/logs/ctdb" \ "${SNAP_COMMON}/data/ctdb" +# Idle until the deployment engine has rendered our config: ctdbd is +# enabled once a node joins an smb cluster, and a start before rendering +# (snap refresh, reboot) must not crashloop into systemd's rate limit. +while [ ! -f "${CTDB_BASE}/ctdb.conf" ]; do + echo "ctdbd: waiting for ${CTDB_BASE}/ctdb.conf" + sleep 5 +done + # The PID file path is compiled in and has no override; the run dir is # writable under devmode and will need the smb-support interface under # strict confinement. From 76cadeaaedb99214d34db7e09a58bb93f6d24a52 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 18:23:20 +0530 Subject: [PATCH 15/31] daemon: fan out smb node operations over direct member clients The leader-proxied UseTarget fan-out canceled long-running enable and regenerate legs mid-flight ('context canceled' surfacing at the next DB or client call on the target node), observed repeatedly during the three-node apply. Member names and addresses now come from the local trust store (heartbeat-fresh, no network call) and each node operation connects straight to its target via Connect().Member(), removing the proxy hop entirely. Cluster-scope client timeouts sized to the full multi-node apply (900s) and node-scope ones to a single node's enable (300s); the unused GetClusterMemberAddresses client wrap is reverted. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/smb.go | 33 ++++++++++++++++++++++-------- microceph/ceph/smb_lifecycle.go | 13 ++++++------ microceph/client/services.go | 10 ++++----- microceph/client/wrap.go | 18 ---------------- microceph/mocks/ClientInterface.go | 26 ----------------------- 5 files changed, 37 insertions(+), 63 deletions(-) diff --git a/microceph/ceph/smb.go b/microceph/ceph/smb.go index c274c2fc..eb21e346 100644 --- a/microceph/ceph/smb.go +++ b/microceph/ceph/smb.go @@ -11,6 +11,7 @@ import ( "sort" "github.com/canonical/lxd/shared/api" + mcTypes "github.com/canonical/microcluster/v3/microcluster/types" "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/microceph/microceph/client" @@ -264,13 +265,29 @@ func DisableSMB(ctx context.Context, s interfaces.StateInterface, clusterID stri return disableSMBLocal(ctx, s, clusterID, NewSMBRenderParams(clusterID, hostname, true)) } -// smbClusterMembers lists the cluster member names. +// smbClusterMembers lists the cluster member names from the local trust +// store (updated on heartbeats; no network round trip). func smbClusterMembers(s interfaces.StateInterface) ([]string, error) { - cli, err := s.ClusterState().Connect().Leader(false) - if err != nil { - return nil, err + addresses := s.ClusterState().Truststore().RemoteAddresses() + members := make([]string, 0, len(addresses)) + for name := range addresses { + members = append(members, name) } - return client.MClient.GetClusterMembers(cli) + sort.Strings(members) + return members, nil +} + +// smbNodeClient returns a client connected directly to the named member. +// Direct connections avoid proxy legs, whose cancellation poisoned +// long-running enable chains when fanned out through the leader. +func smbNodeClient(s interfaces.StateInterface, node string) (mcTypes.Client, error) { + addr, ok := s.ClusterState().Truststore().RemoteAddresses()[node] + if !ok { + return nil, fmt.Errorf("no address known for cluster member '%s'", node) + } + + url := api.NewURL().Scheme("https").Host(addr.String()) + return s.ClusterState().Connect().Member(&url.URL, false, nil) } // smbEnableNode runs the smb placement flow on the given node, locally or @@ -281,7 +298,7 @@ func smbEnableNode(ctx context.Context, s interfaces.StateInterface, node, paylo return ServicePlacementHandler(ctx, s, data) } - cli, err := s.ClusterState().Connect().Leader(false) + cli, err := smbNodeClient(s, node) if err != nil { return err } @@ -295,7 +312,7 @@ func smbRegenerateNode(ctx context.Context, s interfaces.StateInterface, node, c return RegenerateSMBNode(ctx, s, clusterID) } - cli, err := s.ClusterState().Connect().Leader(false) + cli, err := smbNodeClient(s, node) if err != nil { return err } @@ -309,7 +326,7 @@ func smbDisableNode(ctx context.Context, s interfaces.StateInterface, node, clus return DisableSMB(ctx, s, clusterID) } - cli, err := s.ClusterState().Connect().Leader(false) + cli, err := smbNodeClient(s, node) if err != nil { return err } diff --git a/microceph/ceph/smb_lifecycle.go b/microceph/ceph/smb_lifecycle.go index 8eac4644..dff92b81 100644 --- a/microceph/ceph/smb_lifecycle.go +++ b/microceph/ceph/smb_lifecycle.go @@ -10,7 +10,6 @@ import ( "sort" "github.com/canonical/microceph/microceph/api/types" - "github.com/canonical/microceph/microceph/client" "github.com/canonical/microceph/microceph/database" "github.com/canonical/microceph/microceph/interfaces" "github.com/canonical/microceph/microceph/logger" @@ -32,13 +31,15 @@ var smbStockCTDBScripts = []string{ "10.interface.script", } -// smbMemberAddresses maps cluster member names to their host addresses. +// smbMemberAddresses maps cluster member names to their host addresses +// from the local trust store (no network round trip). func smbMemberAddresses(s interfaces.StateInterface) (map[string]string, error) { - cli, err := s.ClusterState().Connect().Leader(false) - if err != nil { - return nil, err + remotes := s.ClusterState().Truststore().RemoteAddresses() + addresses := make(map[string]string, len(remotes)) + for name, addrPort := range remotes { + addresses[name] = addrPort.Addr().String() } - return client.MClient.GetClusterMemberAddresses(cli) + return addresses, nil } // resolveSMBIface returns the local interface whose subnet contains the diff --git a/microceph/client/services.go b/microceph/client/services.go index 92be9280..dc80f82c 100644 --- a/microceph/client/services.go +++ b/microceph/client/services.go @@ -79,7 +79,7 @@ func SendServicePlacementReq(ctx context.Context, c mcTypes.Client, data *types. // ApplySMBSpec submits an SMBSpec JSON document for cluster-wide apply. func ApplySMBSpec(ctx context.Context, c mcTypes.Client, spec []byte) error { - queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + queryCtx, cancel := context.WithTimeout(ctx, time.Second*900) defer cancel() err := c.Query(queryCtx, "PUT", types.ExtendedPathPrefix, &api.NewURL().Path("services", "smb").URL, json.RawMessage(spec), nil) @@ -92,7 +92,7 @@ func ApplySMBSpec(ctx context.Context, c mcTypes.Client, spec []byte) error { // RemoveSMBService removes an smb cluster from all its member nodes. func RemoveSMBService(ctx context.Context, c mcTypes.Client, svc *types.SMBService) error { - queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + queryCtx, cancel := context.WithTimeout(ctx, time.Second*900) defer cancel() err := c.Query(queryCtx, "DELETE", types.ExtendedPathPrefix, &api.NewURL().Path("services", "smb").URL, svc, nil) @@ -120,7 +120,7 @@ func GetSMBServices(ctx context.Context, c mcTypes.Client) ([]types.SMBServiceSt // EnableSMBNodeService requests the target node run the smb placement flow. func EnableSMBNodeService(ctx context.Context, c mcTypes.Client, target string, data *types.EnableService) error { - queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + queryCtx, cancel := context.WithTimeout(ctx, time.Second*300) defer cancel() // Send this request to target. @@ -137,7 +137,7 @@ func EnableSMBNodeService(ctx context.Context, c mcTypes.Client, target string, // RegenerateSMBNodeService requests the target node re-render its smb // configs and restart ctdbd. func RegenerateSMBNodeService(ctx context.Context, c mcTypes.Client, target string, svc *types.SMBService) error { - queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + queryCtx, cancel := context.WithTimeout(ctx, time.Second*300) defer cancel() // Send this request to target. @@ -154,7 +154,7 @@ func RegenerateSMBNodeService(ctx context.Context, c mcTypes.Client, target stri // DeleteSMBNodeService requests the target node tear down its smb cluster // membership. func DeleteSMBNodeService(ctx context.Context, c mcTypes.Client, target string, svc *types.SMBService) error { - queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + queryCtx, cancel := context.WithTimeout(ctx, time.Second*300) defer cancel() // Send this request to target. diff --git a/microceph/client/wrap.go b/microceph/client/wrap.go index 4220097a..e8c7b25c 100644 --- a/microceph/client/wrap.go +++ b/microceph/client/wrap.go @@ -13,7 +13,6 @@ import ( // This is useful for mocking in unit tests type ClientInterface interface { GetClusterMembers(mcTypes.Client) ([]string, error) - GetClusterMemberAddresses(mcTypes.Client) (map[string]string, error) GetDisks(mcTypes.Client) (types.Disks, error) GetServices(mcTypes.Client) (types.Services, error) DeleteService(mcTypes.Client, string, string) error @@ -22,23 +21,6 @@ type ClientInterface interface { type ClientImpl struct{} -// GetClusterMemberAddresses gets a member name to host address mapping -// (ports stripped). -func (c ClientImpl) GetClusterMemberAddresses(cli mcTypes.Client) (map[string]string, error) { - var members []mcTypes.ClusterMember - err := cli.Query(context.Background(), "GET", mcTypes.PublicEndpoint, &api.NewURL().Path("cluster").URL, nil, &members) - if err != nil { - return nil, err - } - - addresses := make(map[string]string, len(members)) - for _, member := range members { - addresses[member.Name] = member.Address.Addr().String() - } - - return addresses, nil -} - // GetClusterMembers gets the cluster member names // We return names only here because the Member type is internal to microclient func (c ClientImpl) GetClusterMembers(cli mcTypes.Client) ([]string, error) { diff --git a/microceph/mocks/ClientInterface.go b/microceph/mocks/ClientInterface.go index b5d89f58..5db40142 100644 --- a/microceph/mocks/ClientInterface.go +++ b/microceph/mocks/ClientInterface.go @@ -43,32 +43,6 @@ func (_m *ClientInterface) DeleteService(_a0 mcTypes.Client, _a1 string, _a2 str return r0 } -// GetClusterMemberAddresses provides a mock function with given fields: _a0 -func (_m *ClientInterface) GetClusterMemberAddresses(_a0 mcTypes.Client) (map[string]string, error) { - ret := _m.Called(_a0) - - var r0 map[string]string - var r1 error - if rf, ok := ret.Get(0).(func(mcTypes.Client) (map[string]string, error)); ok { - return rf(_a0) - } - if rf, ok := ret.Get(0).(func(mcTypes.Client) map[string]string); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - - if rf, ok := ret.Get(1).(func(mcTypes.Client) error); ok { - r1 = rf(_a0) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetClusterMembers provides a mock function with given fields: _a0 func (_m *ClientInterface) GetClusterMembers(_a0 mcTypes.Client) ([]string, error) { ret := _m.Called(_a0) From ac19028a480ed3491915f645f81bceeb15f3bbe5 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 18:37:38 +0530 Subject: [PATCH 16/31] daemon: make the ctdb cluster lock line identical across nodes CTDB shuts a node down when its cluster lock command line differs from the leader's, and ours differed twice per node: the helper path used the revision-specific $SNAP directory (x8 vs x10 across nodes) and the lock entity was the per-node daemon key. The lock line now uses the /snap//current alias and a shared per-cluster entity (client.smb.) that every node ensures on enable; the reclock cap moves from the daemon keys to that entity, and RemoveSMB retires it after the last node leaves. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/smb.go | 6 +++ microceph/ceph/smb_config.go | 32 +++++++----- microceph/ceph/smb_config_test.go | 11 ++-- microceph/ceph/smb_keyring.go | 55 ++++++++++++++++---- microceph/ceph/smb_keyring_test.go | 22 ++++---- microceph/ceph/smb_lifecycle_test.go | 17 ++++-- microceph/ceph/smb_test.go | 5 ++ microceph/ceph/testdata/smb/ctdb.conf.golden | 2 +- 8 files changed, 106 insertions(+), 44 deletions(-) diff --git a/microceph/ceph/smb.go b/microceph/ceph/smb.go index eb21e346..75f479eb 100644 --- a/microceph/ceph/smb.go +++ b/microceph/ceph/smb.go @@ -217,6 +217,12 @@ func RemoveSMB(ctx context.Context, s interfaces.StateInterface, clusterID strin } } + // All nodes are gone: retire the shared cluster lock entity. + _, err = cephRun("auth", "del", SMBClusterEntity(clusterID)) + if err != nil { + logger.Warnf("failed to delete cephx entity '%s': %v", SMBClusterEntity(clusterID), err) + } + return nil } diff --git a/microceph/ceph/smb_config.go b/microceph/ceph/smb_config.go index c4f992cb..226c18e2 100644 --- a/microceph/ceph/smb_config.go +++ b/microceph/ceph/smb_config.go @@ -13,12 +13,15 @@ import ( ) // SMBPaths carries the snap path roots the renderers embed in configs. +// Snap is the revision-specific $SNAP dir; SnapStable is the /snap// +// current alias for content that must be identical across nodes. type SMBPaths struct { - Conf string - Run string - Data string - Log string - Snap string + Conf string + Run string + Data string + Log string + Snap string + SnapStable string } // SMBRenderParams carries per-node, per-cluster rendering inputs. @@ -40,11 +43,12 @@ func NewSMBRenderParams(clusterID, hostname string, clustered bool) SMBRenderPar Entity: SMBDaemonEntity(clusterID, hostname), Clustered: clustered, Paths: SMBPaths{ - Conf: pathConsts.ConfPath, - Run: pathConsts.RunPath, - Data: pathConsts.DataPath, - Log: pathConsts.LogPath, - Snap: strings.TrimRight(pathConsts.SnapPath, "/"), + Conf: pathConsts.ConfPath, + Run: pathConsts.RunPath, + Data: pathConsts.DataPath, + Log: pathConsts.LogPath, + Snap: strings.TrimRight(pathConsts.SnapPath, "/"), + SnapStable: filepath.Join("/snap", os.Getenv("SNAP_NAME"), "current"), }, } } @@ -194,7 +198,11 @@ func RenderCTDBConf(p SMBRenderParams, lockURI string) (string, error) { return "", fmt.Errorf("cannot derive cluster lock from cluster_lock_uri: %w", err) } - helper := filepath.Join(p.Paths.Snap, "libexec", "ctdb", "ctdb_mutex_ceph_rados_helper") + // CTDB refuses to run when the cluster lock command line differs + // between nodes, so it must avoid anything node-specific: the shared + // per-cluster entity (not the per-node daemon key) and the + // revision-independent snap path ($SNAP embeds the local revision). + helper := filepath.Join(p.Paths.SnapStable, "libexec", "ctdb", "ctdb_mutex_ceph_rados_helper") object := smbReclockObject + p.ClusterID dbDir := filepath.Join(p.Paths.Data, "ctdb") @@ -215,7 +223,7 @@ func RenderCTDBConf(p SMBRenderParams, lockURI string) (string, error) { cluster lock = !%s ceph %s %s %s `, filepath.Join(p.Paths.Log, "ctdb", "log.ctdb"), filepath.Join(dbDir, "volatile"), filepath.Join(dbDir, "persistent"), filepath.Join(dbDir, "state"), - helper, p.Entity, pool, object), nil + helper, SMBClusterEntity(p.ClusterID), pool, object), nil } // RenderCTDBNodes renders the nodes file: one private address per line. diff --git a/microceph/ceph/smb_config_test.go b/microceph/ceph/smb_config_test.go index f542974f..ebacd120 100644 --- a/microceph/ceph/smb_config_test.go +++ b/microceph/ceph/smb_config_test.go @@ -43,11 +43,12 @@ func testRenderParams() SMBRenderParams { Entity: "client.smb.dev.host1", Clustered: true, Paths: SMBPaths{ - Conf: "/var/snap/microceph/current/conf", - Run: "/var/snap/microceph/current/run", - Data: "/var/snap/microceph/common/data", - Log: "/var/snap/microceph/common/logs", - Snap: "/snap/microceph/current", + Conf: "/var/snap/microceph/current/conf", + Run: "/var/snap/microceph/current/run", + Data: "/var/snap/microceph/common/data", + Log: "/var/snap/microceph/common/logs", + Snap: "/snap/microceph/x1", + SnapStable: "/snap/microceph/current", }, } } diff --git a/microceph/ceph/smb_keyring.go b/microceph/ceph/smb_keyring.go index a274f14e..16418bd0 100644 --- a/microceph/ceph/smb_keyring.go +++ b/microceph/ceph/smb_keyring.go @@ -25,6 +25,18 @@ func SMBDaemonEntity(clusterID, hostname string) string { return fmt.Sprintf("client.smb.%s.%s", clusterID, hostname) } +// SMBClusterEntity returns the shared cephx entity all of a cluster's +// nodes use for the CTDB cluster lock: CTDB refuses to run when the lock +// command line differs between nodes, so it cannot embed per-node names. +func SMBClusterEntity(clusterID string) string { + return fmt.Sprintf("client.smb.%s", clusterID) +} + +// smbLockCaps returns the caps for the shared cluster lock entity. +func smbLockCaps(lockPool string) (string, string) { + return "allow r", fmt.Sprintf("allow rwx pool=%s object_prefix %s", lockPool, smbReclockObject) +} + // smbPoolCapsFromURI mirrors cephadm's SMBService._pool_caps_from_uri: // read access for foreign pools, and for the smb pool additionally rwx on // the cluster.meta. object prefix (the x perm locks the CTDB reclock @@ -73,16 +85,6 @@ func smbOSDCaps(spec *types.SMBSpec) string { } } - for _, feature := range spec.Features { - if feature != "clustered" { - continue - } - lockPool, _, _, err := parseRADOSURI(spec.ClusterLockURI) - if err == nil { - capSet[fmt.Sprintf("allow rwx pool=%s object_prefix %s", lockPool, smbReclockObject)] = true - } - } - caps := make([]string, 0, len(capSet)) for cap := range capSet { caps = append(caps, cap) @@ -172,6 +174,33 @@ func EnsureSMBKeyrings(spec *types.SMBSpec, hostname, confDir string) error { return err } + // Clustered specs share one lock entity across all nodes (the CTDB + // cluster lock command line must be identical cluster-wide). + for _, feature := range spec.Features { + if feature != "clustered" { + continue + } + lockPool, _, _, err := parseRADOSURI(spec.ClusterLockURI) + if err != nil { + return fmt.Errorf("cannot derive lock pool from cluster_lock_uri: %w", err) + } + + lockEntity := SMBClusterEntity(spec.ClusterID) + _, err = cephRun("auth", "get-or-create", lockEntity) + if err != nil { + return fmt.Errorf("failed to ensure cephx entity '%s': %w", lockEntity, err) + } + monCaps, osdCaps := smbLockCaps(lockPool) + _, err = cephRun("auth", "caps", lockEntity, "mon", monCaps, "osd", osdCaps) + if err != nil { + return fmt.Errorf("failed to set caps for '%s': %w", lockEntity, err) + } + err = fetchSMBKeyring(confDir, lockEntity) + if err != nil { + return err + } + } + for _, user := range spec.IncludeCephUsers { err = fetchSMBKeyring(confDir, user) if err != nil { @@ -198,7 +227,11 @@ func RemoveSMBKeyrings(spec *types.SMBSpec, hostname, confDir string) error { return fmt.Errorf("failed to delete cephx entity '%s': %w", entity, err) } - for _, name := range append([]string{entity}, spec.IncludeCephUsers...) { + // The shared lock entity stays in ceph while other nodes may use it + // (RemoveSMB deletes it after the last node leaves); only this node's + // fetched keyring files are removed. + names := append([]string{entity, SMBClusterEntity(spec.ClusterID)}, spec.IncludeCephUsers...) + for _, name := range names { err = os.Remove(smbKeyringPath(confDir, name)) if err != nil && !os.IsNotExist(err) { return err diff --git a/microceph/ceph/smb_keyring_test.go b/microceph/ceph/smb_keyring_test.go index 0f053192..d4b95df4 100644 --- a/microceph/ceph/smb_keyring_test.go +++ b/microceph/ceph/smb_keyring_test.go @@ -66,17 +66,15 @@ func (s *smbKeyringSuite) TestOSDCaps() { caps := smbOSDCaps(spec) assert.Equal(s.T(), "allow r pool=.smb, allow r pool=users, "+ - "allow rwx pool=.smb namespace=dev object_prefix cluster.meta., "+ - "allow rwx pool=.smb object_prefix microceph.reclock.", + "allow rwx pool=.smb namespace=dev object_prefix cluster.meta.", caps) } -func (s *smbKeyringSuite) TestOSDCapsUnclustered() { - spec := s.spec() - spec.Features = nil - - caps := smbOSDCaps(spec) - assert.NotContains(s.T(), caps, "microceph.reclock.") +func (s *smbKeyringSuite) TestClusterEntityAndLockCaps() { + assert.Equal(s.T(), "client.smb.dev", SMBClusterEntity("dev")) + monCaps, osdCaps := smbLockCaps(".smb") + assert.Equal(s.T(), "allow r", monCaps) + assert.Equal(s.T(), "allow rwx pool=.smb object_prefix microceph.reclock.", osdCaps) } func (s *smbKeyringSuite) TestMonCaps() { @@ -106,13 +104,17 @@ func (s *smbKeyringSuite) TestEnsureSMBKeyrings() { r.On("RunCommand", "ceph", "auth", "caps", entity, "mon", smbMonCaps("dev"), "osd", smbOSDCaps(spec)).Return("", nil).Once() writeKeyringOnGet(s, r, entity) + r.On("RunCommand", "ceph", "auth", "get-or-create", "client.smb.dev").Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "caps", "client.smb.dev", + "mon", "allow r", "osd", "allow rwx pool=.smb object_prefix microceph.reclock.").Return("", nil).Once() + writeKeyringOnGet(s, r, "client.smb.dev") writeKeyringOnGet(s, r, "client.data1") common.ProcessExec = r err := EnsureSMBKeyrings(spec, "host1", confDir) assert.NoError(s.T(), err) - for _, name := range []string{"ceph.client.smb.dev.host1.keyring", "ceph.client.data1.keyring"} { + for _, name := range []string{"ceph.client.smb.dev.host1.keyring", "ceph.client.smb.dev.keyring", "ceph.client.data1.keyring"} { info, err := os.Stat(filepath.Join(confDir, name)) assert.NoError(s.T(), err, name) assert.Equal(s.T(), os.FileMode(0600), info.Mode().Perm(), name) @@ -136,7 +138,7 @@ func (s *smbKeyringSuite) TestRemoveSMBKeyrings() { spec := s.spec() spec.IncludeCephUsers = []string{"client.data1"} - for _, name := range []string{"ceph.client.smb.dev.host1.keyring", "ceph.client.data1.keyring"} { + for _, name := range []string{"ceph.client.smb.dev.host1.keyring", "ceph.client.smb.dev.keyring", "ceph.client.data1.keyring"} { err := os.WriteFile(filepath.Join(confDir, name), []byte("k"), 0600) assert.NoError(s.T(), err) } diff --git a/microceph/ceph/smb_lifecycle_test.go b/microceph/ceph/smb_lifecycle_test.go index 776a4ce6..5998a4f5 100644 --- a/microceph/ceph/smb_lifecycle_test.go +++ b/microceph/ceph/smb_lifecycle_test.go @@ -70,11 +70,12 @@ func (s *smbLifecycleSuite) lifecycleEnv() SMBRenderParams { Entity: "client.smb.dev.host1", Clustered: true, Paths: SMBPaths{ - Conf: filepath.Join(root, "conf"), - Run: filepath.Join(root, "run"), - Data: filepath.Join(root, "data"), - Log: filepath.Join(root, "logs"), - Snap: snapDir, + Conf: filepath.Join(root, "conf"), + Run: filepath.Join(root, "run"), + Data: filepath.Join(root, "data"), + Log: filepath.Join(root, "logs"), + Snap: snapDir, + SnapStable: snapDir, }, } } @@ -114,6 +115,12 @@ func (s *smbLifecycleSuite) TestEnableSMBNodeLocal() { r.On("RunCommand", "ceph", "auth", "get", "client.smb.dev.host1", "-o", mock.Anything).Run(func(args mock.Arguments) { assert.NoError(s.T(), os.WriteFile(args.Get(5).(string), []byte("[client]\nkey=x\n"), 0600)) }).Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "get-or-create", "client.smb.dev").Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "caps", "client.smb.dev", + "mon", "allow r", "osd", "allow rwx pool=.smb object_prefix microceph.reclock.").Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "get", "client.smb.dev", "-o", mock.Anything).Run(func(args mock.Arguments) { + assert.NoError(s.T(), os.WriteFile(args.Get(5).(string), []byte("[client]\nkey=l\n"), 0600)) + }).Return("", nil).Once() r.On("RunCommand", "rados", "get", "--pool", ".smb", "-N", "dev", "scc.dev.json", "-"). Return(string(configJSON), nil).Once() r.On("RunCommand", "python3", "-m", "sambacc.commands.main", diff --git a/microceph/ceph/smb_test.go b/microceph/ceph/smb_test.go index bfb8cf61..aa8363cf 100644 --- a/microceph/ceph/smb_test.go +++ b/microceph/ceph/smb_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/common" "github.com/canonical/microceph/microceph/database" "github.com/canonical/microceph/microceph/interfaces" "github.com/canonical/microceph/microceph/mocks" @@ -260,6 +261,10 @@ func (s *smbSuite) TestRemoveSMB() { db := s.withDB() db.On("GetGroupMembers", context.Background(), s.TestStateInterface, "smb", "dev").Return([]string{"m1", "m2"}, nil).Once() + r := mocks.NewRunner(s.T()) + r.On("RunCommand", "ceph", "auth", "del", "client.smb.dev").Return("", nil).Once() + common.ProcessExec = r + err := RemoveSMB(context.Background(), s.TestStateInterface, "dev") assert.NoError(s.T(), err) assert.Equal(s.T(), []string{"m1", "m2"}, s.disabled) diff --git a/microceph/ceph/testdata/smb/ctdb.conf.golden b/microceph/ceph/testdata/smb/ctdb.conf.golden index 003e719b..ba2bdcf6 100644 --- a/microceph/ceph/testdata/smb/ctdb.conf.golden +++ b/microceph/ceph/testdata/smb/ctdb.conf.golden @@ -8,4 +8,4 @@ state database directory = /var/snap/microceph/common/data/ctdb/state [cluster] - cluster lock = !/snap/microceph/current/libexec/ctdb/ctdb_mutex_ceph_rados_helper ceph client.smb.dev.host1 .smb microceph.reclock.dev + cluster lock = !/snap/microceph/current/libexec/ctdb/ctdb_mutex_ceph_rados_helper ceph client.smb.dev .smb microceph.reclock.dev From 278f762d4b3c2cee81768d1d2bc3a56c51574b1b Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 19:36:24 +0530 Subject: [PATCH 17/31] orch: add smb service support Implement apply_smb and remove_service('smb.') in the microceph orchestrator module, forwarding the SMBSpec to microcephd over the unix-socket REST client (PUT /1.0/services/smb, DELETE on remove). ServiceSpec.to_json() nests subclass fields (cluster_id, config_uri, ...) under a 'spec' key; apply_smb flattens that into microcephd's flat SMBSpec wire format, which the daemon validates strictly (an unflattened payload fails with "cluster_id '' is not a valid ID"). Adds pytest coverage: conftest stubs the mgr runtime imports (mgr_module, orchestrator, ceph.deployment, snaphelpers) so the module imports outside a mgr, FakeSession asserts the wire format, and module routing tests exercise apply/remove dispatch with a fake spec that mirrors the real nested to_json() shape. pyproject gains a dev dependency group (pytest) and a hatchling build so the package is installable; .gitignore keeps __pycache__ and .venv out of the tree. uv run pytest tests/ -> 7 passed Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph-orch/.gitignore | 2 + microceph-orch/pyproject.toml | 12 +++ .../src/microceph/client/cluster.py | 12 +++ microceph-orch/src/microceph/module.py | 26 +++++ microceph-orch/tests/conftest.py | 85 +++++++++++++++++ microceph-orch/tests/test_smb_client.py | 94 +++++++++++++++++++ microceph-orch/tests/test_smb_module.py | 64 +++++++++++++ microceph-orch/uv.lock | 73 +++++++++++++- 8 files changed, 366 insertions(+), 2 deletions(-) create mode 100644 microceph-orch/.gitignore create mode 100644 microceph-orch/tests/conftest.py create mode 100644 microceph-orch/tests/test_smb_client.py create mode 100644 microceph-orch/tests/test_smb_module.py diff --git a/microceph-orch/.gitignore b/microceph-orch/.gitignore new file mode 100644 index 00000000..670a9362 --- /dev/null +++ b/microceph-orch/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +.venv/ diff --git a/microceph-orch/pyproject.toml b/microceph-orch/pyproject.toml index 41e2e24b..5ddff066 100644 --- a/microceph-orch/pyproject.toml +++ b/microceph-orch/pyproject.toml @@ -12,3 +12,15 @@ dependencies = [ [tool.uv.sources] snap-helpers = { git = "https://github.com/albertodonato/snap-helpers" } + +[dependency-groups] +dev = [ + "pytest>=8", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/microceph"] diff --git a/microceph-orch/src/microceph/client/cluster.py b/microceph-orch/src/microceph/client/cluster.py index 6601e999..5bad452f 100644 --- a/microceph-orch/src/microceph/client/cluster.py +++ b/microceph-orch/src/microceph/client/cluster.py @@ -67,6 +67,18 @@ def list_disks(self) -> list[dict]: disks = self._get("/1.0/disks") return disks.get("metadata") + def apply_smb(self, spec_json: str) -> None: + """Apply an SMBSpec JSON document cluster-wide.""" + self._put("/1.0/services/smb", data=spec_json) + + def remove_smb(self, cluster_id: str) -> None: + """Remove an smb cluster from all its member nodes.""" + self._delete("/1.0/services/smb", json={"cluster_id": cluster_id}) + + def list_smb(self) -> list[dict]: + """List smb clusters with their specs and placement.""" + return self._get("/1.0/services/smb").get("metadata") + def get_status(self) -> dict[str, dict]: """Get status of the cluster.""" cluster = self._get("/1.0/status") diff --git a/microceph-orch/src/microceph/module.py b/microceph-orch/src/microceph/module.py index 403c7f02..2ed3a1b4 100644 --- a/microceph-orch/src/microceph/module.py +++ b/microceph-orch/src/microceph/module.py @@ -16,6 +16,7 @@ MONSpec, MDSSpec, NFSServiceSpec, + SMBSpec, ) from mgr_module import MgrModule @@ -221,6 +222,9 @@ def list_daemons(self, info = json.loads(svc['info']) svc_ip = None if "0.0.0.0" in info['bind_address'] else info['bind_address'] svc_ports = [info['bind_port']] + + if svc_daemon_type == 'smb': + svc_ports = [445] descriptions.append(DaemonDescription( service_name=svc_name, @@ -256,6 +260,28 @@ def get_inventory(self, return inventory + @handle_orch_error + def apply_smb(self, spec: SMBSpec) -> str: + """Deploy the smb cluster described by an mgr/smb SMBSpec.""" + logger.info(f"applying smb spec for cluster {spec.cluster_id}") + # ServiceSpec.to_json() nests subclass fields (cluster_id, config_uri, + # ...) under a "spec" key; microcephd's SMBSpec wire format is flat. + data = dict(spec.to_json()) + data.update(data.pop('spec', {})) + self.microceph.services.apply_smb(json.dumps(data)) + return f"Scheduled smb.{spec.service_id} update..." + + @handle_orch_error + def remove_service(self, service_name: str, force: bool = False) -> str: + """Remove a service; only smb. services are supported.""" + svc_type, svc_id = self._elaborate_service(service_name) + if svc_type != 'smb' or not svc_id: + raise NotImplementedError(f"removing service {service_name} is not supported") + + logger.info(f"removing smb cluster {svc_id}") + self.microceph.services.remove_smb(svc_id) + return f"Removed service {service_name}" + def apply_rbd_mirror(self, spec: ServiceSpec) -> OrchResult[str]: logger.info(f"Received Apply Request for RBD Mirror: Spec: {vars(spec).items()}") raise NotImplementedError() diff --git a/microceph-orch/tests/conftest.py b/microceph-orch/tests/conftest.py new file mode 100644 index 00000000..d5b673b3 --- /dev/null +++ b/microceph-orch/tests/conftest.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 +# +# The mgr-runtime packages (mgr_module, orchestrator, ceph.deployment) +# only exist inside a ceph-mgr daemon; stub them so importing the +# microceph package (whose __init__ pulls in module.py) works under +# pytest. Also stub snaphelpers, which requires snap environment vars. + +import sys +import types + + +def _module(name, **attrs): + mod = types.ModuleType(name) + for key, value in attrs.items(): + setattr(mod, key, value) + sys.modules.setdefault(name, mod) + return sys.modules[name] + + +def _cls(name): + """A distinct permissive stub class per name (multiple inheritance + forbids reusing one class as several bases).""" + + def __init__(self, *args, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + return type( + name, + (object,), + { + "__init__": __init__, + # Tolerate generic annotations like OrchResult[str]. + "__class_getitem__": classmethod(lambda cls, item: cls), + }, + ) + + +def _identity_decorator(fn): + return fn + + +_module( + "ceph", +) +_module( + "ceph.deployment", +) +_module( + "ceph.deployment.inventory", + Device=_cls("Device"), + Devices=_cls("Devices"), +) +_module( + "ceph.deployment.service_spec", + ServiceSpec=_cls("ServiceSpec"), + PlacementSpec=_cls("PlacementSpec"), + RGWSpec=_cls("RGWSpec"), + MONSpec=_cls("MONSpec"), + MDSSpec=_cls("MDSSpec"), + NFSServiceSpec=_cls("NFSServiceSpec"), + SMBSpec=_cls("SMBSpec"), +) +_module( + "mgr_module", + MgrModule=_cls("MgrModule"), + NotifyType=_cls("NotifyType"), +) +_module( + "orchestrator", + Orchestrator=_cls("Orchestrator"), + HostSpec=_cls("HostSpec"), + InventoryFilter=_cls("InventoryFilter"), + InventoryHost=_cls("InventoryHost"), + ServiceDescription=_cls("ServiceDescription"), + DaemonDescription=_cls("DaemonDescription"), + CLICommandMeta=type, + handle_orch_error=_identity_decorator, + OrchResult=_cls("OrchResult"), +) +_module( + "snaphelpers", + Snap=_cls("Snap"), +) diff --git a/microceph-orch/tests/test_smb_client.py b/microceph-orch/tests/test_smb_client.py new file mode 100644 index 00000000..441fcebc --- /dev/null +++ b/microceph-orch/tests/test_smb_client.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json + +import pytest + +from microceph.client.cluster import ExtendedAPIService + + +class FakeResponse: + def __init__(self, payload=None, status=200): + self._payload = payload if payload is not None else {} + self.status = status + self.text = json.dumps(self._payload) + + def raise_for_status(self): + if self.status >= 400: + from requests.exceptions import HTTPError + + raise HTTPError(response=self) + + def json(self): + return self._payload + + +class FakeSession: + """Records requests and replays canned responses.""" + + def __init__(self, response=None): + self.calls = [] + self.response = response or FakeResponse() + + def request(self, method, url, **kwargs): + self.calls.append({"method": method, "url": url, **kwargs}) + return self.response + + +ENDPOINT = "http+unix://%2Fpath%2Fcontrol.socket" + + +@pytest.fixture +def session(): + return FakeSession() + + +@pytest.fixture +def service(session): + return ExtendedAPIService(session, ENDPOINT, None) + + +def test_apply_smb_puts_spec_json(service, session): + spec_json = '{"service_type": "smb", "cluster_id": "dev"}' + + service.apply_smb(spec_json) + + assert len(session.calls) == 1 + call = session.calls[0] + assert call["method"] == "put" + assert call["url"] == f"{ENDPOINT}/1.0/services/smb" + assert call["data"] == spec_json + + +def test_remove_smb_deletes_with_cluster_id(service, session): + service.remove_smb("dev") + + assert len(session.calls) == 1 + call = session.calls[0] + assert call["method"] == "delete" + assert call["url"] == f"{ENDPOINT}/1.0/services/smb" + assert call["json"] == {"cluster_id": "dev"} + + +def test_list_smb_returns_metadata(session): + session.response = FakeResponse( + {"metadata": [{"cluster_id": "dev", "placed_on": ["m1"]}]} + ) + service = ExtendedAPIService(session, ENDPOINT, None) + + statuses = service.list_smb() + + assert statuses == [{"cluster_id": "dev", "placed_on": ["m1"]}] + assert session.calls[0]["method"] == "get" + assert session.calls[0]["url"] == f"{ENDPOINT}/1.0/services/smb" + + +def test_apply_smb_surfaces_api_errors(session): + from requests.exceptions import HTTPError + + session.response = FakeResponse({"error": "field 'bind_addrs' is not supported in Phase 1"}, status=400) + service = ExtendedAPIService(session, ENDPOINT, None) + + with pytest.raises(HTTPError): + service.apply_smb("{}") diff --git a/microceph-orch/tests/test_smb_module.py b/microceph-orch/tests/test_smb_module.py new file mode 100644 index 00000000..134115d7 --- /dev/null +++ b/microceph-orch/tests/test_smb_module.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from microceph.module import MicroCephOrchestrator + + +class FakeSMBSpec: + service_id = "dev" + cluster_id = "dev" + + def to_json(self): + # Mirrors the real ServiceSpec.to_json(): subclass fields nested + # under "spec", not flat (verified against the packed mgr tree). + return { + "service_type": "smb", + "service_id": "dev", + "placement": {"hosts": ["m1"]}, + "spec": { + "cluster_id": "dev", + "config_uri": "rados://.smb/dev/scc.dev.json", + }, + } + + +@pytest.fixture +def orch(): + # Bypass __init__: it dials the microcephd unix socket. + instance = object.__new__(MicroCephOrchestrator) + instance.microceph = SimpleNamespace(services=MagicMock()) + return instance + + +def test_apply_smb_serializes_spec(orch): + result = orch.apply_smb(FakeSMBSpec()) + + orch.microceph.services.apply_smb.assert_called_once() + payload = json.loads(orch.microceph.services.apply_smb.call_args[0][0]) + assert payload["cluster_id"] == "dev" + assert payload["config_uri"] == "rados://.smb/dev/scc.dev.json" + assert "spec" not in payload + assert "smb.dev" in result + + +def test_remove_service_routes_smb(orch): + result = orch.remove_service("smb.dev") + + orch.microceph.services.remove_smb.assert_called_once_with("dev") + assert "smb.dev" in result + + +def test_remove_service_rejects_other_services(orch): + with pytest.raises(NotImplementedError): + orch.remove_service("nfs.foo") + + with pytest.raises(NotImplementedError): + orch.remove_service("smb") + + orch.microceph.services.remove_smb.assert_not_called() diff --git a/microceph-orch/uv.lock b/microceph-orch/uv.lock index 7a144d06..0dc9273c 100644 --- a/microceph-orch/uv.lock +++ b/microceph-orch/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" [[package]] @@ -46,6 +46,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" }, ] +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + [[package]] name = "idna" version = "3.10" @@ -55,10 +64,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "microceph-orch" version = "0.1.0" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "requests" }, { name = "requests-unixsocket" }, @@ -66,6 +84,11 @@ dependencies = [ { name = "urllib3" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "requests", specifier = ">=2.32.3" }, @@ -74,6 +97,52 @@ requires-dist = [ { name = "urllib3", specifier = ">=2.4.0" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8" }] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "pyyaml" version = "6.0.2" From d405a0b772abbce041b570cb9fb1f72d4797a3c5 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 19:50:16 +0530 Subject: [PATCH 18/31] daemon: map the smb group to ctdbd in startup re-enablement The grouped-service re-enable loop passes the group's service name to snapctl, but the smb group has no snap app of its own: ctdbd is the unit to start, and it brings up smbd via its event script. On every daemon start with an smb cluster in the DB this logged 'snapctl start microceph.smb: unknown service' and left an inactive smb node down after reboot. go test ./ceph/ -run Start -> PASS (incl. TestReEnableSMBGroupStartsCtdbd) Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/start.go | 12 ++++++++++-- microceph/ceph/start_test.go | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/microceph/ceph/start.go b/microceph/ceph/start.go index d1278713..9a8a765c 100644 --- a/microceph/ceph/start.go +++ b/microceph/ceph/start.go @@ -370,9 +370,17 @@ func reEnableServices(ctx context.Context, s interfaces.StateInterface) { continue } seen[gs.Service] = true - if err := snapCheckActive(gs.Service); err != nil { + // The smb group has no snap app of its own: ctdbd is the unit to + // (re)start, and it brings up smbd via its event script. + snapSvc := gs.Service + if gs.Service == "smb" { + snapSvc = "ctdbd" + } + err := snapCheckActive(snapSvc) + if err != nil { logger.Infof("start: re-enabling inactive grouped service %q", gs.Service) - if err := snapStart(gs.Service, true); err != nil { + err = snapStart(snapSvc, true) + if err != nil { logger.Warnf("start: failed to re-enable grouped service %q: %v", gs.Service, err) } } diff --git a/microceph/ceph/start_test.go b/microceph/ceph/start_test.go index caa9b1ee..f125af12 100644 --- a/microceph/ceph/start_test.go +++ b/microceph/ceph/start_test.go @@ -276,6 +276,22 @@ func (s *startSuite) TestReEnableGroupedServiceRestarted() { reEnableServices(context.Background(), s.newState()) } +func (s *startSuite) TestReEnableSMBGroupStartsCtdbd() { + r := s.setupReEnable( + []database.Service{{Service: "mon", Member: "node1"}}, + []database.GroupedService{ + {Service: "smb", GroupID: "dev", Member: "node1"}, + }, + ) + r.On("RunCommand", "snapctl", "services", "microceph.mon").Return("active", nil).Once() + r.On("RunCommand", "snapctl", "services", "microceph.osd").Return("active", nil).Once() + // The smb group maps to the ctdbd snap app (there is no microceph.smb). + r.On("RunCommand", "snapctl", "services", "microceph.ctdbd").Return("inactive", nil).Once() + r.On("RunCommand", "snapctl", "start", "microceph.ctdbd", "--enable").Return("ok", nil).Once() + + reEnableServices(context.Background(), s.newState()) +} + // TestShouldSkipMonitorRefresh is a regression test for issue #556. func (s *startSuite) TestShouldSkipMonitorRefresh() { // First run should always trigger UpdateConfig. From f35482d7a8515d7a4c6597c369cb7b92ccd2acfb Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 19:55:36 +0530 Subject: [PATCH 19/31] daemon: point cephfs-proxy rejections at samba-vfs/new mgr/smb expands the default 'samba-vfs' share provider to the proxied variant, so every default 'ceph smb share create' produces a spec with the cephfs-proxy feature and fails validation. Name the working provider in the error so users are not left guessing. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/service_placement_smb.go | 4 ++++ microceph/ceph/service_placement_smb_test.go | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/microceph/ceph/service_placement_smb.go b/microceph/ceph/service_placement_smb.go index 11edacaf..43df38cc 100644 --- a/microceph/ceph/service_placement_smb.go +++ b/microceph/ceph/service_placement_smb.go @@ -49,6 +49,10 @@ func (smb *SMBServicePlacement) PopulateParams(s interfaces.StateInterface, payl // CTDB clustering is the Phase 1 deployment model. case "domain": return fmt.Errorf("features: 'domain' (AD membership) is not supported in Phase 1") + case "cephfs-proxy": + return fmt.Errorf("features: 'cephfs-proxy' is not supported; create shares with " + + "--provider=samba-vfs/new (the default 'samba-vfs' provider expands to the " + + "proxied variant, which microceph does not deploy)") default: return fmt.Errorf("features: '%s' is not supported", feature) } diff --git a/microceph/ceph/service_placement_smb_test.go b/microceph/ceph/service_placement_smb_test.go index 5260e74d..f8ae74ba 100644 --- a/microceph/ceph/service_placement_smb_test.go +++ b/microceph/ceph/service_placement_smb_test.go @@ -88,6 +88,12 @@ func (s *servicePlacementSMBSuite) TestDomainFeatureRejected() { assert.ErrorContains(s.T(), err, "domain") } +func (s *servicePlacementSMBSuite) TestCephfsProxyFeatureRejectedWithHint() { + smb := &SMBServicePlacement{} + err := smb.PopulateParams(s.TestStateInterface, `{"cluster_id":"dev","features":["clustered","cephfs-proxy"]}`) + assert.ErrorContains(s.T(), err, "samba-vfs/new") +} + func (s *servicePlacementSMBSuite) TestUnknownFeatureRejected() { smb := &SMBServicePlacement{} err := smb.PopulateParams(s.TestStateInterface, `{"cluster_id":"dev","features":["wormholes"]}`) From 4b3160a9dee6fa5e2dd89b392ab9468a25fc0b66 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 20:13:39 +0530 Subject: [PATCH 20/31] daemon: seed smb passdb users from spec user_sources The enable flow rendered configs and started services but never imported user_sources into the passdb, leaving every mgr-created cluster with no way to authenticate (M3 seeded users by hand). ApplySMB now ends with a node-scoped seeding call on the first placed member: fetch each user_sources document (both rados:// pool objects and the rados:mon-config-key: scheme mgr/smb actually publishes), parse the sambacc users JSON and smbpasswd -s -a each entry against the rendered smb.conf, password over stdin. Seeding runs on every apply, not just spec changes, because the document behind a URI can change while the URI stays identical; the passdb is CTDB-replicated so one member covers the cluster. Imports retry while CTDB finishes recovery. Matching system users remain the admin's task in Phase 1. go test ./ceph/ ./api/... ./client/... -> ok Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/api/servers.go | 1 + microceph/api/services_smb.go | 26 ++++++ microceph/ceph/smb.go | 26 ++++++ microceph/ceph/smb_test.go | 35 ++++++++ microceph/ceph/smb_users.go | 132 +++++++++++++++++++++++++++++++ microceph/ceph/smb_users_test.go | 122 ++++++++++++++++++++++++++++ microceph/client/services.go | 18 +++++ 7 files changed, 360 insertions(+) create mode 100644 microceph/ceph/smb_users.go create mode 100644 microceph/ceph/smb_users_test.go diff --git a/microceph/api/servers.go b/microceph/api/servers.go index d9e9e654..a0874e48 100644 --- a/microceph/api/servers.go +++ b/microceph/api/servers.go @@ -27,6 +27,7 @@ var Servers = map[string]mcTypes.Server{ nfsServiceCmd, smbServiceCmd, smbNodeServiceCmd, + smbUsersServiceCmd, poolsOpCmd, rgwServiceCmd, rbdMirroServiceCmd, diff --git a/microceph/api/services_smb.go b/microceph/api/services_smb.go index dcbd7d67..366f5dbf 100644 --- a/microceph/api/services_smb.go +++ b/microceph/api/services_smb.go @@ -31,6 +31,32 @@ var smbNodeServiceCmd = mcTypes.Endpoint{ Delete: mcTypes.EndpointAction{Handler: cmdSMBNodeDelete, ProxyTarget: true}, } +// /1.0/services/smb/users endpoint: node-scoped passdb user seeding, +// invoked on one placed member per cluster-level apply (the passdb is +// CTDB-replicated). +var smbUsersServiceCmd = mcTypes.Endpoint{ + Path: "services/smb/users", + Put: mcTypes.EndpointAction{Handler: cmdSMBUsersPut, ProxyTarget: true}, +} + +// cmdSMBUsersPut seeds this node's clustered passdb from the SMBSpec +// (JSON body) user_sources. +func cmdSMBUsersPut(s mcTypes.State, r *http.Request) mcTypes.Response { + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + logger.Errorf("failed reading smb users spec body: %v", err) + return mcTypes.InternalError(err) + } + + err = ceph.SeedSMBUsersNode(string(body)) + if err != nil { + logger.Errorf("failed seeding smb users on node: %v", err) + return mcTypes.SmartError(err) + } + + return mcTypes.EmptySyncResponse +} + // cmdSMBNodePost regenerates this node's smb configs from the stored // spec and restarts ctdbd. func cmdSMBNodePost(s mcTypes.State, r *http.Request) mcTypes.Response { diff --git a/microceph/ceph/smb.go b/microceph/ceph/smb.go index 75f479eb..8e7e7960 100644 --- a/microceph/ceph/smb.go +++ b/microceph/ceph/smb.go @@ -30,6 +30,7 @@ var ( smbEnableNodeFunc = smbEnableNode smbDisableNodeFunc = smbDisableNode smbRegenerateNodeFunc = smbRegenerateNode + smbSeedUsersNodeFunc = smbSeedUsersNode ) // ResolveSMBPlacement resolves the spec placement to a sorted set of @@ -194,6 +195,17 @@ func ApplySMB(ctx context.Context, s interfaces.StateInterface, payload string) } } + // Seed passdb users on every apply, not just on spec changes: the + // documents behind user_sources URIs can change while the URIs (and + // so the spec) stay identical. The passdb is CTDB-replicated, so one + // member seeding covers the cluster. + if len(sp.Spec.UserSources) > 0 && len(desired) > 0 { + err = smbSeedUsersNodeFunc(ctx, s, desired[0], canonical) + if err != nil { + return fmt.Errorf("failed to seed smb users for cluster '%s' on node '%s': %w", sp.Spec.ClusterID, desired[0], err) + } + } + return nil } @@ -325,6 +337,20 @@ func smbRegenerateNode(ctx context.Context, s interfaces.StateInterface, node, c return client.RegenerateSMBNodeService(ctx, cli, node, &types.SMBService{ClusterID: clusterID}) } +// smbSeedUsersNode imports the spec's users into the clustered passdb on +// the given node, locally or via the node-scoped endpoint. +func smbSeedUsersNode(ctx context.Context, s interfaces.StateInterface, node, payload string) error { + if node == s.ClusterState().Name() { + return SeedSMBUsersNode(payload) + } + + cli, err := smbNodeClient(s, node) + if err != nil { + return err + } + return client.SeedSMBUsersNodeService(ctx, cli, node, payload) +} + // smbDisableNode tears down smb membership on the given node, locally or // via the node-scoped endpoint. func smbDisableNode(ctx context.Context, s interfaces.StateInterface, node, clusterID string) error { diff --git a/microceph/ceph/smb_test.go b/microceph/ceph/smb_test.go index aa8363cf..313948d1 100644 --- a/microceph/ceph/smb_test.go +++ b/microceph/ceph/smb_test.go @@ -35,6 +35,7 @@ type smbSuite struct { enabled []string disabled []string regenerated []string + seeded []string } func TestSMBSuite(t *testing.T) { @@ -50,16 +51,19 @@ func (s *smbSuite) SetupTest() { s.enabled = nil s.disabled = nil s.regenerated = nil + s.seeded = nil originalMembers := smbClusterMembersFunc originalEnable := smbEnableNodeFunc originalDisable := smbDisableNodeFunc originalRegenerate := smbRegenerateNodeFunc + originalSeed := smbSeedUsersNodeFunc s.T().Cleanup(func() { smbClusterMembersFunc = originalMembers smbEnableNodeFunc = originalEnable smbDisableNodeFunc = originalDisable smbRegenerateNodeFunc = originalRegenerate + smbSeedUsersNodeFunc = originalSeed }) smbClusterMembersFunc = func(s interfaces.StateInterface) ([]string, error) { @@ -77,6 +81,10 @@ func (s *smbSuite) SetupTest() { s.regenerated = append(s.regenerated, node) return nil } + smbSeedUsersNodeFunc = func(ctx context.Context, st interfaces.StateInterface, node, payload string) error { + s.seeded = append(s.seeded, node) + return nil + } } // withDB patches the grouped-services query with a fresh mock and restores @@ -242,6 +250,33 @@ func (s *smbSuite) TestApplyConfigChange() { assert.Equal(s.T(), []string{"m1", "m2"}, s.regenerated) } +func (s *smbSuite) TestApplySeedsUsersOnEveryApply() { + // Steady state, no config change: user seeding still runs, on the + // first placed member only. + payload := `{"service_type": "smb", "service_id": "dev", "cluster_id": "dev", ` + + `"placement": {"hosts": ["m1", "m2"]}, ` + + `"user_sources": ["rados:mon-config-key:smb/config/dev/users-groups.0.json"]}` + canonical := mustCompactJSON(payload) + + db := s.withDB() + db.On("GetGroupMembers", context.Background(), s.TestStateInterface, "smb", "dev").Return([]string{"m1", "m2"}, nil).Once() + db.On("GetGroupConfig", context.Background(), s.TestStateInterface, "smb", "dev").Return(canonical, nil).Once() + + err := ApplySMB(context.Background(), s.TestStateInterface, payload) + assert.NoError(s.T(), err) + assert.Empty(s.T(), s.regenerated) + assert.Equal(s.T(), []string{"m1"}, s.seeded) +} + +func (s *smbSuite) TestApplyNoUserSourcesSkipsSeeding() { + db := s.withDB() + db.On("GetGroupMembers", context.Background(), s.TestStateInterface, "smb", "dev").Return([]string{}, nil).Once() + + err := ApplySMB(context.Background(), s.TestStateInterface, smbPayload(`"m1"`, 0)) + assert.NoError(s.T(), err) + assert.Empty(s.T(), s.seeded) +} + func (s *smbSuite) TestApplyInvalidSpec() { err := ApplySMB(context.Background(), s.TestStateInterface, `{"cluster_id": "-bad-"}`) assert.ErrorIs(s.T(), err, ErrInvalidSMBSpec) diff --git a/microceph/ceph/smb_users.go b/microceph/ceph/smb_users.go new file mode 100644 index 00000000..646c1b65 --- /dev/null +++ b/microceph/ceph/smb_users.go @@ -0,0 +1,132 @@ +package ceph + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/logger" +) + +// smbMonConfigKeyPrefix is the URI scheme mgr/smb uses for user/group +// sources stored in the mon config-key store. +const smbMonConfigKeyPrefix = "rados:mon-config-key:" + +// Injectable seams for unit tests. +var ( + smbPasswdImportFunc = smbPasswdImport + smbUserImportRetries = 10 + smbUserImportInterval = 3 * time.Second + fetchSMBUserSourceFunc = fetchSMBUserSource +) + +// smbUsersDoc is the subset of the sambacc users-and-groups document the +// passdb import needs. +type smbUsersDoc struct { + Users struct { + AllEntries []struct { + Name string `json:"name"` + Password string `json:"password"` + } `json:"all_entries"` + } `json:"users"` +} + +// fetchSMBUserSource reads the document behind a user_sources URI: +// mgr/smb publishes them either as mon config-key entries or as RADOS +// pool objects. +func fetchSMBUserSource(uri string) ([]byte, error) { + key, found := strings.CutPrefix(uri, smbMonConfigKeyPrefix) + if found { + out, err := cephRun("config-key", "get", key) + if err != nil { + return nil, fmt.Errorf("failed to fetch '%s': %w", uri, err) + } + return []byte(out), nil + } + return fetchSMBConfigObject(uri) +} + +// smbPasswdImport adds (or re-adds) one user to the clustered passdb via +// smbpasswd against the rendered smb.conf. The password goes over stdin +// (-s reads it twice), never through argv. +func smbPasswdImport(confPath, name, password string) error { + cmd := exec.Command("smbpasswd", "-c", confPath, "-s", "-a", name) + cmd.Stdin = strings.NewReader(password + "\n" + password + "\n") + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("smbpasswd -a %s failed: %v (%s)", name, err, strings.TrimSpace(string(out))) + } + return nil +} + +// SeedSMBUsers imports every user from the spec's user_sources into the +// cluster's passdb. It must run on a placed member with ctdbd up: the +// passdb is CTDB-replicated, so one node seeding is cluster-wide. Each +// user needs a matching system user to already exist (Phase 1 leaves +// system-user provisioning to the admin). Imports retry while the CTDB +// cluster settles. +func SeedSMBUsers(spec *types.SMBSpec, p SMBRenderParams) error { + confPath := filepath.Join(p.Paths.Conf, "samba", "smb.conf") + + for _, uri := range spec.UserSources { + raw, err := fetchSMBUserSourceFunc(uri) + if err != nil { + return err + } + + var doc smbUsersDoc + err = json.Unmarshal(raw, &doc) + if err != nil { + return fmt.Errorf("cannot parse user source '%s': %w", uri, err) + } + + for _, user := range doc.Users.AllEntries { + err = importSMBUserWithRetry(confPath, user.Name, user.Password) + if err != nil { + return err + } + logger.Infof("seeded smb user '%s' for cluster '%s'", user.Name, spec.ClusterID) + } + } + + return nil +} + +// SeedSMBUsersNode is the env-wired entry point used by the node-scoped +// users endpoint: it parses the spec payload and seeds this node. +func SeedSMBUsersNode(payload string) error { + var spec types.SMBSpec + err := json.Unmarshal([]byte(payload), &spec) + if err != nil { + return fmt.Errorf("cannot parse smb spec for user seeding: %w", err) + } + + hostname, err := os.Hostname() + if err != nil { + return err + } + + return SeedSMBUsers(&spec, NewSMBRenderParams(spec.ClusterID, hostname, true)) +} + +// importSMBUserWithRetry retries the passdb import while ctdbd finishes +// recovery; smbpasswd fails against a ctdb-backed passdb until then. +func importSMBUserWithRetry(confPath, name, password string) error { + var err error + for attempt := 0; attempt < smbUserImportRetries; attempt++ { + if attempt > 0 { + time.Sleep(smbUserImportInterval) + } + err = smbPasswdImportFunc(confPath, name, password) + if err == nil { + return nil + } + logger.Infof("smb user import attempt %d for '%s' failed: %v", attempt+1, name, err) + } + return fmt.Errorf("failed to import smb user '%s': %w", name, err) +} diff --git a/microceph/ceph/smb_users_test.go b/microceph/ceph/smb_users_test.go new file mode 100644 index 00000000..9442bf90 --- /dev/null +++ b/microceph/ceph/smb_users_test.go @@ -0,0 +1,122 @@ +package ceph + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microceph/microceph/mocks" + "github.com/canonical/microceph/microceph/tests" +) + +type smbUsersSuite struct { + tests.BaseSuite +} + +func TestSMBUsersSuite(t *testing.T) { + suite.Run(t, new(smbUsersSuite)) +} + +// stubImports replaces the passdb import seam with a recorder and makes +// retries immediate. +func (s *smbUsersSuite) stubImports(fail int) *[][3]string { + calls := &[][3]string{} + + originalImport := smbPasswdImportFunc + originalRetries := smbUserImportRetries + originalInterval := smbUserImportInterval + s.T().Cleanup(func() { + smbPasswdImportFunc = originalImport + smbUserImportRetries = originalRetries + smbUserImportInterval = originalInterval + }) + + smbUserImportRetries = 3 + smbUserImportInterval = time.Duration(0) + smbPasswdImportFunc = func(confPath, name, password string) error { + *calls = append(*calls, [3]string{confPath, name, password}) + if len(*calls) <= fail { + return fmt.Errorf("ctdb not ready") + } + return nil + } + + return calls +} + +func (s *smbUsersSuite) stubUserSource(doc string) { + original := fetchSMBUserSourceFunc + s.T().Cleanup(func() { fetchSMBUserSourceFunc = original }) + fetchSMBUserSourceFunc = func(uri string) ([]byte, error) { + return []byte(doc), nil + } +} + +func (s *smbUsersSuite) TestSeedImportsAllUsers() { + calls := s.stubImports(0) + s.stubUserSource(`{"samba-container-config": "v0", "users": {"all_entries": [` + + `{"name": "alice", "password": "pw1"}, {"name": "bob", "password": "pw2"}]}}`) + + spec := &types.SMBSpec{ClusterID: "dev", UserSources: []string{"rados:mon-config-key:smb/config/dev/users-groups.0.json"}} + err := SeedSMBUsers(spec, NewSMBRenderParams("dev", "m1", true)) + assert.NoError(s.T(), err) + + assert.Len(s.T(), *calls, 2) + assert.Equal(s.T(), "alice", (*calls)[0][1]) + assert.Equal(s.T(), "pw1", (*calls)[0][2]) + assert.Equal(s.T(), "bob", (*calls)[1][1]) +} + +func (s *smbUsersSuite) TestSeedRetriesWhileCTDBSettles() { + calls := s.stubImports(2) + s.stubUserSource(`{"users": {"all_entries": [{"name": "alice", "password": "pw"}]}}`) + + spec := &types.SMBSpec{ClusterID: "dev", UserSources: []string{"rados://.smb/dev/users.json"}} + err := SeedSMBUsers(spec, NewSMBRenderParams("dev", "m1", true)) + assert.NoError(s.T(), err) + assert.Len(s.T(), *calls, 3) +} + +func (s *smbUsersSuite) TestSeedFailsAfterRetriesExhausted() { + calls := s.stubImports(99) + s.stubUserSource(`{"users": {"all_entries": [{"name": "alice", "password": "pw"}]}}`) + + spec := &types.SMBSpec{ClusterID: "dev", UserSources: []string{"rados://.smb/dev/users.json"}} + err := SeedSMBUsers(spec, NewSMBRenderParams("dev", "m1", true)) + assert.ErrorContains(s.T(), err, "alice") + assert.Len(s.T(), *calls, 3) +} + +func (s *smbUsersSuite) TestSeedRejectsBadDocument() { + s.stubImports(0) + s.stubUserSource(`not json`) + + spec := &types.SMBSpec{ClusterID: "dev", UserSources: []string{"rados://.smb/dev/users.json"}} + err := SeedSMBUsers(spec, NewSMBRenderParams("dev", "m1", true)) + assert.ErrorContains(s.T(), err, "cannot parse user source") +} + +func (s *smbUsersSuite) TestFetchDispatchesMonConfigKey() { + r := mocks.NewRunner(s.T()) + common.ProcessExec = r + r.On("RunCommand", "ceph", "config-key", "get", "smb/config/dev/users-groups.0.json").Return(`{"users": {}}`, nil).Once() + + out, err := fetchSMBUserSource("rados:mon-config-key:smb/config/dev/users-groups.0.json") + assert.NoError(s.T(), err) + assert.Equal(s.T(), `{"users": {}}`, string(out)) +} + +func (s *smbUsersSuite) TestFetchDispatchesRADOSURI() { + r := mocks.NewRunner(s.T()) + common.ProcessExec = r + r.On("RunCommand", "rados", "get", "--pool", ".smb", "-N", "dev", "users.json", "-").Return(`{"users": {}}`, nil).Once() + + out, err := fetchSMBUserSource("rados://.smb/dev/users.json") + assert.NoError(s.T(), err) + assert.Equal(s.T(), `{"users": {}}`, string(out)) +} diff --git a/microceph/client/services.go b/microceph/client/services.go index dc80f82c..96006c52 100644 --- a/microceph/client/services.go +++ b/microceph/client/services.go @@ -151,6 +151,24 @@ func RegenerateSMBNodeService(ctx context.Context, c mcTypes.Client, target stri return nil } +// SeedSMBUsersNodeService requests the target node seed its clustered +// passdb from the spec's user_sources. Sized to cover the import retry +// window while CTDB settles. +func SeedSMBUsersNodeService(ctx context.Context, c mcTypes.Client, target string, spec string) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + defer cancel() + + // Send this request to target. + c = c.UseTarget(target) + + err := c.Query(queryCtx, "PUT", types.ExtendedPathPrefix, &api.NewURL().Path("services", "smb", "users").URL, json.RawMessage(spec), nil) + if err != nil { + return fmt.Errorf("failed seeding smb users on %s: %w", target, err) + } + + return nil +} + // DeleteSMBNodeService requests the target node tear down its smb cluster // membership. func DeleteSMBNodeService(ctx context.Context, c mcTypes.Client, target string, svc *types.SMBService) error { From 40542626b7e0438d1b00428a8cae2bb29614ddca Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 20:22:33 +0530 Subject: [PATCH 21/31] daemon: converge smb keyrings on regenerate A spec change with steady membership only regenerates: enable is the sole place keyrings were fetched, so entities added by later spec revisions never reached the nodes. Concretely, the first share on an mgr-created cluster adds client.smb.fs.cluster. (the vfs_ceph user) to include_ceph_users, the regenerated smb.conf references it, and every tree connect fails NT_STATUS_UNSUCCESSFUL on the missing keyring. EnsureSMBKeyrings is idempotent and also re-converges URI-derived caps, so regenerate now runs it before rendering. go test ./ceph/ -> ok Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/smb_lifecycle.go | 8 ++++++++ microceph/ceph/smb_lifecycle_test.go | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/microceph/ceph/smb_lifecycle.go b/microceph/ceph/smb_lifecycle.go index dff92b81..bae0ce20 100644 --- a/microceph/ceph/smb_lifecycle.go +++ b/microceph/ceph/smb_lifecycle.go @@ -336,6 +336,14 @@ func regenerateSMBNodeLocal(ctx context.Context, s interfaces.StateInterface, cl return fmt.Errorf("cannot parse stored spec for smb cluster '%s': %w", clusterID, err) } + // Spec changes can add include_ceph_users entities (e.g. the first + // share creating the cluster's cephfs user) or alter URI-derived + // caps, so keyrings must converge on regenerate, not just enable. + err = EnsureSMBKeyrings(&spec, p.Hostname, p.Paths.Conf) + if err != nil { + return err + } + ips, err := smbOrderedNodeIPs(ctx, s, clusterID, p.Hostname) if err != nil { return err diff --git a/microceph/ceph/smb_lifecycle_test.go b/microceph/ceph/smb_lifecycle_test.go index 5998a4f5..a0f39db0 100644 --- a/microceph/ceph/smb_lifecycle_test.go +++ b/microceph/ceph/smb_lifecycle_test.go @@ -213,7 +213,27 @@ func (s *smbLifecycleSuite) TestRegenerateSMBNodeLocal() { db.On("GetGroupConfig", ctx, s.TestStateInterface, "smb", "dev").Return(canonical, nil).Once() db.On("GetGroupMemberRecords", ctx, s.TestStateInterface, "smb", "dev").Return(s.memberRecords(), nil).Once() + var spec types.SMBSpec + assert.NoError(s.T(), json.Unmarshal([]byte(validSMBPayload), &spec)) + + // Regenerate assumes the node dirs from enable already exist. + assert.NoError(s.T(), os.MkdirAll(filepath.Join(p.Paths.Conf, "samba"), 0755)) + r := mocks.NewRunner(s.T()) + // Keyrings converge on regenerate too (spec changes can add + // include_ceph_users or alter URI-derived caps). + r.On("RunCommand", "ceph", "auth", "get-or-create", "client.smb.dev.host1").Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "caps", "client.smb.dev.host1", + "mon", smbMonCaps("dev"), "osd", smbOSDCaps(&spec)).Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "get", "client.smb.dev.host1", "-o", mock.Anything).Run(func(args mock.Arguments) { + assert.NoError(s.T(), os.WriteFile(args.Get(5).(string), []byte("[client]\nkey=x\n"), 0600)) + }).Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "get-or-create", "client.smb.dev").Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "caps", "client.smb.dev", + "mon", "allow r", "osd", "allow rwx pool=.smb object_prefix microceph.reclock.").Return("", nil).Once() + r.On("RunCommand", "ceph", "auth", "get", "client.smb.dev", "-o", mock.Anything).Run(func(args mock.Arguments) { + assert.NoError(s.T(), os.WriteFile(args.Get(5).(string), []byte("[client]\nkey=l\n"), 0600)) + }).Return("", nil).Once() r.On("RunCommand", "rados", "get", "--pool", ".smb", "-N", "dev", "scc.dev.json", "-"). Return(string(configJSON), nil).Once() r.On("RunCommand", "python3", "-m", "sambacc.commands.main", From f64da2f7320bb2ceeb26903221402bc01fd20684 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 20:34:26 +0530 Subject: [PATCH 22/31] daemon: anchor CTDB_BASE symlinks at the stable snap path populateCTDBBase linked event scripts, functions and notify.sh into the revisioned $SNAP dir. snapd keeps only two revisions, so the second refresh after an enable garbage-collects the link targets and ctdbd crash-loops on 'Failed to run init event'. Link through /snap//current instead (SnapStable, already used for the cluster lock helper line). The lifecycle test env now keeps Snap and SnapStable distinct so a regression fails the symlink assertions. go test ./ceph/ -> ok Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/smb_lifecycle.go | 5 ++++- microceph/ceph/smb_lifecycle_test.go | 12 +++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/microceph/ceph/smb_lifecycle.go b/microceph/ceph/smb_lifecycle.go index bae0ce20..e99461be 100644 --- a/microceph/ceph/smb_lifecycle.go +++ b/microceph/ceph/smb_lifecycle.go @@ -230,7 +230,10 @@ func enableSMBNodeLocal(ctx context.Context, s interfaces.StateInterface, spec * return err } - err = populateCTDBBase(filepath.Join(p.Paths.Conf, "ctdb"), p.Paths.Snap) + // Symlink targets must ride the current symlink, not the revisioned + // $SNAP dir: snapd garbage-collects old revisions on refresh, which + // would strand every CTDB_BASE link and crash-loop ctdbd. + err = populateCTDBBase(filepath.Join(p.Paths.Conf, "ctdb"), p.Paths.SnapStable) if err != nil { return err } diff --git a/microceph/ceph/smb_lifecycle_test.go b/microceph/ceph/smb_lifecycle_test.go index a0f39db0..4fce1321 100644 --- a/microceph/ceph/smb_lifecycle_test.go +++ b/microceph/ceph/smb_lifecycle_test.go @@ -70,11 +70,13 @@ func (s *smbLifecycleSuite) lifecycleEnv() SMBRenderParams { Entity: "client.smb.dev.host1", Clustered: true, Paths: SMBPaths{ - Conf: filepath.Join(root, "conf"), - Run: filepath.Join(root, "run"), - Data: filepath.Join(root, "data"), - Log: filepath.Join(root, "logs"), - Snap: snapDir, + Conf: filepath.Join(root, "conf"), + Run: filepath.Join(root, "run"), + Data: filepath.Join(root, "data"), + Log: filepath.Join(root, "logs"), + // Distinct on purpose: CTDB_BASE symlinks must target the + // stable path, so only SnapStable holds the fake snap tree. + Snap: filepath.Join(root, "snap-revisioned"), SnapStable: snapDir, }, } From 408c08dd32dfdc632362a59408bc81ca2588af50 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 20:43:25 +0530 Subject: [PATCH 23/31] orch: describe smb services with their stored SMBSpec ceph orch ls constructs a spec per service; a generic ServiceSpec with service_type='smb' dispatches to SMBSpec through the service registry and fails validation without a cluster_id, breaking orch ls for every deployment with an smb cluster. describe_service now pulls the verbatim specs microcephd stores (one list_smb call when any smb service is recorded) and rebuilds them via ServiceSpec.from_json, skipping clusters with no stored spec. uv run pytest tests/ -> 9 passed Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph-orch/src/microceph/module.py | 18 ++++++++++- microceph-orch/tests/conftest.py | 3 ++ microceph-orch/tests/test_smb_module.py | 41 +++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/microceph-orch/src/microceph/module.py b/microceph-orch/src/microceph/module.py index 2ed3a1b4..e5ace72b 100644 --- a/microceph-orch/src/microceph/module.py +++ b/microceph-orch/src/microceph/module.py @@ -167,6 +167,17 @@ def describe_service(self, recorded_services = self.microceph.services.list_services() service_hostlist = self._get_service_hostlist(recorded_services) + # smb specs are stored verbatim in microcephd; reuse them so the + # description carries a valid SMBSpec (a generic ServiceSpec with + # service_type='smb' dispatches to SMBSpec and fails validation + # without cluster_id). + smb_specs = {} + if any(name.split('.')[0] == 'smb' for name in service_hostlist): + try: + smb_specs = {st['cluster_id']: st['spec'] for st in self.microceph.services.list_smb()} + except RemoteException as e: + logger.warning(f"failed to fetch smb specs: {e}") + service_descs = [] for svc_name, hostlist in service_hostlist.items(): spec = None @@ -177,7 +188,12 @@ def describe_service(self, if service_type and svc_type != service_type: continue - if svc_type in daemon_spec_map: + if svc_type == 'smb': + if svc_id not in smb_specs: + logger.warning(f"no stored spec for smb cluster '{svc_id}'; skipping") + continue + spec = ServiceSpec.from_json(smb_specs[svc_id]) + elif svc_type in daemon_spec_map: spec = daemon_spec_map[svc_type]( service_id=svc_id, service_type=svc_type, placement=PlacementSpec(hosts=hostlist, count=len(hostlist)) ) diff --git a/microceph-orch/tests/conftest.py b/microceph-orch/tests/conftest.py index d5b673b3..dfdfb82b 100644 --- a/microceph-orch/tests/conftest.py +++ b/microceph-orch/tests/conftest.py @@ -33,6 +33,9 @@ def __init__(self, *args, **kwargs): "__init__": __init__, # Tolerate generic annotations like OrchResult[str]. "__class_getitem__": classmethod(lambda cls, item: cls), + # Mirror ServiceSpec.from_json: build an instance carrying the + # document's keys as attributes. + "from_json": classmethod(lambda cls, data: cls(**data)), }, ) diff --git a/microceph-orch/tests/test_smb_module.py b/microceph-orch/tests/test_smb_module.py index 134115d7..995d4c81 100644 --- a/microceph-orch/tests/test_smb_module.py +++ b/microceph-orch/tests/test_smb_module.py @@ -58,6 +58,47 @@ def test_remove_service_rejects_other_services(orch): with pytest.raises(NotImplementedError): orch.remove_service("nfs.foo") + +def test_describe_service_uses_stored_smb_spec(orch): + orch.microceph.services.list_services.return_value = [ + {"service": "smb", "group_id": "dev", "location": "m1", "info": "{}"}, + {"service": "smb", "group_id": "dev", "location": "m2", "info": "{}"}, + ] + orch.microceph.services.list_smb.return_value = [ + { + "cluster_id": "dev", + "spec": { + "service_type": "smb", + "service_id": "dev", + "cluster_id": "dev", + "config_uri": "rados://.smb/dev/config.smb", + "placement": {"count": 2}, + }, + "placed_on": ["m1", "m2"], + } + ] + + descs = orch.describe_service() + + assert len(descs) == 1 + desc = descs[0] + # A generic ServiceSpec with service_type='smb' dispatches to SMBSpec + # and fails validation; the stored spec must be used instead. + assert desc.spec.cluster_id == "dev" + assert desc.spec.config_uri == "rados://.smb/dev/config.smb" + assert desc.running == 2 + + +def test_describe_service_skips_smb_without_stored_spec(orch): + orch.microceph.services.list_services.return_value = [ + {"service": "smb", "group_id": "ghost", "location": "m1", "info": "{}"}, + ] + orch.microceph.services.list_smb.return_value = [] + + descs = orch.describe_service() + + assert descs == [] + with pytest.raises(NotImplementedError): orch.remove_service("smb") From ed7462ce9aa075b06a35e966a9cfe0f5827b512a Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 22:21:18 +0530 Subject: [PATCH 24/31] tests: add smb-tests robot suite Covers the Phase 1 mgr-driven lifecycle on a 3-node inner cluster: orchestrator backend enablement, cluster+share creation via ceph smb (declarative apply for the share, since the imperative default provider expands to the proxied vfs), roundtrip via a CTDB public address with smbclient, VIP-holder kill/failover/rejoin, and cluster removal. The snap is re-installed in devmode by the suite (confined smbd needs the pending smb-support snapd interface). Per the harness rules, decisions parse in Python: smb_ops.py carries the pure helpers (VIP derivation from the cluster CIDR, ctdb -X status health count, VIP-to-pnn lookup, share spec document) with pytest coverage, and the harness class gains the ctdb exec wrapper, a _poll_until-based health waiter and the devmode reinstall step. tox -e robot -- --dryrun --test-suite smb-tests -> 5 tests resolve pytest test_harness_helpers.py -> 116 passed Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- tests/robot/resources/microceph_harness.py | 54 +++++++ .../resources/microceph_harness.resource | 1 + tests/robot/resources/smb_ops.py | 80 ++++++++++ tests/robot/resources/test_harness_helpers.py | 70 +++++++++ tests/robot/smb-tests/smb_tests.robot | 137 ++++++++++++++++++ 5 files changed, 342 insertions(+) create mode 100644 tests/robot/resources/smb_ops.py create mode 100644 tests/robot/smb-tests/smb_tests.robot diff --git a/tests/robot/resources/microceph_harness.py b/tests/robot/resources/microceph_harness.py index 98baf1cf..0d13664d 100644 --- a/tests/robot/resources/microceph_harness.py +++ b/tests/robot/resources/microceph_harness.py @@ -28,6 +28,7 @@ rbd_primary_image_count, rbd_synced_image_count, ) +from smb_ops import ctdb_ok_node_count from snap_services import enabled_active_services from streaming_process import run_streaming_process @@ -894,6 +895,59 @@ def predicate(): fail_msg=f"CephFS snaps_synced for {vol} never reached {threshold} after {attempts} attempts", ) + # ----------------------------------------------------------------------- + # SMB / CTDB helpers + # ----------------------------------------------------------------------- + + # The snap ships no ctdb CLI app, so the binary needs the snap's library + # path and the revision-stable socket location spelled out. + _CTDB_ENV = ( + "LD_LIBRARY_PATH=/snap/microceph/current/lib" + ":/snap/microceph/current/lib/x86_64-linux-gnu" + ":/snap/microceph/current/lib/x86_64-linux-gnu/samba" + " CTDB_SOCKET=/var/snap/microceph/current/run/ctdb/ctdbd.socket" + ) + + def run_ctdb_in_node(self, container, args): + """Runs ``ctdb `` inside *container* against the snap's ctdbd. + + Returns the result OBJECT (non-raising): callers decide on rc/stdout, + and pollers treat a connection failure as "not ready yet". + """ + cmd = f"{self._CTDB_ENV} /snap/microceph/current/bin/ctdb {args}" + return self.run_in_container_unchecked(container, cmd, 30) + + def wait_for_ctdb_healthy(self, container, expected_nodes, attempts=30): + """Polls until ``ctdb -X status`` on *container* reports *expected_nodes* healthy nodes.""" + def predicate(): + out = self.run_ctdb_in_node(container, "-X status").stdout + return ctdb_ok_node_count(out) >= int(expected_nodes) + + self._poll_until( + predicate, + attempts=attempts, + interval=10, + fail_msg=f"CTDB never reached {expected_nodes} healthy nodes after {attempts} attempts", + ) + + def get_ctdb_vip_output(self, container): + """Returns the stdout of ``ctdb ip`` on *container* (VIP -> pnn table).""" + return self.run_ctdb_in_node(container, "ip").stdout + + def reinstall_snap_devmode_on_all_nodes(self): + """Re-installs the pre-baked local snap with --devmode on all inner nodes. + + Confined smbd panics on setgroups (no smb-support interface yet), so + the smb suite runs the snap in devmode; see the Phase 1 design doc. + """ + logger.console("[smb] Re-installing snap in devmode on all nodes...") + for container in NODES: + self.run_in_container_and_check( + container, f"sudo snap install --dangerous --devmode {MNT_SNAP_GLOB}", 600 + ) + # Give the daemons a moment to settle before the suite polls health. + time.sleep(15) + # ----------------------------------------------------------------------- # File / snap-mount helpers # ----------------------------------------------------------------------- diff --git a/tests/robot/resources/microceph_harness.resource b/tests/robot/resources/microceph_harness.resource index 4a3a5e8a..1cdbdef9 100644 --- a/tests/robot/resources/microceph_harness.resource +++ b/tests/robot/resources/microceph_harness.resource @@ -9,6 +9,7 @@ Library microceph_harness.py Library streaming_process.py Library snap_services.py Library cephfs_replication.py +Library smb_ops.py *** Variables *** ${SNAP_PATH} ${EMPTY} diff --git a/tests/robot/resources/smb_ops.py b/tests/robot/resources/smb_ops.py new file mode 100644 index 00000000..0336c5a4 --- /dev/null +++ b/tests/robot/resources/smb_ops.py @@ -0,0 +1,80 @@ +"""Robot Framework library: pure helpers for the smb-tests suite. + +Follows the "fetch raw, decide in Python" harness rule: the suite runs the +minimum remote command (ctdb -X status, ctdb ip) and every parse/decision +lives here, unit-tested in test_harness_helpers.py with no LXD. +""" + +import ipaddress + + +def smb_vip_addresses(cidr, count, base_offset=200): + """Return *count* CTDB public addresses ("IP/prefix") from *cidr*. + + VIPs are taken from a high host offset (default .200 upwards) so they do + not collide with the DHCP range LXD hands to the inner containers. Raises + ValueError when the request walks past the last usable host address. + """ + network = ipaddress.ip_network(cidr, strict=False) + base = int(network.network_address) + addresses = [] + for i in range(count): + candidate = ipaddress.ip_address(base + base_offset + i) + if candidate not in network or candidate == network.broadcast_address: + raise ValueError(f"VIP offset {base_offset + i} is outside {cidr}") + addresses.append(f"{candidate}/{network.prefixlen}") + return addresses + + +def ctdb_ok_node_count(xstatus_text): + """Return the number of healthy nodes in ``ctdb -X status`` output. + + The machine-readable format is one header row plus one row per node: + |Node|IP|Disconnected|Unknown|Banned|Disabled|Unhealthy|Stopped|Inactive|PartiallyOnline|ThisNode| + A node is healthy when every flag column (Disconnected through + PartiallyOnline) is 0. + """ + count = 0 + for line in xstatus_text.splitlines(): + cols = line.strip().strip("|").split("|") + if len(cols) < 11 or cols[0] == "Node": + continue + if all(flag == "0" for flag in cols[2:10]): + count += 1 + return count + + +def ctdb_vip_pnn(ip_output, vip): + """Return the node number hosting *vip* from ``ctdb ip`` output, or -1. + + The plain format is a "Public IPs on node N" header followed by + "
" lines; *vip* may be given with or without a /prefix. + """ + address = vip.split("/")[0] + for line in ip_output.splitlines(): + cols = line.split() + if len(cols) == 2 and cols[0] == address: + try: + return int(cols[1]) + except ValueError: + return -1 + return -1 + + +def smb_share_spec_yaml(cluster_id, share_id, volume, subvolume): + """Return a declarative ceph.smb.share document for ``ceph smb apply``. + + The provider must be spelled samba-vfs/new: the imperative + ``ceph smb share create`` defaults to plain samba-vfs, which mgr/smb + expands to the proxied variant that microceph rejects. + """ + return ( + "resource_type: ceph.smb.share\n" + f"cluster_id: {cluster_id}\n" + f"share_id: {share_id}\n" + "cephfs:\n" + f" volume: {volume}\n" + f" subvolume: {subvolume}\n" + " path: /\n" + " provider: samba-vfs/new\n" + ) diff --git a/tests/robot/resources/test_harness_helpers.py b/tests/robot/resources/test_harness_helpers.py index 9bb1627c..6ed30b17 100644 --- a/tests/robot/resources/test_harness_helpers.py +++ b/tests/robot/resources/test_harness_helpers.py @@ -15,6 +15,12 @@ import placement_status from microceph_harness import microceph_harness as H from cluster_ops import parse_migration_status +from smb_ops import ( + ctdb_ok_node_count, + ctdb_vip_pnn, + smb_share_spec_yaml, + smb_vip_addresses, +) from snap_services import enabled_active_services from cephfs_replication import cephfs_replication_list_has_volume, verify_cephfs_list_entry_types from rbd_replication import ( @@ -960,3 +966,67 @@ def test_member_in_ceph_status_substring(): assert placement_status.member_in_ceph_status(status, "node-wrk3") is False assert placement_status.member_in_ceph_status("", "node-wrk0") is False assert placement_status.member_in_ceph_status(None, "node-wrk0") is False + + +# --- smb_ops ----------------------------------------------------------- + +_CTDB_XSTATUS = ( + "|Node|IP|Disconnected|Unknown|Banned|Disabled|Unhealthy|Stopped" + "|Inactive|PartiallyOnline|ThisNode|\n" + "|0|10.0.0.11|0|0|0|0|0|0|0|0|Y|\n" + "|1|10.0.0.12|0|0|0|0|0|0|0|0|N|\n" + "|2|10.0.0.13|0|0|0|0|0|0|0|0|N|\n" +) + + +def test_ctdb_ok_node_count_all_healthy(): + assert ctdb_ok_node_count(_CTDB_XSTATUS) == 3 + + +def test_ctdb_ok_node_count_skips_flagged_nodes(): + text = _CTDB_XSTATUS.replace("|1|10.0.0.12|0|0|", "|1|10.0.0.12|1|0|") + assert ctdb_ok_node_count(text) == 2 + + +def test_ctdb_ok_node_count_empty_output(): + # Connection-refused output has no table rows: count is 0, not an error. + assert ctdb_ok_node_count("connect() failed, errno=111\n") == 0 + + +def test_ctdb_vip_pnn_finds_holder(): + text = "Public IPs on node 0\n10.0.0.201 1\n10.0.0.202 2\n10.0.0.203 0\n" + assert ctdb_vip_pnn(text, "10.0.0.202/24") == 2 + assert ctdb_vip_pnn(text, "10.0.0.203") == 0 + + +def test_ctdb_vip_pnn_missing_vip_is_minus_one(): + assert ctdb_vip_pnn("Public IPs on node 0\n", "10.0.0.201/24") == -1 + + +def test_smb_vip_addresses_from_cidr(): + assert smb_vip_addresses("10.0.0.0/24", 3) == [ + "10.0.0.200/24", + "10.0.0.201/24", + "10.0.0.202/24", + ] + + +def test_smb_vip_addresses_accepts_host_cidr(): + # Callers pass the network as reported with a host address in it. + assert smb_vip_addresses("10.0.0.7/24", 1) == ["10.0.0.200/24"] + + +def test_smb_vip_addresses_rejects_offsets_outside_network(): + try: + smb_vip_addresses("10.0.0.0/28", 1) + except ValueError: + pass + else: + raise AssertionError("expected ValueError for /28 with offset 200") + + +def test_smb_share_spec_yaml_provider_is_non_proxied(): + doc = smb_share_spec_yaml("dev", "share1", "smbfs", "s1") + assert "provider: samba-vfs/new" in doc + assert "cluster_id: dev" in doc + assert "subvolume: s1" in doc diff --git a/tests/robot/smb-tests/smb_tests.robot b/tests/robot/smb-tests/smb_tests.robot new file mode 100644 index 00000000..e5d1bdea --- /dev/null +++ b/tests/robot/smb-tests/smb_tests.robot @@ -0,0 +1,137 @@ +*** Settings *** +Documentation smb-tests +... Tests MicroCeph native SMB in a multi-node LXD cluster: creates an +... smb cluster through mgr/smb (ceph smb CLI, microceph orchestrator +... backend), verifies a share roundtrip via a CTDB public address, kills +... the VIP holder to exercise failover, rejoins it, then removes the +... cluster. +Resource ../resources/microceph_harness.resource +Suite Setup SMB Multinode Suite Setup +Suite Teardown Teardown MicroCeph Environment +Test Tags multi-node smb cephfs lxd slow integration + +*** Variables *** +${SMB_CLUSTER} dev +${SMB_SHARE} share1 +${SMB_VOLUME} smbfs +${SMB_SUBVOLUME} s1 +${SMB_USER} smbuser +${SMB_PASSWORD} s3cr3tpass + +*** Keywords *** +SMB Multinode Suite Setup + Provision Multinode VM microceph-smb-vm ${OUTER_VM_DISK} public + Bootstrap Head Node public + Join Worker Nodes To Cluster public + Add OSD To Node node-wrk0 + Add OSD To Node node-wrk1 + Add OSD To Node node-wrk2 + Wait For OSD Count Head 3 + # Confined smbd panics on setgroups until the smb-support snapd + # interface lands; the smb suite runs the snap in devmode. + Reinstall Snap Devmode On All Nodes + Wait For Cluster Health OK node-wrk0 + +Enable Microceph Orchestrator + [Documentation] Points mgr/smb at the microceph orchestrator backend. + Run In Head Node And Check microceph.ceph mgr module enable smb + Run In Head Node And Check microceph.ceph mgr module enable microceph + Run In Head Node And Check microceph.ceph orch set backend microceph + ${result}= Run In Head Node microceph.ceph orch status 60 + Should Contain ${result.stdout} Available: Yes + +Run In Head Node And Check + [Documentation] Runs a command on the head container and asserts rc 0. + [Arguments] ${cmd} ${timeout}=120 + ${result}= Run In Head Node ${cmd} ${timeout} + Should Be Equal As Integers ${result.rc} 0 msg=${cmd} failed: ${result.stderr} + +Provision SMB Backing Volume + [Documentation] Creates the CephFS volume and a world-writable subvolume + ... (share write permissions are the admin's task in Phase 1). + Run In Head Node And Check microceph.ceph fs volume create ${SMB_VOLUME} 180 + Run In Head Node And Check microceph.ceph fs subvolume create ${SMB_VOLUME} ${SMB_SUBVOLUME} --mode 0777 60 + +Provision SMB System Users + [Documentation] Creates the matching unix user on every node: the passdb + ... is CTDB-replicated but each smbd maps sessions via local NSS. + FOR ${container} IN node-wrk0 node-wrk1 node-wrk2 + Run In Container And Check ${container} id ${SMB_USER} >/dev/null 2>&1 || useradd -M -s /usr/sbin/nologin ${SMB_USER} 30 + END + +Create SMB Cluster Via Mgr + [Documentation] Creates the CTDB-clustered smb cluster through ceph smb, + ... with public addresses computed from the cluster network. + ${cidr}= Get Public Network Cidr + ${vips}= Smb Vip Addresses ${cidr} ${3} + Set Suite Variable ${SMB_VIPS} ${vips} + ${addr_flags}= Evaluate " ".join(f"--public-addrs={a}" for a in $vips) + Run In Head Node And Check + ... microceph.ceph smb cluster create ${SMB_CLUSTER} user --define-user-pass=${SMB_USER}%${SMB_PASSWORD} --placement=count:3 --clustering=always ${addr_flags} + ... 900 + +Create SMB Share Via Mgr + [Documentation] Applies the share declaratively: the imperative create + ... defaults to the proxied vfs provider, which microceph rejects. + ${yaml}= Smb Share Spec Yaml ${SMB_CLUSTER} ${SMB_SHARE} ${SMB_VOLUME} ${SMB_SUBVOLUME} + # No stdin plumbing through the nested lxc exec: ship the document base64ed. + ${b64}= Evaluate base64.b64encode($yaml.encode()).decode() modules=base64 + Run In Container And Check node-wrk0 printf '%s' ${b64} | base64 -d > /root/${SMB_SHARE}.yaml 30 + Run In Head Node And Check microceph.ceph smb apply -i /root/${SMB_SHARE}.yaml 900 + +SMB Roundtrip Via Address + [Documentation] put + get via smbclient from the outer VM and compares content. + [Arguments] ${address} + ${ip}= Evaluate $address.split("/")[0] + Run In VM And Check echo "smb roundtrip $(date -u)" > /tmp/smb-rt.txt 10 + Run In VM And Check smbclient //${ip}/${SMB_SHARE} -U ${SMB_USER}%${SMB_PASSWORD} -c "put /tmp/smb-rt.txt rt.txt" 120 + Run In VM And Check smbclient //${ip}/${SMB_SHARE} -U ${SMB_USER}%${SMB_PASSWORD} -c "get rt.txt /tmp/smb-rt-back.txt" 120 + Run In VM And Check diff /tmp/smb-rt.txt /tmp/smb-rt-back.txt 10 + Run In VM And Check rm -f /tmp/smb-rt.txt /tmp/smb-rt-back.txt 10 + +*** Test Cases *** +Test Enable Microceph Orchestrator Backend + [Documentation] Enables mgr/smb plus the microceph orchestrator module. + [Tags] smb multi-node + Enable Microceph Orchestrator + +Test Create SMB Cluster And Share + [Documentation] Provisions the backing volume and creates cluster+share via mgr. + [Tags] smb multi-node + Provision SMB Backing Volume + Provision SMB System Users + Create SMB Cluster Via Mgr + Create SMB Share Via Mgr + Wait For Ctdb Healthy node-wrk0 3 + +Test SMB Roundtrip Via VIP + [Documentation] Writes and reads back a file through the first CTDB VIP. + [Tags] smb multi-node + Run In VM And Check sudo apt-get install -y smbclient 300 + SMB Roundtrip Via Address ${SMB_VIPS}[0] + +Test SMB VIP Failover And Rejoin + [Documentation] Force-stops the node holding the first VIP, verifies the + ... share recovers on the same address, then rejoins the node. + [Tags] smb multi-node slow + ${ip_table}= Get Ctdb Vip Output node-wrk0 + ${pnn}= Ctdb Vip Pnn ${ip_table} ${SMB_VIPS}[0] + Should Be True ${pnn} >= 0 msg=VIP ${SMB_VIPS}[0] is not assigned + # CTDB pnn N is line N of the nodes file, which follows join order. + ${holder}= Set Variable node-wrk${pnn} + ${observer}= Set Variable IF "${holder}" == "node-wrk0" node-wrk1 node-wrk0 + Run In VM And Check lxc stop --force ${holder} 120 + Wait Until Keyword Succeeds 180s 10s SMB Roundtrip Via Address ${SMB_VIPS}[0] + Run In VM And Check lxc start ${holder} 120 + Wait For Ctdb Healthy ${observer} 3 attempts=45 + +Test Remove SMB Cluster + [Documentation] Removes share and cluster through mgr and verifies teardown. + [Tags] smb multi-node + Run In Head Node And Check microceph.ceph smb share rm ${SMB_CLUSTER} ${SMB_SHARE} 300 + Run In Head Node And Check microceph.ceph smb cluster rm ${SMB_CLUSTER} 900 + ${result}= Run In Head Node microceph.ceph smb show 60 + Should Not Contain ${result.stdout} ceph.smb.cluster + ${services}= Run In Container Unchecked node-wrk0 snap services microceph 30 + ${active}= Enabled Active Services ${services.stdout} + Should Not Contain ${active} microceph.ctdbd From 353a81118f9a792b6bd0b8b8785c8f0d63374a98 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 22:21:36 +0530 Subject: [PATCH 25/31] docs: add smb how-to How-to page for serving SMB shares: devmode requirement and the pending smb-support snapd interface, orchestrator backend enablement, subvolume and system-user preparation, CTDB-clustered cluster create with public addresses, declarative share creation (and why the imperative default provider is rejected), client connection, the reconnect-based failover semantics, and removal. Indexed under 'Consuming cluster storage'. docs: make html (--fail-on-warning) -> build succeeded Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- docs/snap/how-to/enable-smb.rst | 137 ++++++++++++++++++++++++++++++++ docs/snap/how-to/index.rst | 1 + 2 files changed, 138 insertions(+) create mode 100644 docs/snap/how-to/enable-smb.rst diff --git a/docs/snap/how-to/enable-smb.rst b/docs/snap/how-to/enable-smb.rst new file mode 100644 index 00000000..320bbda8 --- /dev/null +++ b/docs/snap/how-to/enable-smb.rst @@ -0,0 +1,137 @@ +.. _enable-smb: + +Serve SMB shares from MicroCeph +=============================== + +MicroCeph can serve CephFS subvolumes over SMB using Samba, clustered +with CTDB for high availability. The feature is driven entirely through +the upstream ``ceph smb`` manager module: MicroCeph acts as its +orchestrator backend and deploys ``smbd``/``ctdbd`` on the placed nodes. + +.. note:: + + SMB support currently requires the snap to be installed in devmode: + strictly confined ``smbd`` needs ``setgroups`` and the ``setuid``/ + ``setgid`` capabilities, which no existing snapd interface grants. A + dedicated ``smb-support`` interface is being proposed to snapd; until + it lands, install with ``--devmode``. + +Prerequisites +------------- + +- A bootstrapped MicroCeph cluster with OSDs and a CephFS filesystem. +- One unused IP address per placed node, in the nodes' subnet, to serve + as CTDB public addresses (VIPs). Clients connect to these. + +Enable the orchestrator backend +------------------------------- + +The ``smb`` manager module submits deployment specs to an orchestrator. +Point it at MicroCeph's: + +.. code-block:: none + + $ sudo microceph.ceph mgr module enable smb + $ sudo microceph.ceph mgr module enable microceph + $ sudo microceph.ceph orch set backend microceph + $ sudo microceph.ceph orch status + Backend: microceph + Available: Yes + +Prepare the share path and users +-------------------------------- + +Create a subvolume to back the share. Setting the mode at creation +avoids having to mount the filesystem just to fix permissions: + +.. code-block:: none + + $ sudo microceph.ceph fs subvolume create newfs s1 --mode 0777 + +SMB users authenticate against Samba's clustered password database, +which MicroCeph seeds automatically, but each one must map to a system +user present on every placed node: + +.. code-block:: none + + $ sudo useradd -M -s /usr/sbin/nologin smbuser + +Create the SMB cluster +---------------------- + +Create a CTDB-clustered SMB cluster with user authentication, placed on +three nodes, listing one public address per node: + +.. code-block:: none + + $ sudo microceph.ceph smb cluster create dev user \ + --define-user-pass=smbuser%s3cr3t \ + --placement=count:3 --clustering=always \ + --public-addrs=10.0.0.200/24 \ + --public-addrs=10.0.0.201/24 \ + --public-addrs=10.0.0.202/24 + +Create the share +---------------- + +Shares are declared against the cluster. Use the declarative interface +with the ``samba-vfs/new`` provider; the imperative +``ceph smb share create`` defaults to a proxied provider that MicroCeph +does not deploy: + +.. code-block:: none + + $ cat share1.yaml + resource_type: ceph.smb.share + cluster_id: dev + share_id: share1 + cephfs: + volume: newfs + subvolume: s1 + path: / + provider: samba-vfs/new + + $ sudo microceph.ceph smb apply -i share1.yaml + +Inspect the deployment: + +.. code-block:: none + + $ sudo microceph.ceph smb show + $ sudo microceph.ceph orch ls + NAME PORTS RUNNING PLACEMENT + smb.dev 3/3 count:3 + +Connect from a client +--------------------- + +Any SMB client can connect through a public address: + +.. code-block:: none + + $ smbclient //10.0.0.200/share1 -U smbuser%s3cr3t + smb: \> put file.txt + +Failover semantics +------------------ + +When a node fails, CTDB moves its public addresses to a surviving node. +This is reconnect-based failover, not transparent state migration: open +sessions against a failed node drop, and clients re-establish them +against the same address once it is re-hosted (typically well under two +minutes with default timers). Applications should treat an SMB session +drop as retryable. + +Remove the cluster +------------------ + +Removal is also driven through the manager module: + +.. code-block:: none + + $ sudo microceph.ceph smb share rm dev share1 + $ sudo microceph.ceph smb cluster rm dev + +This stops and removes ``smbd``/``ctdbd`` from all placed nodes and +deletes the per-cluster service state. The CephFS data backing the +share is left untouched. diff --git a/docs/snap/how-to/index.rst b/docs/snap/how-to/index.rst index 908b2ab2..b8e9bdf5 100644 --- a/docs/snap/how-to/index.rst +++ b/docs/snap/how-to/index.rst @@ -87,6 +87,7 @@ Follow these guides to learn how to make use of the storage provided by your clu mount-block-device mount-cephfs-share + Serve SMB shares Contact us From c691bfc0fc854501ffce478742e4739e0c762e37 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 23:21:34 +0530 Subject: [PATCH 26/31] snap,daemon: default the mgr/smb share provider to direct vfs Upstream expands the abbreviated 'samba-vfs' provider to the proxied variant, so the plain 'ceph smb share create' always produced a spec with the cephfs-proxy feature and failed microcephd validation, breaking the native CLI experience the mgr/smb integration exists for. Patch the staged smb module so the default expands to samba-vfs/new: microceph serves shares via direct libcephfs and deploys no proxy daemon. Stored resources keep the abbreviated provider, so the same document still expands to proxied on cephadm. An explicit samba-vfs/proxied request keeps its upstream meaning and is still rejected; the rejection message no longer blames the default. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/service_placement_smb.go | 6 ++-- ...002-mgr-smb-default-provider-vfs-new.patch | 35 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 patches/0002-mgr-smb-default-provider-vfs-new.patch diff --git a/microceph/ceph/service_placement_smb.go b/microceph/ceph/service_placement_smb.go index 43df38cc..2ced6722 100644 --- a/microceph/ceph/service_placement_smb.go +++ b/microceph/ceph/service_placement_smb.go @@ -50,9 +50,9 @@ func (smb *SMBServicePlacement) PopulateParams(s interfaces.StateInterface, payl case "domain": return fmt.Errorf("features: 'domain' (AD membership) is not supported in Phase 1") case "cephfs-proxy": - return fmt.Errorf("features: 'cephfs-proxy' is not supported; create shares with " + - "--provider=samba-vfs/new (the default 'samba-vfs' provider expands to the " + - "proxied variant, which microceph does not deploy)") + return fmt.Errorf("features: 'cephfs-proxy' is not supported; microceph does not " + + "deploy the cephfs proxy daemon (use the default samba-vfs or samba-vfs/new " + + "share provider instead of samba-vfs/proxied)") default: return fmt.Errorf("features: '%s' is not supported", feature) } diff --git a/patches/0002-mgr-smb-default-provider-vfs-new.patch b/patches/0002-mgr-smb-default-provider-vfs-new.patch new file mode 100644 index 00000000..19641b91 --- /dev/null +++ b/patches/0002-mgr-smb-default-provider-vfs-new.patch @@ -0,0 +1,35 @@ +From: MicroCeph maintainers +Subject: [PATCH] mgr/smb: expand the default provider to the non-proxied VFS + +Upstream expands the abbreviated share provider 'samba-vfs' to +'samba-vfs/proxied', which assumes the orchestrator co-deploys the +cephfs-proxy daemon (libcephfsd). MicroCeph serves CephFS from smbd +via direct libcephfs and deploys no proxy, so specs carrying the +cephfs-proxy feature are rejected by microcephd and the plain +`ceph smb share create` CLI would always fail. + +Expand the default to 'samba-vfs/new' instead. The stored resource +keeps the abbreviated provider, so the same document remains portable +to cephadm (where it still expands to the proxied variant). Requesting +'samba-vfs/proxied' explicitly keeps its upstream meaning and is still +rejected by microcephd. + +Drop this patch if mgr/smb ever makes the default expansion +configurable. +--- + +--- a/share/ceph/mgr/smb/enums.py ++++ b/share/ceph/mgr/smb/enums.py +@@ -23,8 +23,11 @@ + def expand(self) -> 'CephFSStorageProvider': + """Expand abbreviated/default values into the full/expanded form.""" + if self is self.SAMBA_VFS: ++ # MicroCeph serves CephFS from smbd via direct libcephfs and ++ # deploys no cephfs-proxy daemon, so the abbreviated provider ++ # expands to the non-proxied VFS here. + # mypy gets confused by enums +- return self.__class__(self.SAMBA_VFS_PROXIED) ++ return self.__class__(self.SAMBA_VFS_NEW) + return self + + def is_vfs(self) -> bool: From bb1d31c3235721a5232335b03c5ee3cd93ec2412 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 23:21:42 +0530 Subject: [PATCH 27/31] tests: create the smb share imperatively in smb-tests With the default provider patched to the non-proxied vfs, the declarative-apply workaround (yaml document shipped base64 into the container) collapses to one 'ceph smb share create' line; drop the now-unused smb_share_spec_yaml helper and its pytest case. tox -e robot -- --dryrun --test-suite smb-tests -> 5 tests resolve pytest test_harness_helpers.py -> 115 passed Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- tests/robot/resources/smb_ops.py | 19 ------------------- tests/robot/resources/test_harness_helpers.py | 7 ------- tests/robot/smb-tests/smb_tests.robot | 11 ++++------- 3 files changed, 4 insertions(+), 33 deletions(-) diff --git a/tests/robot/resources/smb_ops.py b/tests/robot/resources/smb_ops.py index 0336c5a4..768a66ab 100644 --- a/tests/robot/resources/smb_ops.py +++ b/tests/robot/resources/smb_ops.py @@ -59,22 +59,3 @@ def ctdb_vip_pnn(ip_output, vip): except ValueError: return -1 return -1 - - -def smb_share_spec_yaml(cluster_id, share_id, volume, subvolume): - """Return a declarative ceph.smb.share document for ``ceph smb apply``. - - The provider must be spelled samba-vfs/new: the imperative - ``ceph smb share create`` defaults to plain samba-vfs, which mgr/smb - expands to the proxied variant that microceph rejects. - """ - return ( - "resource_type: ceph.smb.share\n" - f"cluster_id: {cluster_id}\n" - f"share_id: {share_id}\n" - "cephfs:\n" - f" volume: {volume}\n" - f" subvolume: {subvolume}\n" - " path: /\n" - " provider: samba-vfs/new\n" - ) diff --git a/tests/robot/resources/test_harness_helpers.py b/tests/robot/resources/test_harness_helpers.py index 6ed30b17..37b3073b 100644 --- a/tests/robot/resources/test_harness_helpers.py +++ b/tests/robot/resources/test_harness_helpers.py @@ -18,7 +18,6 @@ from smb_ops import ( ctdb_ok_node_count, ctdb_vip_pnn, - smb_share_spec_yaml, smb_vip_addresses, ) from snap_services import enabled_active_services @@ -1024,9 +1023,3 @@ def test_smb_vip_addresses_rejects_offsets_outside_network(): else: raise AssertionError("expected ValueError for /28 with offset 200") - -def test_smb_share_spec_yaml_provider_is_non_proxied(): - doc = smb_share_spec_yaml("dev", "share1", "smbfs", "s1") - assert "provider: samba-vfs/new" in doc - assert "cluster_id: dev" in doc - assert "subvolume: s1" in doc diff --git a/tests/robot/smb-tests/smb_tests.robot b/tests/robot/smb-tests/smb_tests.robot index e5d1bdea..8ea57df6 100644 --- a/tests/robot/smb-tests/smb_tests.robot +++ b/tests/robot/smb-tests/smb_tests.robot @@ -71,13 +71,10 @@ Create SMB Cluster Via Mgr ... 900 Create SMB Share Via Mgr - [Documentation] Applies the share declaratively: the imperative create - ... defaults to the proxied vfs provider, which microceph rejects. - ${yaml}= Smb Share Spec Yaml ${SMB_CLUSTER} ${SMB_SHARE} ${SMB_VOLUME} ${SMB_SUBVOLUME} - # No stdin plumbing through the nested lxc exec: ship the document base64ed. - ${b64}= Evaluate base64.b64encode($yaml.encode()).decode() modules=base64 - Run In Container And Check node-wrk0 printf '%s' ${b64} | base64 -d > /root/${SMB_SHARE}.yaml 30 - Run In Head Node And Check microceph.ceph smb apply -i /root/${SMB_SHARE}.yaml 900 + [Documentation] Creates the share with the plain imperative CLI: the + ... snap patches mgr/smb's default provider to the non-proxied VFS, + ... so no provider flag or declarative workaround is needed. + Run In Head Node And Check microceph.ceph smb share create ${SMB_CLUSTER} ${SMB_SHARE} ${SMB_VOLUME} / --subvolume=${SMB_SUBVOLUME} 900 SMB Roundtrip Via Address [Documentation] put + get via smbclient from the outer VM and compares content. From 33ebd790de06498f23aa5229dced001bec9415fc Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 23:21:43 +0530 Subject: [PATCH 28/31] docs: use the imperative smb share create The provider workaround is gone; document the plain CLI and note the patched default expansion plus the proxied-provider rejection. docs: make html (--fail-on-warning) -> build succeeded Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- docs/snap/how-to/enable-smb.rst | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/docs/snap/how-to/enable-smb.rst b/docs/snap/how-to/enable-smb.rst index 320bbda8..b3b5e98d 100644 --- a/docs/snap/how-to/enable-smb.rst +++ b/docs/snap/how-to/enable-smb.rst @@ -74,24 +74,17 @@ three nodes, listing one public address per node: Create the share ---------------- -Shares are declared against the cluster. Use the declarative interface -with the ``samba-vfs/new`` provider; the imperative -``ceph smb share create`` defaults to a proxied provider that MicroCeph -does not deploy: - .. code-block:: none - $ cat share1.yaml - resource_type: ceph.smb.share - cluster_id: dev - share_id: share1 - cephfs: - volume: newfs - subvolume: s1 - path: / - provider: samba-vfs/new - - $ sudo microceph.ceph smb apply -i share1.yaml + $ sudo microceph.ceph smb share create dev share1 newfs / --subvolume=s1 + +.. note:: + + MicroCeph serves shares from ``smbd`` via direct libcephfs, so its + build of the ``smb`` module expands the default share provider to + the non-proxied ``samba-vfs/new`` variant. Shares explicitly + requesting ``samba-vfs/proxied`` are rejected: MicroCeph does not + deploy the cephfs-proxy daemon. Inspect the deployment: From 3f1ccf4715bb226dbc012047744330d3288bf4b2 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sat, 11 Jul 2026 23:35:45 +0530 Subject: [PATCH 29/31] snap: apply the mgr-smb provider patch in its own part The ceph part's patch loop runs against CRAFT_STAGE, where another part's files are only present if that part happened to stage first; patching mgr/smb from there failed on a clean rebuild (and the loop also tripped over the retired stub patch still cached in the part source). Apply the provider-default patch from the mgr-smb part's own override-build against its CRAFT_PART_INSTALL, and move it to patches/mgr-smb/ so the ceph loop's glob no longer picks it up. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- .../0001-default-provider-vfs-new.patch} | 0 snap/snapcraft.yaml | 4 ++++ 2 files changed, 4 insertions(+) rename patches/{0002-mgr-smb-default-provider-vfs-new.patch => mgr-smb/0001-default-provider-vfs-new.patch} (100%) diff --git a/patches/0002-mgr-smb-default-provider-vfs-new.patch b/patches/mgr-smb/0001-default-provider-vfs-new.patch similarity index 100% rename from patches/0002-mgr-smb-default-provider-vfs-new.patch rename to patches/mgr-smb/0001-default-provider-vfs-new.patch diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index d5176249..a5d119d8 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -564,6 +564,10 @@ parts: mkdir -p "${CRAFT_PART_INSTALL}/share/ceph/mgr" cp -r ceph-src/src/pybind/mgr/smb "${CRAFT_PART_INSTALL}/share/ceph/mgr/" rm -rf "${CRAFT_PART_INSTALL}/share/ceph/mgr/smb/tests" + # Patched here, not in the ceph part's patch loop: that loop runs + # against $CRAFT_STAGE and cannot see this part's files reliably + # (stage order); a part patches only what it installs itself. + patch -p1 -d "${CRAFT_PART_INSTALL}" < "${CRAFT_PROJECT_DIR}/patches/mgr-smb/0001-default-provider-vfs-new.patch" sambacc: plugin: nil From 95efb05384baea81fc0dc0607d37db5af8aa413b Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sun, 12 Jul 2026 14:48:41 +0530 Subject: [PATCH 30/31] tests: stub updateConfigFunc in the smb placement suite EnableService now renders ceph.conf via updateConfigFunc before enabling any service (main's deferred bootstrap change), so the SMB placement tests must bypass the database-dependent rendering the same way the NFS and generic placement suites do. Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- microceph/ceph/service_placement_smb_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/microceph/ceph/service_placement_smb_test.go b/microceph/ceph/service_placement_smb_test.go index f8ae74ba..9664a739 100644 --- a/microceph/ceph/service_placement_smb_test.go +++ b/microceph/ceph/service_placement_smb_test.go @@ -9,6 +9,7 @@ import ( "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/interfaces" "github.com/canonical/microceph/microceph/mocks" "github.com/canonical/microceph/microceph/tests" @@ -45,6 +46,13 @@ func TestServicesPlacementSMB(t *testing.T) { func (s *servicePlacementSMBSuite) SetupTest() { s.BaseSuite.SetupTest() s.TestStateInterface = mocks.NewStateInterface(s.T()) + // Bypass database-dependent ceph.conf rendering: these tests exercise the + // SMB placement pipeline with a mock state, not a real cluster database. + updateConfigFunc = func(_ context.Context, _ interfaces.StateInterface) error { return nil } +} + +func (s *servicePlacementSMBSuite) TearDownTest() { + updateConfigFunc = UpdateConfig } // populated returns an SMBServicePlacement loaded from a payload that must From fce3393e932505af80cc199bd08a60306b8e4cc0 Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Sun, 12 Jul 2026 17:15:50 +0530 Subject: [PATCH 31/31] docs: reword 'retryable' flagged by the docs spell check Assisted-by: claude-code:claude-fable-5 Signed-off-by: Utkarsh Bhatt --- docs/snap/how-to/enable-smb.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/snap/how-to/enable-smb.rst b/docs/snap/how-to/enable-smb.rst index b3b5e98d..141b16f9 100644 --- a/docs/snap/how-to/enable-smb.rst +++ b/docs/snap/how-to/enable-smb.rst @@ -113,7 +113,7 @@ This is reconnect-based failover, not transparent state migration: open sessions against a failed node drop, and clients re-establish them against the same address once it is re-hosted (typically well under two minutes with default timers). Applications should treat an SMB session -drop as retryable. +drop as transient and retry the connection. Remove the cluster ------------------