diff --git a/core/lifecycle.go b/core/lifecycle.go index e0fece77f7..65102d5d84 100644 --- a/core/lifecycle.go +++ b/core/lifecycle.go @@ -3,6 +3,7 @@ package core import ( "context" "fmt" + "slices" "strings" "time" @@ -10,6 +11,7 @@ import ( claberrors "github.com/srl-labs/containerlab/errors" clabnodes "github.com/srl-labs/containerlab/nodes" clabruntime "github.com/srl-labs/containerlab/runtime" + clabtypes "github.com/srl-labs/containerlab/types" ) // lifecycleNodes resolves the requested node names without applying lifecycle policy. @@ -39,6 +41,176 @@ func (c *CLab) lifecycleNodes(nodeNames []string) ([]clabnodes.Node, error) { return nodes, nil } +func (c *CLab) lifecycleStartNodes( + nodeNames []string, +) ([]clabnodes.Node, map[string]clabtypes.WaitForList, error) { + selected := map[string]struct{}{} + deps := map[string]clabtypes.WaitForList{} + + var addWithDependencies func(string) error + addWithDependencies = func(name string) error { + n, ok := c.Nodes[name] + if !ok { + return fmt.Errorf( + "%w: node %q is not present in the topology", + claberrors.ErrIncorrectInput, + name, + ) + } + + if _, exists := selected[name]; exists { + return nil + } + selected[name] = struct{}{} + + for _, dep := range c.lifecycleDependencies(n) { + if _, ok := c.Nodes[dep.Node]; !ok { + return fmt.Errorf("dependee node %s not found", dep.Node) + } + deps[name] = append(deps[name], dep) + if err := addWithDependencies(dep.Node); err != nil { + return err + } + } + + return nil + } + + if len(nodeNames) > 0 { + for _, name := range nodeNames { + if err := addWithDependencies(name); err != nil { + return nil, nil, err + } + } + } else { + names := make([]string, 0, len(c.Nodes)) + for name := range c.Nodes { + names = append(names, name) + } + slices.Sort(names) + + for _, name := range names { + if err := addWithDependencies(name); err != nil { + return nil, nil, err + } + } + } + + orderedNames, err := lifecycleToposort(selected, deps) + if err != nil { + return nil, nil, err + } + + nodes := make([]clabnodes.Node, 0, len(orderedNames)) + for _, name := range orderedNames { + nodes = append(nodes, c.Nodes[name]) + } + + return nodes, deps, nil +} + +func (c *CLab) lifecycleDependencies(n clabnodes.Node) clabtypes.WaitForList { + var deps clabtypes.WaitForList + + if stages := n.Config().Stages; stages != nil { + for _, waitForNodes := range stages.GetWaitFor() { + deps = append(deps, waitForNodes...) + } + } + + netModeArr := strings.SplitN(n.Config().NetworkMode, ":", 2) //nolint: mnd + if netModeArr[0] == "container" && len(netModeArr) == 2 { + if _, exists := c.Nodes[netModeArr[1]]; exists { + deps = append(deps, &clabtypes.WaitFor{ + Node: netModeArr[1], + Stage: clabtypes.WaitForCreate, + }) + } + } + + return deps +} + +func lifecycleToposort( + selected map[string]struct{}, + deps map[string]clabtypes.WaitForList, +) ([]string, error) { + var ordered []string + visiting := map[string]bool{} + visited := map[string]bool{} + + var visit func(string) error + visit = func(name string) error { + if visited[name] { + return nil + } + if visiting[name] { + return fmt.Errorf("cyclic lifecycle dependencies found at node %q", name) + } + + visiting[name] = true + for _, dep := range deps[name] { + if _, ok := selected[dep.Node]; !ok { + continue + } + if err := visit(dep.Node); err != nil { + return err + } + } + visiting[name] = false + visited[name] = true + ordered = append(ordered, name) + + return nil + } + + names := make([]string, 0, len(selected)) + for name := range selected { + names = append(names, name) + } + slices.Sort(names) + + for _, name := range names { + if err := visit(name); err != nil { + return nil, err + } + } + + return ordered, nil +} + +func (c *CLab) waitForLifecycleNodeHealthy( + ctx context.Context, + nodeName string, + depName string, +) error { + dep, ok := c.Nodes[depName] + if !ok { + return fmt.Errorf("dependee node %s not found", depName) + } + + for { + healthy, err := dep.IsHealthy(ctx) + if err != nil { + return fmt.Errorf( + "node %q waiting for node %q healthy stage: %w", + nodeName, + depName, + err, + ) + } + if healthy { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } +} + func (c *CLab) parkRecreatedNodes(ctx context.Context, plan *applyPlan) error { for _, nodeName := range sortedStringSet(plan.parkedNodeSet) { node, exists := c.Nodes[nodeName] diff --git a/core/lifecycle_test.go b/core/lifecycle_test.go new file mode 100644 index 0000000000..e4356c2669 --- /dev/null +++ b/core/lifecycle_test.go @@ -0,0 +1,84 @@ +package core + +import ( + "slices" + "testing" + + clabtypes "github.com/srl-labs/containerlab/types" + "go.uber.org/mock/gomock" +) + +func TestLifecycleStartNodesIncludesDependenciesInOrder(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + c := CLab{ + Nodes: getNodeMap(mockCtrl), + } + + nodes, deps, err := c.lifecycleStartNodes([]string{"node5"}) + if err != nil { + t.Fatalf("lifecycleStartNodes() error = %v", err) + } + + got := make([]string, 0, len(nodes)) + for _, n := range nodes { + got = append(got, n.Config().ShortName) + } + + want := []string{"node1", "node2", "node3", "node4", "node5"} + if !slices.Equal(got, want) { + t.Fatalf("lifecycleStartNodes() order = %v, want %v", got, want) + } + + if len(deps["node5"]) != 2 { + t.Fatalf("node5 dependencies = %d, want 2", len(deps["node5"])) + } +} + +func TestLifecycleStartNodesIncludesContainerNetworkModeDependency(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + c := CLab{ + Nodes: getNodeMap(mockCtrl), + } + + _, deps, err := c.lifecycleStartNodes([]string{"node3"}) + if err != nil { + t.Fatalf("lifecycleStartNodes() error = %v", err) + } + + var found bool + for _, dep := range deps["node3"] { + if dep.Node == "node2" && dep.Stage == clabtypes.WaitForCreate { + found = true + } + } + + if !found { + t.Fatalf("node3 dependencies = %#v, want network-mode dependency on node2 create", deps["node3"]) + } +} + +func TestLifecycleStartNodesDetectsCycles(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + nodes := getNodeMap(mockCtrl) + nodes["node1"].Config().Stages.Create.WaitFor = clabtypes.WaitForList{ + &clabtypes.WaitFor{ + Node: "node2", + Stage: clabtypes.WaitForCreate, + }, + } + + c := CLab{ + Nodes: nodes, + } + + _, _, err := c.lifecycleStartNodes([]string{"node2"}) + if err == nil { + t.Fatal("lifecycleStartNodes() error = nil, want cycle error") + } +} diff --git a/core/start.go b/core/start.go index ff16ec21ed..46eb7b0653 100644 --- a/core/start.go +++ b/core/start.go @@ -2,6 +2,8 @@ package core import ( "context" + + clabtypes "github.com/srl-labs/containerlab/types" ) // StartNodes starts one or more stopped nodes and restores their parked interfaces back into the @@ -11,12 +13,22 @@ func (c *CLab) StartNodes(ctx context.Context, nodeNames []string) error { return err } - nodes, err := c.lifecycleNodes(nodeNames) + nodes, deps, err := c.lifecycleStartNodes(nodeNames) if err != nil { return err } for _, n := range nodes { + for _, dep := range deps[n.GetShortName()] { + if dep.Stage != clabtypes.WaitForHealthy { + continue + } + + if err := c.waitForLifecycleNodeHealthy(ctx, n.GetShortName(), dep.Node); err != nil { + return err + } + } + if err := n.Start(ctx); err != nil { return err } diff --git a/docs/manual/nodes.md b/docs/manual/nodes.md index 57e8d8296f..da2425cfb6 100644 --- a/docs/manual/nodes.md +++ b/docs/manual/nodes.md @@ -1007,6 +1007,8 @@ In the example below node four nodes are defined with different stages and `wait Containerlab's built-in Dependency Manger takes care of all the dependencies, both explicitly-defined and implicit ones. It will inspect the dependency graph and make sure it is acyclic. The output of the Dependency Manager graph is visible in the debug mode. +The same dependencies are honored by `containerlab start`. When a stopped lab is started with dependency-aware stages, containerlab starts prerequisites first. A dependency on the `healthy` stage waits until the prerequisite node reports healthy before starting the dependent node. + Note, that `wait-for` is a list, a node's stage may depend on several other nodes' stages. /// admonition | Usage scenarios diff --git a/nodes/ext_container/ext_container.go b/nodes/ext_container/ext_container.go index 1dcba87f45..b42a7603d8 100644 --- a/nodes/ext_container/ext_container.go +++ b/nodes/ext_container/ext_container.go @@ -55,8 +55,10 @@ func (e *extcont) Deploy(ctx context.Context, _ *clabnodes.DeployParams) error { // Delete we will not mess with external containers on delete. func (*extcont) Delete(_ context.Context) error { return nil } -func (*extcont) Stop(context.Context) error { return nil } +// Stop we will not mess with external containers on stop. +func (*extcont) Stop(_ context.Context) error { return nil } +// Start waits for external containers to be running without attempting to start them. func (e *extcont) Start(ctx context.Context) error { return clabruntime.WaitForContainerRunning(ctx, e.Runtime, e.Cfg.ShortName, e.Cfg.ShortName) }