Honor lifecycle dependencies when starting nodes

core/lifecycle.go

@@ -3,13 +3,15 @@ package core
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
 	"time"
 
 	"github.com/charmbracelet/log"
 	claberrors "github.com/srl-labs/containerlab/errors"
 	clabnodes "github.com/srl-labs/containerlab/nodes"
 	clabruntime "github.com/srl-labs/containerlab/runtime"
+	clabtypes "github.com/srl-labs/containerlab/types"
 )
 
 // lifecycleNodes resolves the requested node names without applying lifecycle policy.
@@ -39,6 +41,176 @@ func (c *CLab) lifecycleNodes(nodeNames []string) ([]clabnodes.Node, error) {
 	return nodes, nil
 }
 
+func (c *CLab) lifecycleStartNodes(
+	nodeNames []string,
+) ([]clabnodes.Node, map[string]clabtypes.WaitForList, error) {
+	selected := map[string]struct{}{}
+	deps := map[string]clabtypes.WaitForList{}
+
+	var addWithDependencies func(string) error
+	addWithDependencies = func(name string) error {
+		n, ok := c.Nodes[name]
+		if !ok {
+			return fmt.Errorf(
+				"%w: node %q is not present in the topology",
+				claberrors.ErrIncorrectInput,
+				name,
+			)
+		}
+
+		if _, exists := selected[name]; exists {
+			return nil
+		}
+		selected[name] = struct{}{}
+
+		for _, dep := range c.lifecycleDependencies(n) {
+			if _, ok := c.Nodes[dep.Node]; !ok {
+				return fmt.Errorf("dependee node %s not found", dep.Node)
+			}
+			deps[name] = append(deps[name], dep)
+			if err := addWithDependencies(dep.Node); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	if len(nodeNames) > 0 {
+		for _, name := range nodeNames {
+			if err := addWithDependencies(name); err != nil {
+				return nil, nil, err
+			}
+		}
+	} else {
+		names := make([]string, 0, len(c.Nodes))
+		for name := range c.Nodes {
+			names = append(names, name)
+		}
+		slices.Sort(names)
+
+		for _, name := range names {
+			if err := addWithDependencies(name); err != nil {
+				return nil, nil, err
+			}
+		}
+	}
+
+	orderedNames, err := lifecycleToposort(selected, deps)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	nodes := make([]clabnodes.Node, 0, len(orderedNames))
+	for _, name := range orderedNames {
+		nodes = append(nodes, c.Nodes[name])
+	}
+
+	return nodes, deps, nil
+}
+
+func (c *CLab) lifecycleDependencies(n clabnodes.Node) clabtypes.WaitForList {
+	var deps clabtypes.WaitForList
+
+	if stages := n.Config().Stages; stages != nil {
+		for _, waitForNodes := range stages.GetWaitFor() {
+			deps = append(deps, waitForNodes...)
+		}
+	}
+
+	netModeArr := strings.SplitN(n.Config().NetworkMode, ":", 2) //nolint: mnd
+	if netModeArr[0] == "container" && len(netModeArr) == 2 {
+		if _, exists := c.Nodes[netModeArr[1]]; exists {
+			deps = append(deps, &clabtypes.WaitFor{
+				Node:  netModeArr[1],
+				Stage: clabtypes.WaitForCreate,
+			})
+		}
+	}
+
+	return deps
+}
+
+func lifecycleToposort(
+	selected map[string]struct{},
+	deps map[string]clabtypes.WaitForList,
+) ([]string, error) {
+	var ordered []string
+	visiting := map[string]bool{}
+	visited := map[string]bool{}
+
+	var visit func(string) error
+	visit = func(name string) error {
+		if visited[name] {
+			return nil
+		}
+		if visiting[name] {
+			return fmt.Errorf("cyclic lifecycle dependencies found at node %q", name)
+		}
+
+		visiting[name] = true
+		for _, dep := range deps[name] {
+			if _, ok := selected[dep.Node]; !ok {
+				continue
+			}
+			if err := visit(dep.Node); err != nil {
+				return err
+			}
+		}
+		visiting[name] = false
+		visited[name] = true
+		ordered = append(ordered, name)
+
+		return nil
+	}
+
+	names := make([]string, 0, len(selected))
+	for name := range selected {
+		names = append(names, name)
+	}
+	slices.Sort(names)
+
+	for _, name := range names {
+		if err := visit(name); err != nil {
+			return nil, err
+		}
+	}
+
+	return ordered, nil
+}
+
+func (c *CLab) waitForLifecycleNodeHealthy(
+	ctx context.Context,
+	nodeName string,
+	depName string,
+) error {
+	dep, ok := c.Nodes[depName]
+	if !ok {
+		return fmt.Errorf("dependee node %s not found", depName)
+	}
+
+	for {
+		healthy, err := dep.IsHealthy(ctx)
+		if err != nil {
+			return fmt.Errorf(
+				"node %q waiting for node %q healthy stage: %w",
+				nodeName,
+				depName,
+				err,
+			)
+		}
+		if healthy {
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(time.Second):
+		}
+	}
+}
+
 func (c *CLab) parkRecreatedNodes(ctx context.Context, plan *applyPlan) error {
 	for _, nodeName := range sortedStringSet(plan.parkedNodeSet) {
 		node, exists := c.Nodes[nodeName]

core/lifecycle_test.go

@@ -0,0 +1,84 @@
+package core
+
+import (
+	"slices"
+	"testing"
+
+	clabtypes "github.com/srl-labs/containerlab/types"
+	"go.uber.org/mock/gomock"
+)
+
+func TestLifecycleStartNodesIncludesDependenciesInOrder(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	c := CLab{
+		Nodes: getNodeMap(mockCtrl),
+	}
+
+	nodes, deps, err := c.lifecycleStartNodes([]string{"node5"})
+	if err != nil {
+		t.Fatalf("lifecycleStartNodes() error = %v", err)
+	}
+
+	got := make([]string, 0, len(nodes))
+	for _, n := range nodes {
+		got = append(got, n.Config().ShortName)
+	}
+
+	want := []string{"node1", "node2", "node3", "node4", "node5"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("lifecycleStartNodes() order = %v, want %v", got, want)
+	}
+
+	if len(deps["node5"]) != 2 {
+		t.Fatalf("node5 dependencies = %d, want 2", len(deps["node5"]))
+	}
+}
+
+func TestLifecycleStartNodesIncludesContainerNetworkModeDependency(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	c := CLab{
+		Nodes: getNodeMap(mockCtrl),
+	}
+
+	_, deps, err := c.lifecycleStartNodes([]string{"node3"})
+	if err != nil {
+		t.Fatalf("lifecycleStartNodes() error = %v", err)
+	}
+
+	var found bool
+	for _, dep := range deps["node3"] {
+		if dep.Node == "node2" && dep.Stage == clabtypes.WaitForCreate {
+			found = true
+		}
+	}
+
+	if !found {
+		t.Fatalf("node3 dependencies = %#v, want network-mode dependency on node2 create", deps["node3"])
+	}
+}
+
+func TestLifecycleStartNodesDetectsCycles(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	nodes := getNodeMap(mockCtrl)
+	nodes["node1"].Config().Stages.Create.WaitFor = clabtypes.WaitForList{
+		&clabtypes.WaitFor{
+			Node:  "node2",
+			Stage: clabtypes.WaitForCreate,
+		},
+	}
+
+	c := CLab{
+		Nodes: nodes,
+	}
+
+	_, _, err := c.lifecycleStartNodes([]string{"node2"})
+	if err == nil {
+		t.Fatal("lifecycleStartNodes() error = nil, want cycle error")
+	}
+}

core/start.go

@@ -2,6 +2,8 @@ package core
 
 import (
 	"context"
+
+	clabtypes "github.com/srl-labs/containerlab/types"
 )
 
 // StartNodes starts one or more stopped nodes and restores their parked interfaces back into the
@@ -11,12 +13,22 @@ func (c *CLab) StartNodes(ctx context.Context, nodeNames []string) error {
 		return err
 	}
 
-	nodes, err := c.lifecycleNodes(nodeNames)
+	nodes, deps, err := c.lifecycleStartNodes(nodeNames)
 	if err != nil {
 		return err
 	}
 
 	for _, n := range nodes {
+		for _, dep := range deps[n.GetShortName()] {
+			if dep.Stage != clabtypes.WaitForHealthy {
+				continue
+			}
+
+			if err := c.waitForLifecycleNodeHealthy(ctx, n.GetShortName(), dep.Node); err != nil {
+				return err
+			}
+		}
+
 		if err := n.Start(ctx); err != nil {
 			return err
 		}

docs/manual/nodes.md

@@ -1007,6 +1007,8 @@ In the example below node four nodes are defined with different stages and `wait
 
 Containerlab's built-in Dependency Manger takes care of all the dependencies, both explicitly-defined and implicit ones. It will inspect the dependency graph and make sure it is acyclic. The output of the Dependency Manager graph is visible in the debug mode.
 
+The same dependencies are honored by `containerlab start`. When a stopped lab is started with dependency-aware stages, containerlab starts prerequisites first. A dependency on the `healthy` stage waits until the prerequisite node reports healthy before starting the dependent node.
+
 Note, that `wait-for` is a list, a node's stage may depend on several other nodes' stages.
 
 /// admonition | Usage scenarios

nodes/ext_container/ext_container.go

@@ -55,8 +55,10 @@ func (e *extcont) Deploy(ctx context.Context, _ *clabnodes.DeployParams) error {
 // Delete we will not mess with external containers on delete.
 func (*extcont) Delete(_ context.Context) error { return nil }
 
-func (*extcont) Stop(context.Context) error { return nil }
+// Stop we will not mess with external containers on stop.
+func (*extcont) Stop(_ context.Context) error { return nil }
 
+// Start waits for external containers to be running without attempting to start them.
 func (e *extcont) Start(ctx context.Context) error {
 	return clabruntime.WaitForContainerRunning(ctx, e.Runtime, e.Cfg.ShortName, e.Cfg.ShortName)
 }