Project and AZ handling optimizations, Network preparations for OS and CBT error handling via full-copy on cutover

internal/nbdkit/builder.go

@@ -79,7 +79,6 @@ func (b *NbdkitBuilder) Build() (*NbdkitServer, error) {
 	socket := fmt.Sprintf("%s/nbdkit.sock", tmp)
 	pidFile := fmt.Sprintf("%s/nbdkit.pid", tmp)
 
-	os.Setenv("LD_LIBRARY_PATH", "/usr/lib64/vmware-vix-disklib/lib64")
 	cmd := exec.Command(
 		"nbdkit",
 		"--exit-with-parent",
@@ -98,6 +97,7 @@ func (b *NbdkitBuilder) Build() (*NbdkitServer, error) {
 		"transports=file:nbdssl:nbd",
 		b.filename,
 	)
+	cmd.Env = append(os.Environ(), "LD_LIBRARY_PATH=/usr/lib64/vmware-vix-disklib/lib64")
 
 	return &NbdkitServer{
 		cmd:     cmd,

internal/openstack/client.go

@@ -27,6 +27,9 @@ type ClientSet struct {
 	BlockStorage *gophercloud.ServiceClient
 	Compute      *gophercloud.ServiceClient
 	Networking   *gophercloud.ServiceClient
+	// LocalCompute is scoped to the project running migratekit (no --project override).
+	// Used for volumeattach operations where the instance lives in the source project.
+	LocalCompute *gophercloud.ServiceClient
 }
 
 type PortCreateOpts struct {
@@ -39,57 +42,82 @@ func NewClientSet(ctx context.Context) (*ClientSet, error) {
 		return nil, err
 	}
 
-	provider, err := openstack.NewClient(opts.IdentityEndpoint)
-	if err != nil {
-		return nil, err
-	}
-
-	ua := gophercloud.UserAgent{}
-	ua.Prepend("migratekit")
-	provider.UserAgent = ua
-
-	config := &tls.Config{
-		MinVersion: tls.VersionTLS12,
+	tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
+	if os.Getenv("OS_INSECURE") == "true" {
+		tlsConfig.InsecureSkipVerify = true
 	}
 
-	if os.Getenv("OS_INSECURE") == "true" {
-		config.InsecureSkipVerify = true
+	newProvider := func(authOpts gophercloud.AuthOptions) (*gophercloud.ProviderClient, error) {
+		p, err := openstack.NewClient(authOpts.IdentityEndpoint)
+		if err != nil {
+			return nil, err
+		}
+		ua := gophercloud.UserAgent{}
+		ua.Prepend("migratekit")
+		p.UserAgent = ua
+		p.HTTPClient.Transport = &http.Transport{TLSClientConfig: tlsConfig}
+		if err := openstack.Authenticate(ctx, p, authOpts); err != nil {
+			return nil, err
+		}
+		return p, nil
 	}
 
-	provider.HTTPClient.Transport = &http.Transport{
-		TLSClientConfig: config,
+	project, _ := ctx.Value("osProject").(string)
+	if project != "" {
+		if isUUID(project) {
+			opts.TenantID = project
+			opts.TenantName = ""
+		} else {
+			opts.TenantName = project
+			opts.TenantID = ""
+		}
 	}
 
-	err = openstack.Authenticate(ctx, provider, opts)
+	provider, err := newProvider(opts)
 	if err != nil {
 		return nil, err
 	}
 
-	blockStorageClient, err := openstack.NewBlockStorageV3(provider, gophercloud.EndpointOpts{
-		Region: os.Getenv("OS_REGION_NAME"),
-	})
+	region := os.Getenv("OS_REGION_NAME")
+
+	blockStorageClient, err := openstack.NewBlockStorageV3(provider, gophercloud.EndpointOpts{Region: region})
 	if err != nil {
 		return nil, err
 	}
 
-	computeClient, err := openstack.NewComputeV2(provider, gophercloud.EndpointOpts{
-		Region: os.Getenv("OS_REGION_NAME"),
-	})
+	computeClient, err := openstack.NewComputeV2(provider, gophercloud.EndpointOpts{Region: region})
 	if err != nil {
 		return nil, err
 	}
 
-	networkingClient, err := openstack.NewNetworkV2(provider, gophercloud.EndpointOpts{
-		Region: os.Getenv("OS_REGION_NAME"),
-	})
+	networkingClient, err := openstack.NewNetworkV2(provider, gophercloud.EndpointOpts{Region: region})
 	if err != nil {
 		return nil, err
 	}
 
+	// When --project overrides the scope, volumeattach calls must use the source project
+	// (where the migratekit instance lives). Create a separate compute client for that.
+	localComputeClient := computeClient
+	if project != "" {
+		localOpts, err := openstack.AuthOptionsFromEnv()
+		if err != nil {
+			return nil, err
+		}
+		localProvider, err := newProvider(localOpts)
+		if err != nil {
+			return nil, err
+		}
+		localComputeClient, err = openstack.NewComputeV2(localProvider, gophercloud.EndpointOpts{Region: region})
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return &ClientSet{
 		BlockStorage: blockStorageClient,
 		Compute:      computeClient,
 		Networking:   networkingClient,
+		LocalCompute: localComputeClient,
 	}, nil
 }
 
@@ -294,3 +322,19 @@ func (c *ClientSet) CreateResourcesForVirtualMachine(ctx context.Context, vm *ob
 
 	return nil
 }
+
+func isUUID(s string) bool {
+	if len(s) != 36 {
+		return false
+	}
+	for i, c := range s {
+		if i == 8 || i == 13 || i == 18 || i == 23 {
+			if c != '-' {
+				return false
+			}
+		} else if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
+			return false
+		}
+	}
+	return true
+}

internal/target/openstack.go

@@ -195,7 +195,7 @@ func (t *OpenStack) Connect(ctx context.Context) error {
 			"instance_uuid": instanceUUID,
 		}).Info("Detected instance UUID, attaching volume...")
 
-		_, err = volumeattach.Create(ctx, t.ClientSet.Compute, instanceUUID, volumeattach.CreateOpts{
+		_, err = volumeattach.Create(ctx, t.ClientSet.LocalCompute, instanceUUID, volumeattach.CreateOpts{
 			VolumeID: volume.ID,
 		}).Extract()
 		if err != nil {
@@ -292,7 +292,7 @@ func (t *OpenStack) Disconnect(ctx context.Context) error {
 			return err
 		}
 
-		err = volumeattach.Delete(ctx, t.ClientSet.Compute, instanceUUID, volume.ID).ExtractErr()
+		err = volumeattach.Delete(ctx, t.ClientSet.LocalCompute, instanceUUID, volume.ID).ExtractErr()
 		if err != nil {
 			return err
 		}

internal/target/util.go

@@ -41,6 +41,10 @@ func NeedsFullCopy(ctx context.Context, t Target) (bool, bool, error) {
 
 	snapshotChangeId, err := vmware.GetChangeID(t.GetDisk())
 	if err != nil {
+		if errors.Is(err, vmware.ErrCBTNotEnabled) {
+			log.Warning("CBT is not enabled, forcing full copy")
+			return true, false, nil
+		}
 		return false, false, err
 	}
 

internal/vmware/change_id.go

@@ -9,6 +9,7 @@ import (
 )
 
 var ErrInvalidChangeID = errors.New("invalid change ID")
+var ErrCBTNotEnabled = errors.New("CBT is not enabled")
 
 type ChangeID struct {
 	UUID   string
@@ -45,7 +46,7 @@ func GetChangeID(disk *types.VirtualDisk) (*ChangeID, error) {
 	}
 
 	if changeId == "" {
-		return nil, fmt.Errorf("CBT is not enabled on disk %d", disk.Key)
+		return nil, fmt.Errorf("%w on disk %d", ErrCBTNotEnabled, disk.Key)
 	}
 
 	return ParseChangeID(changeId)

internal/vmware_nbdkit/vmware_nbdkit.go

@@ -2,10 +2,13 @@ package vmware_nbdkit
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"net/url"
 	"os"
 	"os/exec"
 	"os/signal"
+	"strings"
 	"syscall"
 
 	log "github.com/sirupsen/logrus"
@@ -184,6 +187,8 @@ func (s *NbdkitServers) MigrationCycle(ctx context.Context, runV2V bool) error {
 		}
 	}()
 
+	fixNicNames, _ := ctx.Value("fixNicNames").(bool)
+
 	for index, server := range s.Servers {
 		t, err := target.NewOpenStack(ctx, s.VirtualMachine, server.Disk)
 		if err != nil {
@@ -194,7 +199,7 @@ func (s *NbdkitServers) MigrationCycle(ctx context.Context, runV2V bool) error {
 			runV2V = false
 		}
 
-		err = server.SyncToTarget(ctx, t, runV2V)
+		err = server.SyncToTarget(ctx, t, runV2V, fixNicNames && index == 0)
 		if err != nil {
 			return err
 		}
@@ -308,10 +313,13 @@ func (s *NbdkitServer) IncrementalCopyToTarget(ctx context.Context, t target.Tar
 	return nil
 }
 
-func (s *NbdkitServer) SyncToTarget(ctx context.Context, t target.Target, runV2V bool) error {
+func (s *NbdkitServer) SyncToTarget(ctx context.Context, t target.Target, runV2V bool, fixNicNames bool) error {
 	snapshotChangeId, err := vmware.GetChangeID(s.Disk)
 	if err != nil {
-		return err
+		if !errors.Is(err, vmware.ErrCBTNotEnabled) {
+			return err
+		}
+		snapshotChangeId = &vmware.ChangeID{}
 	}
 
 	needFullCopy, targetIsClean, err := target.NeedsFullCopy(ctx, t)
@@ -387,5 +395,78 @@ func (s *NbdkitServer) SyncToTarget(ctx context.Context, t target.Target, runV2V
 		}
 	}
 
+	if fixNicNames {
+		if err := s.injectUdevNicRules(ctx, path); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
+
+func (s *NbdkitServer) injectUdevNicRules(ctx context.Context, path string) error {
+	if path == "" {
+		log.Warning("No block device path available, skipping udev NIC rule injection")
+		return nil
+	}
+
+	devices, err := s.Servers.VirtualMachine.Device(ctx)
+	if err != nil {
+		return err
+	}
+
+	nics := devices.SelectByType((*types.VirtualEthernetCard)(nil))
+	if len(nics) == 0 {
+		log.Info("No network adapters found, skipping udev NIC rule injection")
+		return nil
+	}
+
+	var echoLines []string
+	for i, nic := range nics {
+		card := nic.(types.BaseVirtualEthernetCard).GetVirtualEthernetCard()
+		name := nicInterfaceName(nic, i)
+
+		log.WithFields(log.Fields{
+			"mac":  card.MacAddress,
+			"name": name,
+		}).Info("Adding udev NIC rule")
+
+		rule := fmt.Sprintf(
+			`SUBSYSTEM=="net", ACTION=="add", ATTR{address}=="%s", NAME="%s"`,
+			card.MacAddress, name,
+		)
+		echoLines = append(echoLines, fmt.Sprintf("echo '%s'", rule))
+	}
+
+	// Write the rules file inline via --run-command to avoid inode corruption
+	// that --upload can cause on ext4 filesystems.
+	shellCmd := fmt.Sprintf(
+		"{ %s; } > /etc/udev/rules.d/70-persistent-net.rules",
+		strings.Join(echoLines, "; "),
+	)
+
+	os.Setenv("LIBGUESTFS_BACKEND", "direct")
+	cmd := exec.Command("virt-customize", "-a", path, "--run-command", shellCmd)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	log.WithField("path", path).Info("Injecting udev NIC rules into guest")
+	return cmd.Run()
+}
+
+func nicInterfaceName(nic types.BaseVirtualDevice, index int) string {
+	device := nic.GetVirtualDevice()
+
+	if device.SlotInfo != nil {
+		if pci, ok := device.SlotInfo.(*types.VirtualDevicePciBusSlotInfo); ok && pci.PciSlotNumber > 0 {
+			return fmt.Sprintf("ens%d", pci.PciSlotNumber)
+		}
+	}
+
+	// Fallback: VMXNET3 uses PCI slots 192, 224, 256, ... (192 + 32*index)
+	if _, ok := nic.(*types.VirtualVmxnet3); ok {
+		return fmt.Sprintf("ens%d", 192+32*index)
+	}
+
+	return fmt.Sprintf("eth%d", index)
+}