lxd/instance/drivers/qemu: Pick a random vsock Context ID

lxd-agent/api_1.0.go

@@ -98,7 +98,7 @@ func api10Put(d *Daemon, r *http.Request) response.Response {
 	}
 
 	// Try connecting to LXD server.
-	client, err := getClient(int(d.serverCID), int(d.serverPort), d.serverCertificate)
+	client, err := getClient(d.serverCID, int(d.serverPort), d.serverCertificate)
 	if err != nil {
 		return response.ErrorResponse(http.StatusInternalServerError, err.Error())
 	}
@@ -171,7 +171,7 @@ func stopDevlxdServer(d *Daemon) error {
 	return servers["devlxd"].Close()
 }
 
-func getClient(CID int, port int, serverCertificate string) (*http.Client, error) {
+func getClient(CID uint32, port int, serverCertificate string) (*http.Client, error) {
 	agentCert, err := os.ReadFile("agent.crt")
 	if err != nil {
 		return nil, err

lxd-agent/devlxd.go

@@ -43,7 +43,7 @@ type devLxdHandler struct {
 
 func getVsockClient(d *Daemon) (lxd.InstanceServer, error) {
 	// Try connecting to LXD server.
-	client, err := getClient(int(d.serverCID), int(d.serverPort), d.serverCertificate)
+	client, err := getClient(d.serverCID, int(d.serverPort), d.serverCertificate)
 	if err != nil {
 		return nil, err
 	}

lxd/instance/drivers/driver_qemu.go

@@ -375,16 +375,10 @@ func (d *qemu) getAgentClient() (*http.Client, error) {
 		return nil, err
 	}
 
-	vsockID := d.vsockID() // Default to using the vsock ID that will be used on next start.
-
-	// But if vsock ID from last VM start is present in volatile, then use that.
-	// This allows a running VM to be recovered after DB record deletion and that agent connection still work
-	// after the VM's instance ID has changed.
-	if d.localConfig["volatile.vsock_id"] != "" {
-		volatileVsockID, err := strconv.Atoi(d.localConfig["volatile.vsock_id"])
-		if err == nil {
-			vsockID = volatileVsockID
-		}
+	// Existing vsock ID from volatile.
+	vsockID, err := d.getVsockID()
+	if err != nil {
+		return nil, err
 	}
 
 	agent, err := lxdvsock.HTTPClient(vsockID, shared.HTTPSDefaultPort, clientCert, clientKey, agentCert)
@@ -1151,9 +1145,15 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error {
 
 	volatileSet := make(map[string]string)
 
+	// New or existing vsock ID from volatile.
+	vsockID, err := d.nextVsockID()
+	if err != nil {
+		return err
+	}
+
 	// Update vsock ID in volatile if needed for recovery (do this before UpdateBackupFile() call).
 	oldVsockID := d.localConfig["volatile.vsock_id"]
-	newVsockID := strconv.Itoa(d.vsockID())
+	newVsockID := strconv.FormatUint(uint64(vsockID), 10)
 	if oldVsockID != newVsockID {
 		volatileSet["volatile.vsock_id"] = newVsockID
 	}
@@ -2943,6 +2943,12 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo
 
 	cfg = append(cfg, qemuTablet(&tabletOpts)...)
 
+	// Existing vsock ID from volatile.
+	vsockID, err := d.getVsockID()
+	if err != nil {
+		return "", nil, err
+	}
+
 	devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric)
 	vsockOpts := qemuVsockOpts{
 		dev: qemuDevOpts{
@@ -2951,7 +2957,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo
 			devAddr:       devAddr,
 			multifunction: multi,
 		},
-		vsockID: d.vsockID(),
+		vsockID: vsockID,
 	}
 
 	cfg = append(cfg, qemuVsock(&vsockOpts)...)
@@ -7426,22 +7432,96 @@ func (d *qemu) DeviceEventHandler(runConf *deviceConfig.RunConfig) error {
 	return nil
 }
 
-// vsockID returns the vsock Context ID for the VM.
-func (d *qemu) vsockID() int {
-	// We use the system's own VsockID as the base.
-	//
-	// This is either "2" for a physical system or the VM's own id if
-	// running inside of a VM.
-	//
-	// To this we add 1 for backward compatibility with prior logic
-	// which would start at id 3 rather than id 2. Removing that offset
-	// would cause conflicts between existing VMs until they're all rebooted.
-	//
-	// We then add the VM's own instance id (1 or higher) to give us a
-	// unique, non-clashing context ID for our guest.
+// reservedVsockID returns true if the given vsockID equals 0, 1 or 2.
+// Those are reserved and we cannot use them.
+func (d *qemu) reservedVsockID(vsockID uint32) bool {
+	return vsockID <= 2
+}
 
-	info := DriverStatuses()[instancetype.VM].Info
-	return info.Features["vhost_vsock"].(int) + 1 + d.id
+// getVsockID returns the vsock Context ID for the VM.
+func (d *qemu) getVsockID() (uint32, error) {
+	existingVsockID, ok := d.localConfig["volatile.vsock_id"]
+	if ok {
+		vsockID, err := strconv.ParseUint(existingVsockID, 10, 32)
+		if err != nil {
+			return 0, fmt.Errorf("Failed to parse volatile.vsock_id: %q: %w", existingVsockID, err)
+		}
+
+		if d.reservedVsockID(uint32(vsockID)) {
+			return 0, fmt.Errorf("Failed to use reserved vsock Context ID: %q", vsockID)
+		}
+
+		return uint32(vsockID), nil
+	}
+
+	return 0, fmt.Errorf("Context ID not set in volatile.vsock_id")
+}
+
+// freeVsockID returns true if the given vsockID is not yet acquired.
+func (d *qemu) freeVsockID(vsockID uint32) bool {
+	c, err := lxdvsock.Dial(vsockID, shared.HTTPSDefaultPort)
+	if err != nil {
+		var unixErrno unix.Errno
+
+		if !errors.As(err, &unixErrno) {
+			return false
+		}
+
+		if unixErrno == unix.ENODEV {
+			// The syscall to the vsock device returned "no such device".
+			// This means the address (Context ID) is free.
+			return true
+		}
+	}
+
+	// Address is already in use.
+	c.Close()
+	return false
+}
+
+// nextVsockID returns the next free vsock Context ID for the VM.
+// It tries to acquire one randomly until the timeout exceeds.
+func (d *qemu) nextVsockID() (uint32, error) {
+	// Check if vsock ID from last VM start is present in volatile, then use that.
+	// This allows a running VM to be recovered after DB record deletion and that an agent connection still works
+	// after the VM's instance ID has changed.
+	// Continue in case of error since the caller requires a valid vsockID in any case.
+	vsockID, err := d.getVsockID()
+	if err == nil {
+		// Check if the vsock ID from last VM start is still not acquired in case the VM was stopped.
+		if d.freeVsockID(vsockID) {
+			return vsockID, nil
+		}
+	}
+
+	instanceUUID := uuid.Parse(d.localConfig["volatile.uuid"])
+	if instanceUUID == nil {
+		return 0, fmt.Errorf("Failed to parse instance UUID from volatile.uuid")
+	}
+
+	r, err := util.GetStableRandomGenerator(instanceUUID.String())
+	if err != nil {
+		return 0, fmt.Errorf("Failed generating stable random seed from instance UUID %q: %w", instanceUUID, err)
+	}
+
+	timeout := 5 * time.Second
+
+	// Try to find a new Context ID.
+	for start := time.Now(); time.Since(start) <= timeout; {
+		candidateVsockID := r.Uint32()
+
+		if d.reservedVsockID(candidateVsockID) {
+			continue
+		}
+
+		if d.freeVsockID(candidateVsockID) {
+			return candidateVsockID, nil
+		}
+
+		continue
+	}
+
+	return 0, fmt.Errorf("Timeout exceeded whilst trying to acquire the next vsock Context ID")
 }
 
 // InitPID returns the instance's current process ID.
@@ -7994,14 +8074,6 @@ func (d *qemu) checkFeatures(hostArch int, qemuPath string) (map[string]any, err
 		features["vhost_net"] = struct{}{}
 	}
 
-	vsockID, err := vsock.ContextID()
-	if err != nil || vsockID > 2147483647 {
-		// Fallback to the default ID for a host system
-		features["vhost_vsock"] = vsock.Host
-	} else {
-		features["vhost_vsock"] = int(vsockID)
-	}
-
 	return features, nil
 }
 

lxd/instance/drivers/driver_qemu_templates.go

@@ -326,7 +326,7 @@ func qemuSEV(opts *qemuSevOpts) []cfgSection {
 
 type qemuVsockOpts struct {
 	dev     qemuDevOpts
-	vsockID int
+	vsockID uint32
 }
 
 func qemuVsock(opts *qemuVsockOpts) []cfgSection {

lxd/vsock/vsock.go

@@ -23,7 +23,7 @@ func Dial(cid, port uint32) (net.Conn, error) {
 }
 
 // HTTPClient provides an HTTP client for using over vsock.
-func HTTPClient(vsockID int, port int, tlsClientCert string, tlsClientKey string, tlsServerCert string) (*http.Client, error) {
+func HTTPClient(vsockID uint32, port int, tlsClientCert string, tlsClientKey string, tlsServerCert string) (*http.Client, error) {
 	client := &http.Client{}
 
 	// Get the TLS configuration.
@@ -41,15 +41,15 @@ func HTTPClient(vsockID int, port int, tlsClientCert string, tlsClientKey string
 
 			// Retry for up to 1s at 100ms interval to handle various failures.
 			for i := 0; i < 10; i++ {
-				conn, err = Dial(uint32(vsockID), uint32(port))
+				conn, err = Dial(vsockID, uint32(port))
 				if err == nil {
 					break
 				} else {
 					// Handle some fatal errors.
 					msg := err.Error()
 					if strings.Contains(msg, "connection timed out") {
 						// Retry once.
-						conn, err = Dial(uint32(vsockID), uint32(port))
+						conn, err = Dial(vsockID, uint32(port))
 						break
 					} else if strings.Contains(msg, "connection refused") {
 						break