717b8003-5b59-444d-bc4d-5099b6c3f635

Committed 17 Jul 2026 11:28AM UTC coverage: 72.447%. Remained the same

Build # 717b8003-5b59-444d-bc4d-5099b6c3f635

Build Type

push

prow

Committed by

web-flow

Commit Message

Merge pull request #18044 from maxwmsft/fix-hotplug-volume-detach-deadlock

Fix hotplug volume detach deadlock in virt-handler

Coverage Stats

10 of 10 new or added lines in 1 file covered. (100.0%)

91 existing lines in 2 files now uncovered.

84123 of 116116 relevant lines covered (72.45%)

363.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

57.66

/pkg/virt-handler/hotplug-disk/mount.go

/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright The KubeVirt Authors.
 *
 */

package hotplug_volume

import (
        "errors"
        "fmt"
        "os"
        "path/filepath"
        "sync"
        "syscall"

        "kubevirt.io/kubevirt/pkg/checkpoint"
        "kubevirt.io/kubevirt/pkg/unsafepath"

        "golang.org/x/sys/unix"

        "kubevirt.io/kubevirt/pkg/safepath"
        virt_chroot "kubevirt.io/kubevirt/pkg/virt-handler/virt-chroot"

        diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils"
        hotplugdisk "kubevirt.io/kubevirt/pkg/hotplug-disk"
        storagetypes "kubevirt.io/kubevirt/pkg/storage/types"
        "kubevirt.io/kubevirt/pkg/virt-handler/cgroup"
        "kubevirt.io/kubevirt/pkg/virt-handler/isolation"

        "github.com/opencontainers/cgroups"

        devices "github.com/opencontainers/cgroups/devices/config"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/apimachinery/pkg/util/sets"

        v1 "kubevirt.io/api/core/v1"
        "kubevirt.io/client-go/log"
)

//go:generate mockgen -source $GOFILE -package=$GOPACKAGE -destination=generated_mock_$GOFILE

var ErrWaitingForHotplugMount = errors.New("waiting for hotplug mount")

const (
        unableFindHotplugMountedDir = "unable to find hotplug mounted directories for vmi without uid"
)

var (
        nodeIsolationResult = func() isolation.IsolationResult {
                return isolation.NodeIsolationResult()
        }
        deviceBasePath = func(podUID types.UID, kubeletPodsDir string) (*safepath.Path, error) {
                return safepath.JoinAndResolveWithRelativeRoot("/proc/1/root", kubeletPodsDir, fmt.Sprintf("/%s/volumes/kubernetes.io~empty-dir/hotplug-disks", string(podUID)))
        }

        socketPath = func(podUID types.UID) string {
                return fmt.Sprintf("/pods/%s/volumes/kubernetes.io~empty-dir/hotplug-disks/hp.sock", string(podUID))
        }

        statDevice = func(fileName *safepath.Path) (os.FileInfo, error) {
                info, err := safepath.StatAtNoFollow(fileName)
                if err != nil {
                        return nil, err
                }
                if info.Mode()&os.ModeDevice == 0 {
                        return info, fmt.Errorf("%v is not a block device", fileName)
                }
                return info, nil
        }

        statSourceDevice = func(fileName *safepath.Path) (os.FileInfo, error) {
                // we don't know the device name, we only know that it is the only
                // device in a specific directory, let's look it up
                var devName string
                err := fileName.ExecuteNoFollow(func(safePath string) error {
                        entries, err := os.ReadDir(safePath)
                        if err != nil {
                                return err
                        }
                        for _, entry := range entries {
                                info, err := entry.Info()
                                if err != nil {
                                        return err
                                }
                                if info.Mode()&os.ModeDevice == 0 {
                                        // not a device
                                        continue
                                }
                                devName = entry.Name()
                                return nil
                        }
                        return fmt.Errorf("no device in %v", fileName)
                })
                if err != nil {
                        return nil, err
                }
                devPath, err := safepath.JoinNoFollow(fileName, devName)
                if err != nil {
                        return nil, err
                }
                return statDevice(devPath)
        }

        mknodCommand = func(basePath *safepath.Path, deviceName string, dev uint64, blockDevicePermissions os.FileMode) error {
                return safepath.MknodAtNoFollow(basePath, deviceName, blockDevicePermissions|syscall.S_IFBLK, dev)
        }

        mountCommand = func(sourcePath, targetPath *safepath.Path) ([]byte, error) {
                return virt_chroot.MountChroot(sourcePath, targetPath, false).CombinedOutput()
        }

        unmountCommand = func(diskPath *safepath.Path) ([]byte, error) {
                return virt_chroot.UmountChroot(diskPath).CombinedOutput()
        }

        isMounted = func(path *safepath.Path) (bool, error) {
                return isolation.IsMounted(path)
        }

        isBlockDevice = func(path *safepath.Path) (bool, error) {
                return isolation.IsBlockDevice(path)
        }

        isolationDetector = func() isolation.PodIsolationDetector {
                return isolation.NewSocketBasedIsolationDetector()
        }

        parentPathForMount = func(
                parent isolation.IsolationResult,
                child isolation.IsolationResult,
                findmntInfo FindmntInfo,
                podUID string,
                kubeletPodsDir string,
        ) (*safepath.Path, error) {
                return isolation.ParentPathForMount(parent, child, findmntInfo.Source, findmntInfo.Target, podUID, kubeletPodsDir)
        }
)

type volumeMounter struct {
        checkpointManager  checkpoint.CheckpointManager
        mountRecords       map[types.UID]*vmiMountTargetRecord
        mountRecordsLock   sync.Mutex
        skipSafetyCheck    bool
        hotplugDiskManager hotplugdisk.HotplugDiskManagerInterface
        ownershipManager   diskutils.OwnershipManagerInterface
        kubeletPodsDir     string
        host               string
}

// VolumeMounter is the interface used to mount and unmount volumes to/from a running virtlauncher pod.
type VolumeMounter interface {
        // Mount any new volumes defined in the VMI
        Mount(vmi *v1.VirtualMachineInstance, cgroupManager cgroup.Manager) error
        MountFromPod(vmi *v1.VirtualMachineInstance, sourceUID types.UID, cgroupManager cgroup.Manager) error
        // Unmount any volumes no longer defined in the VMI
        Unmount(vmi *v1.VirtualMachineInstance, cgroupManager cgroup.Manager) error
        //UnmountAll cleans up all hotplug volumes
        UnmountAll(vmi *v1.VirtualMachineInstance, cgroupManager cgroup.Manager) error
        //IsMounted returns if the volume is mounted or not.
        IsMounted(vmi *v1.VirtualMachineInstance, volume string, sourceUID types.UID) (bool, error)
}

type vmiMountTargetEntry struct {
        TargetFile string `json:"targetFile"`
}

type vmiMountTargetRecord struct {
        MountTargetEntries []vmiMountTargetEntry `json:"mountTargetEntries"`
        UsesSafePaths      bool                  `json:"usesSafePaths"`
}

func (r *vmiMountTargetRecord) appendPath(path string) bool {
        for _, entry := range r.MountTargetEntries {
                if entry.TargetFile == path {
                        return false // skip appending if already present
                }
        }
        r.MountTargetEntries = append(r.MountTargetEntries, vmiMountTargetEntry{TargetFile: path})
        return true
}

// NewVolumeMounter creates a new VolumeMounter
func NewVolumeMounter(mountStateDir string, kubeletPodsDir string, host string) VolumeMounter {
        return &volumeMounter{
                mountRecords:       make(map[types.UID]*vmiMountTargetRecord),
                checkpointManager:  checkpoint.NewSimpleCheckpointManager(mountStateDir),
                hotplugDiskManager: hotplugdisk.NewHotplugDiskManager(kubeletPodsDir),
                ownershipManager:   diskutils.DefaultOwnershipManager,
                kubeletPodsDir:     kubeletPodsDir,
                host:               host,
        }
}

func (m *volumeMounter) deleteMountTargetRecord(vmi *v1.VirtualMachineInstance) error {
        if string(vmi.UID) == "" {
                return fmt.Errorf(unableFindHotplugMountedDir)
        }

        record := vmiMountTargetRecord{}
        err := m.checkpointManager.Get(string(vmi.UID), &record)
        if err != nil && !errors.Is(err, os.ErrNotExist) {
                return fmt.Errorf("failed to get checkpoint %s, %w", vmi.UID, err)
        }

        if err == nil {
                for _, target := range record.MountTargetEntries {
                        os.Remove(target.TargetFile)
                }

                if err := m.checkpointManager.Delete(string(vmi.UID)); err != nil {
                        return fmt.Errorf("failed to delete checkpoint %s, %w", vmi.UID, err)
                }
        }

        m.mountRecordsLock.Lock()
        defer m.mountRecordsLock.Unlock()
        delete(m.mountRecords, vmi.UID)

        return nil
}

func (m *volumeMounter) getMountTargetRecord(vmi *v1.VirtualMachineInstance) (*vmiMountTargetRecord, error) {
        var ok bool
        var existingRecord *vmiMountTargetRecord

        if string(vmi.UID) == "" {
                return nil, fmt.Errorf(unableFindHotplugMountedDir)
        }

        m.mountRecordsLock.Lock()
        defer m.mountRecordsLock.Unlock()
        existingRecord, ok = m.mountRecords[vmi.UID]

        // first check memory cache
        if ok {
                return existingRecord, nil
        }

        // if not there, see if record is on disk, this can happen if virt-handler restarts
        record := vmiMountTargetRecord{}
        err := m.checkpointManager.Get(string(vmi.UID), &record)
        if err != nil && !errors.Is(err, os.ErrNotExist) {
                return nil, fmt.Errorf("failed to get checkpoint %s, %w", vmi.UID, err)
        }

        if err == nil {
                // XXX: backward compatibility for old unresolved paths, can be removed in July 2023
                // After a one-time convert and persist, old records are safe too.
                if !record.UsesSafePaths {
                        for i, path := range record.MountTargetEntries {
                                record.UsesSafePaths = true
                                safePath, err := safepath.JoinAndResolveWithRelativeRoot("/", path.TargetFile)
                                if err != nil {
                                        return nil, fmt.Errorf("failed converting legacy path to safepath: %v", err)
                                }
                                record.MountTargetEntries[i].TargetFile = unsafepath.UnsafeAbsolute(safePath.Raw())
                        }
                }

                m.mountRecords[vmi.UID] = &record
                return &record, nil
        }

        // not found
        return &vmiMountTargetRecord{UsesSafePaths: true}, nil
}

func (m *volumeMounter) setMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error {
        if string(vmi.UID) == "" {
                return fmt.Errorf(unableFindHotplugMountedDir)
        }

        // XXX: backward compatibility for old unresolved paths, can be removed in July 2023
        // After a one-time convert and persist, old records are safe too.
        record.UsesSafePaths = true

        m.mountRecordsLock.Lock()
        defer m.mountRecordsLock.Unlock()

        if err := m.checkpointManager.Store(string(vmi.UID), record); err != nil {
                return fmt.Errorf("failed to checkpoint %s, %w", vmi.UID, err)
        }
        m.mountRecords[vmi.UID] = record
        return nil
}

func (m *volumeMounter) writePathToMountRecord(path string, vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error {
        if !record.appendPath(path) {
                return nil
        }
        if err := m.setMountTargetRecord(vmi, record); err != nil {
                return err
        }
        return nil
}

func (m *volumeMounter) mountHotplugVolume(
        vmi *v1.VirtualMachineInstance,
        volumeName string,
        sourceUID types.UID,
        record *vmiMountTargetRecord,
        mountDirectory bool,
        cgroupManager cgroup.Manager,
) error {
        logger := log.Log.Object(vmi)
        logger.V(4).Infof("Hotplug check volume name: %s", volumeName)
        if sourceUID != "" {
                if m.isBlockVolume(&vmi.Status, volumeName) {
                        logger.V(3).Infof("Mounting block volume: %s", volumeName)
                        if err := m.mountBlockHotplugVolume(vmi, volumeName, sourceUID, record, cgroupManager); err != nil {
                                return fmt.Errorf("failed to mount block hotplug volume %s: %w", volumeName, err)
                        }
                } else {
                        logger.V(3).Infof("Mounting file system volume: %s", volumeName)
                        if err := m.mountFileSystemHotplugVolume(vmi, volumeName, sourceUID, record, mountDirectory); err != nil {
                                return fmt.Errorf("failed to mount filesystem hotplug volume %s: %w", volumeName, err)
                        }
                }
        }
        return nil
}

func (m *volumeMounter) Mount(vmi *v1.VirtualMachineInstance, cgroupManager cgroup.Manager) error {
        return m.mountFromPod(vmi, "", cgroupManager)
}

func (m *volumeMounter) MountFromPod(vmi *v1.VirtualMachineInstance, sourceUID types.UID, cgroupManager cgroup.Manager) error {
        return m.mountFromPod(vmi, sourceUID, cgroupManager)
}

func (m *volumeMounter) mountFromPod(vmi *v1.VirtualMachineInstance, sourceUID types.UID, cgroupManager cgroup.Manager) error {
        record, err := m.getMountTargetRecord(vmi)
        if err != nil {
                return err
        }

        specVolumes := sets.New[string]()
        for i := range vmi.Spec.Volumes {
                specVolumes.Insert(vmi.Spec.Volumes[i].Name)
        }
        for i := range vmi.Spec.UtilityVolumes {
                specVolumes.Insert(vmi.Spec.UtilityVolumes[i].Name)
        }

        for _, volumeStatus := range vmi.Status.VolumeStatus {
                if volumeStatus.HotplugVolume == nil {
                        // Skip non hotplug volumes
                        continue
                }

                if !specVolumes.Has(volumeStatus.Name) {
                        log.Log.Object(vmi).V(3).Infof("Skipping mount for volume %s: no longer in VMI spec", volumeStatus.Name)
                        continue
                }

                if storagetypes.IsUtilityVolume(vmi, volumeStatus.Name) && m.isBlockVolume(&vmi.Status, volumeStatus.Name) {
                        log.Log.Object(vmi).Warningf("Skipping mount for utility volume %s: configured with block mode PVC, utility volumes require filesystem mode", volumeStatus.Name)
                        continue
                }

                mountDirectory := m.isDirectoryMounted(vmi, volumeStatus.Name)
                volumeSourceUID := sourceUID
                if volumeSourceUID == "" {
                        volumeSourceUID = volumeStatus.HotplugVolume.AttachPodUID
                }
                if err := m.mountHotplugVolume(vmi, volumeStatus.Name, volumeSourceUID, record, mountDirectory, cgroupManager); err != nil {
                        return err
                }
        }
        return nil
}

func (m *volumeMounter) isDirectoryMounted(vmi *v1.VirtualMachineInstance, volumeName string) bool {
        for _, utilityVolume := range vmi.Spec.UtilityVolumes {
                if utilityVolume.Name == volumeName {
                        return true
                }
        }
        for _, volume := range vmi.Spec.Volumes {
                if volume.Name == volumeName {
                        return volume.MemoryDump != nil
                }
        }
        return false
}

// isBlockVolume checks if the volumeDevices directory exists in the pod path, we assume there is a single volume associated with
// each pod, we use this knowledge to determine if we have a block volume or not.
func (m *volumeMounter) isBlockVolume(vmiStatus *v1.VirtualMachineInstanceStatus, volumeName string) bool {
        // First evaluate the migrated volumed. In the case of a migrated volume, virt-handler needs to understand if it needs to consider the
        // volume mode of the source or the destination. Therefore, it evaluates if its is running on the destiation host or otherwise, it is the source.
        isDstHost := false
        if vmiStatus.MigrationState != nil {
                isDstHost = vmiStatus.MigrationState.TargetNode == m.host
        }
        for _, migVol := range vmiStatus.MigratedVolumes {
                if migVol.VolumeName == volumeName {
                        if isDstHost && migVol.DestinationPVCInfo != nil {
                                return storagetypes.IsPVCBlock(migVol.DestinationPVCInfo.VolumeMode)
                        }
                        if !isDstHost && migVol.SourcePVCInfo != nil {
                                return storagetypes.IsPVCBlock(migVol.SourcePVCInfo.VolumeMode)
                        }
                }
        }
        // Check if the volumeDevices directory exists in the attachment pod, if so, its a block device, otherwise its file system.
        for _, status := range vmiStatus.VolumeStatus {
                if status.Name == volumeName {
                        return status.PersistentVolumeClaimInfo != nil && storagetypes.IsPVCBlock(status.PersistentVolumeClaimInfo.VolumeMode)
                }
        }
        return false
}

func (m *volumeMounter) mountBlockHotplugVolume(
        vmi *v1.VirtualMachineInstance,
        volume string,
        sourceUID types.UID,
        record *vmiMountTargetRecord,
        cgroupManager cgroup.Manager,
) error {
        virtlauncherUID := m.findVirtlauncherUID(vmi)
        if virtlauncherUID == "" {
                // This is not the node the pod is running on.
                return nil
        }
        targetPath, err := m.hotplugDiskManager.GetHotplugTargetPodPathOnHost(virtlauncherUID)
        if err != nil {
                return err
        }

        if _, err := safepath.JoinNoFollow(targetPath, volume); errors.Is(err, os.ErrNotExist) {
                dev, permissions, err := m.getSourceMajorMinor(sourceUID, volume)
                if err != nil {
                        return err
                }

                if err := m.writePathToMountRecord(filepath.Join(unsafepath.UnsafeAbsolute(targetPath.Raw()), volume), vmi, record); err != nil {
                        return err
                }

                if err := m.createBlockDeviceFile(targetPath, volume, dev, permissions); err != nil && !os.IsExist(err) {
                        return err
                }
                log.Log.Object(vmi).V(1).Infof("successfully created block device %s", volume)
        } else if err != nil {
                return err
        }

        devicePath, err := safepath.JoinNoFollow(targetPath, volume)
        if err != nil {
                return err
        }
        if isBlockExists, err := isBlockDevice(devicePath); err != nil {
                return err
        } else if !isBlockExists {
                return fmt.Errorf("target device %v exists but it is not a block device", devicePath)
        }

        dev, _, err := m.getBlockFileMajorMinor(devicePath, statDevice)
        if err != nil {
                return err
        }
        // allow block devices
        if err := m.allowBlockMajorMinor(dev, cgroupManager); err != nil {
                return err
        }

        return m.ownershipManager.SetFileOwnership(devicePath)
}

func (m *volumeMounter) getSourceMajorMinor(sourceUID types.UID, volumeName string) (uint64, os.FileMode, error) {
        basePath, err := deviceBasePath(sourceUID, m.kubeletPodsDir)
        if err != nil {
                return 0, 0, err
        }
        devicePath, err := basePath.AppendAndResolveWithRelativeRoot(volumeName)
        if err != nil {
                return 0, 0, err
        }
        return m.getBlockFileMajorMinor(devicePath, statSourceDevice)
}

func (m *volumeMounter) getBlockFileMajorMinor(devicePath *safepath.Path, getter func(fileName *safepath.Path) (os.FileInfo, error)) (uint64, os.FileMode, error) {
        fileInfo, err := getter(devicePath)
        if err != nil {
                return 0, 0, err
        }
        info := fileInfo.Sys().(*syscall.Stat_t)
        return info.Rdev, fileInfo.Mode(), nil
}

func (m *volumeMounter) removeBlockMajorMinor(dev uint64, cgroupManager cgroup.Manager) error {
        return m.updateBlockMajorMinor(dev, false, cgroupManager)
}

func (m *volumeMounter) allowBlockMajorMinor(dev uint64, cgroupManager cgroup.Manager) error {
        return m.updateBlockMajorMinor(dev, true, cgroupManager)
}

func (m *volumeMounter) updateBlockMajorMinor(dev uint64, allow bool, cgroupManager cgroup.Manager) error {
        deviceRule := &devices.Rule{
                Type:        devices.BlockDevice,
                Major:       int64(unix.Major(dev)),
                Minor:       int64(unix.Minor(dev)),
                Permissions: "rwm",
                Allow:       allow,
        }

        if cgroupManager == nil {
                return fmt.Errorf("failed to apply device rule %+v: cgroup manager is nil", *deviceRule)
        }

        err := cgroupManager.Set(&cgroups.Resources{
                Devices: []*devices.Rule{deviceRule},
        })

        if err != nil {
                log.Log.Errorf("cgroup %s had failed to set device rule. error: %v. rule: %+v", cgroupManager.GetCgroupVersion(), err, *deviceRule)
        } else {
                log.Log.Infof("cgroup %s device rule is set successfully. rule: %+v", cgroupManager.GetCgroupVersion(), *deviceRule)
        }

        return err
}

func (m *volumeMounter) createBlockDeviceFile(basePath *safepath.Path, deviceName string, dev uint64, blockDevicePermissions os.FileMode) error {
        if _, err := safepath.JoinNoFollow(basePath, deviceName); errors.Is(err, os.ErrNotExist) {
                return mknodCommand(basePath, deviceName, dev, blockDevicePermissions)
        } else {
                return err
        }
}

func (m *volumeMounter) mountFileSystemHotplugVolume(vmi *v1.VirtualMachineInstance, volume string, sourceUID types.UID, record *vmiMountTargetRecord, mountDirectory bool) error {
        virtlauncherUID := m.findVirtlauncherUID(vmi)
        if virtlauncherUID == "" {
                // This is not the node the pod is running on.
                return nil
        }
        var target *safepath.Path
        var err error
        if mountDirectory {
                target, err = m.hotplugDiskManager.GetFileSystemDirectoryTargetPathFromHostView(virtlauncherUID, volume, true)
        } else {
                target, err = m.hotplugDiskManager.GetFileSystemDiskTargetPathFromHostView(virtlauncherUID, volume, true)
        }
        if err != nil {
                return err
        }

        isMounted, err := isMounted(target)
        if err != nil {
                return fmt.Errorf("failed to determine if %s is already mounted: %v", target, err)
        }
        if !isMounted {
                sourcePath, err := m.getSourcePodFilePath(sourceUID, vmi, volume)
                if err != nil {
                        return fmt.Errorf("failed to get source path for volume %s from source pod %s: %v: %w", volume, sourceUID, err, ErrWaitingForHotplugMount)
                }
                if err := m.writePathToMountRecord(unsafepath.UnsafeAbsolute(target.Raw()), vmi, record); err != nil {
                        return err
                }
                if !mountDirectory {
                        sourcePath, err = sourcePath.AppendAndResolveWithRelativeRoot("disk.img")
                        if err != nil {
                                return err
                        }
                }
                if out, err := mountCommand(sourcePath, target); err != nil {
                        return fmt.Errorf("failed to bindmount hotplug volume source from %v to %v: %v : %v", sourcePath, target, string(out), err)
                }
                log.Log.Object(vmi).V(1).Infof("successfully mounted hotplug volume %s", volume)
        }

        return m.ownershipManager.SetFileOwnership(target)
}

func (m *volumeMounter) findVirtlauncherUID(vmi *v1.VirtualMachineInstance) (uid types.UID) {
        cnt := 0
        for podUID := range vmi.Status.ActivePods {
                _, err := m.hotplugDiskManager.GetHotplugTargetPodPathOnHost(podUID)
                if err == nil {
                        uid = podUID
                        cnt++
                }
        }
        if cnt == 1 {
                return
        }
        // Either no pods, or multiple pods, skip.
        return ""
}

func (m *volumeMounter) getSourcePodFilePath(sourceUID types.UID, vmi *v1.VirtualMachineInstance, volume string) (*safepath.Path, error) {
        iso := isolationDetector()
        isoRes, err := iso.DetectForSocket(socketPath(sourceUID))
        if err != nil {
                return nil, err
        }
        findmounts, err := LookupFindmntInfoByVolume(volume, isoRes.Pid())
        if err != nil {
                return nil, err
        }
        nodeIsoRes := nodeIsolationResult()
        mountRoot, err := nodeIsoRes.MountRoot()
        if err != nil {
                return nil, err
        }

        for _, findmnt := range findmounts {
                if filepath.Base(findmnt.Target) == volume {
                        source := findmnt.GetSourcePath()
                        path, err := parentPathForMount(nodeIsoRes, isoRes, findmnt, string(sourceUID), m.kubeletPodsDir)
                        exists := !errors.Is(err, os.ErrNotExist)
                        if err != nil && !errors.Is(err, os.ErrNotExist) {
                                return nil, err
                        }

                        isBlock := false
                        if exists {
                                isBlock, _ = isBlockDevice(path)
                        }

                        if !exists || isBlock {
                                // file not found, or block device, or directory check if we can find the mount.
                                deviceFindMnt, err := LookupFindmntInfoByDevice(source)
                                if err != nil {
                                        // Try the device found from the source
                                        deviceFindMnt, err = LookupFindmntInfoByDevice(findmnt.GetSourceDevice())
                                        if err != nil {
                                                return nil, err
                                        }
                                        // Check if the path was relative to the device.
                                        if !exists {
                                                return mountRoot.AppendAndResolveWithRelativeRoot(deviceFindMnt[0].Target, source)
                                        }
                                        return nil, err
                                }
                                return mountRoot.AppendAndResolveWithRelativeRoot(deviceFindMnt[0].Target)
                        } else {
                                return path, nil
                        }
                }
        }
        // Did not find the disk image file, return error
        return nil, fmt.Errorf("unable to find source disk image path for pod %s", sourceUID)
}

// Unmount unmounts all hotplug disk that are no longer part of the VMI
func (m *volumeMounter) Unmount(vmi *v1.VirtualMachineInstance, cgroupManager cgroup.Manager) error {
        if vmi.UID != "" {
                record, err := m.getMountTargetRecord(vmi)
                if err != nil {
                        return err
                } else if record == nil {
                        // no entries to unmount
                        return nil
                }
                if len(record.MountTargetEntries) == 0 {
                        return nil
                }

                currentHotplugPaths := make(map[string]types.UID)
                virtlauncherUID := m.findVirtlauncherUID(vmi)

                basePath, err := m.hotplugDiskManager.GetHotplugTargetPodPathOnHost(virtlauncherUID)
                if err != nil {
                        if errors.Is(err, os.ErrNotExist) {
                                // no mounts left, the base path does not even exist anymore
                                if err := m.deleteMountTargetRecord(vmi); err != nil {
                                        return fmt.Errorf("failed to delete mount target records: %v", err)
                                }
                                return nil
                        }
                        return err
                }
                // Ideally we would be looking at the actual disks in the domain but:
                // 1. The domain is built from the VMI spec
                // 2. The domain syncs before unmount is called
                // 3. Unmount will not get called if VMI sync fails
                // we should be good
                for _, volume := range vmi.Spec.Volumes {
                        if !storagetypes.IsHotplugVolume(&volume) {
                                continue
                        }
                        var path *safepath.Path
                        var err error
                        if m.isBlockVolume(&vmi.Status, volume.Name) {
                                path, err = safepath.JoinNoFollow(basePath, volume.Name)
                                if errors.Is(err, os.ErrNotExist) {
                                        // already unmounted or never mounted
                                        continue
                                }
                        } else if m.isDirectoryMounted(vmi, volume.Name) {
                                path, err = m.hotplugDiskManager.GetFileSystemDirectoryTargetPathFromHostView(virtlauncherUID, volume.Name, false)
                                if errors.Is(err, os.ErrNotExist) {
                                        // already unmounted or never mounted
                                        continue
                                }
                        } else {
                                path, err = m.hotplugDiskManager.GetFileSystemDiskTargetPathFromHostView(virtlauncherUID, volume.Name, false)
                                if errors.Is(err, os.ErrNotExist) {
                                        // already unmounted or never mounted
                                        continue
                                }
                        }
                        if err != nil {
                                return err
                        }
                        currentHotplugPaths[unsafepath.UnsafeAbsolute(path.Raw())] = virtlauncherUID
                }
                for _, utilityVolume := range vmi.Spec.UtilityVolumes {
                        if m.isBlockVolume(&vmi.Status, utilityVolume.Name) {
                                log.Log.Object(vmi).Warningf("Skipping unmount cleanup for utility volume %s: configured with block mode PVC", utilityVolume.Name)
                                continue
                        }

                        var path *safepath.Path
                        var err error
                        path, err = m.hotplugDiskManager.GetFileSystemDirectoryTargetPathFromHostView(virtlauncherUID, utilityVolume.Name, false)
                        if errors.Is(err, os.ErrNotExist) {
                                // already unmounted or never mounted
                                continue
                        }
                        if err != nil {
                                return err
                        }
                        currentHotplugPaths[unsafepath.UnsafeAbsolute(path.Raw())] = virtlauncherUID
                }
                newRecord := vmiMountTargetRecord{
                        MountTargetEntries: make([]vmiMountTargetEntry, 0),
                }
                var errs []error
                logErrAndKeepMount := func(path string, format string, args ...any) {
                        err := fmt.Errorf(format, args...)
                        log.Log.Object(vmi).Error(err.Error())
                        newRecord.appendPath(path)
                        errs = append(errs, err)
                }
                for _, entry := range record.MountTargetEntries {
                        fd, err := safepath.NewFileNoFollow(entry.TargetFile)
                        if err != nil {
                                if errors.Is(err, os.ErrNotExist) {
                                        log.Log.Object(vmi).Infof("Volume %v is not mounted anymore, continuing", entry.TargetFile)
                                } else {
                                        logErrAndKeepMount(entry.TargetFile, "Unable to unmount volume at path %s: %v", entry.TargetFile, err)
                                }
                                continue
                        }
                        fd.Close()
                        diskPath := fd.Path()
                        diskPathAbs := unsafepath.UnsafeAbsolute(diskPath.Raw())

                        if _, ok := currentHotplugPaths[diskPathAbs]; !ok {
                                if blockDevice, err := isBlockDevice(diskPath); err != nil {
                                        logErrAndKeepMount(diskPathAbs, "Unable to unmount volume at path %s: %v", diskPath, err)
                                        continue
                                } else if blockDevice {
                                        if err := m.unmountBlockHotplugVolumes(diskPath, cgroupManager); err != nil {
                                                logErrAndKeepMount(diskPathAbs, "Unable to remove block device at path %s: %v", diskPath, err)
                                                continue
                                        }
                                } else if err := m.unmountFileSystemHotplugVolumes(diskPath); err != nil {
                                        logErrAndKeepMount(diskPathAbs, "Unable to unmount filesystem volume at path %s: %v", diskPath, err)
                                        continue
                                }
                                log.Log.Object(vmi).V(3).Infof("Unmounted hotplug volume path %s", diskPath)
                        } else {
                                _ = newRecord.appendPath(diskPathAbs)
                        }
                }
                if len(newRecord.MountTargetEntries) > 0 {
                        err = m.setMountTargetRecord(vmi, &newRecord)
                } else {
                        err = m.deleteMountTargetRecord(vmi)
                }
                if err != nil {
                        return err
                }
                if len(errs) > 0 {
                        return fmt.Errorf("failed to cleanup hotplug mounts for VMI %s: %w", vmi.Name, errors.Join(errs...))
                }
        }
        return nil
}

func (m *volumeMounter) unmountFileSystemHotplugVolumes(diskPath *safepath.Path) error {
        if mounted, err := isMounted(diskPath); err != nil {
                return fmt.Errorf("failed to check mount point for hotplug disk %v: %v", diskPath, err)
        } else if mounted {
                out, err := unmountCommand(diskPath)
                if err != nil {
                        return fmt.Errorf("failed to unmount hotplug disk %v: %v : %v", diskPath, string(out), err)
                }
                err = safepath.UnlinkAtNoFollow(diskPath)
                if err != nil {
                        return fmt.Errorf("failed to remove hotplug disk directory %v: %v : %v", diskPath, string(out), err)
                }

        }
        return nil
}

func (m *volumeMounter) unmountBlockHotplugVolumes(diskPath *safepath.Path, cgroupManager cgroup.Manager) error {
        // Get major and minor so we can deny the container.
        dev, _, err := m.getBlockFileMajorMinor(diskPath, statDevice)
        if err != nil {
                return err
        }
        // Delete block device file
        if err := safepath.UnlinkAtNoFollow(diskPath); err != nil {
                return err
        }
        return m.removeBlockMajorMinor(dev, cgroupManager)
}

// UnmountAll unmounts all hotplug disks of a given VMI.
func (m *volumeMounter) UnmountAll(vmi *v1.VirtualMachineInstance, cgroupManager cgroup.Manager) error {
        if vmi.UID != "" {
                logger := log.DefaultLogger().Object(vmi)
                logger.Info("Cleaning up remaining hotplug volumes")
                record, err := m.getMountTargetRecord(vmi)
                if err != nil {
                        return err
                } else if record == nil {
                        // no entries to unmount
                        logger.Info("No hotplug volumes found to unmount")
                        return nil
                }

                for _, entry := range record.MountTargetEntries {
                        diskPath, err := safepath.NewFileNoFollow(entry.TargetFile)
                        if err != nil {
                                if errors.Is(err, os.ErrNotExist) {
                                        logger.Infof("Device %v is not mounted anymore, continuing.", entry.TargetFile)
                                        continue
                                }
                                logger.Warningf("Unable to unmount volume at path %s: %v", entry.TargetFile, err)
                                continue
                        }
                        diskPath.Close()
                        if isBlock, err := isBlockDevice(diskPath.Path()); err != nil {
                                logger.Warningf("Unable to unmount volume at path %s: %v", diskPath, err)
                        } else if isBlock {
                                if err := m.unmountBlockHotplugVolumes(diskPath.Path(), cgroupManager); err != nil {
                                        logger.Warningf("Unable to remove block device at path %s: %v", diskPath, err)
                                        // Don't return error, try next.
                                }
                        } else {
                                if err := m.unmountFileSystemHotplugVolumes(diskPath.Path()); err != nil {
                                        logger.Warningf("Unable to unmount volume at path %s: %v", diskPath, err)
                                        // Don't return error, try next.
                                }
                        }
                }
                err = m.deleteMountTargetRecord(vmi)
                if err != nil {
                        return err
                }
        }
        return nil
}

func (m *volumeMounter) IsMounted(vmi *v1.VirtualMachineInstance, volume string, sourceUID types.UID) (bool, error) {
        virtlauncherUID := m.findVirtlauncherUID(vmi)
        if virtlauncherUID == "" {
                // This is not the node the pod is running on.
                return false, fmt.Errorf("Unable to determine virt-launcher UID")
        }
        targetPath, err := m.hotplugDiskManager.GetHotplugTargetPodPathOnHost(virtlauncherUID)
        if err != nil {
                if errors.Is(err, os.ErrNotExist) {
                        return false, nil
                }
                return false, err
        }
        if m.isBlockVolume(&vmi.Status, volume) {
                deviceName, err := safepath.JoinNoFollow(targetPath, volume)
                if err != nil {
                        if errors.Is(err, os.ErrNotExist) {
                                return false, nil
                        }
                        return false, err
                }
                isBlockExists, _ := isBlockDevice(deviceName)
                return isBlockExists, nil
        }
        if m.isDirectoryMounted(vmi, volume) {
                path, err := safepath.JoinNoFollow(targetPath, volume)
                if err != nil {
                        if errors.Is(err, os.ErrNotExist) {
                                return false, nil
                        }
                        return false, err
                }
                return isMounted(path)
        }
        path, err := safepath.JoinNoFollow(targetPath, fmt.Sprintf("%s.img", volume))
        if err != nil {
                if errors.Is(err, os.ErrNotExist) {
                        return false, nil
                }
                return false, err
        }
        return isMounted(path)
}

kubevirt / kubevirt / 717b8003-5b59-444d-bc4d-5099b6c3f635

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous