#5755

Committed 08 Jan 2026 02:28AM UTC coverage: 49.428% (+0.04%) from 49.388%

Build # #5755

Build Type

Pull #3901

travis-ci

Committed by

howardgao

Commit Message

Host assisted clone should adjust requested storage
when cloning from block to filesystem volume mode (#3900)

When cloning from a source, the host cloner should
check the requested size for the target tmp pvc if
the tmp source pvc is of block volume mode and the
target pvc is of filesystem volume mode because it
should take filesystem overhead into consideration
and enlarge the original request size accordingly
before creating the tmp pvc.

Because currently the check is missing the cloning
will fail on validation.

Signed-off-by: Howard Gao <howard.gao@gmail.com>

Pull Request Pull Request #3901: Host assisted clone should adjust requested storage when cloning from block to filesystem volume mode (#3900)

Run Details

38 of 89 new or added lines in 2 files covered. (42.7%)

2 existing lines in 1 file now uncovered.

14657 of 29653 relevant lines covered (49.43%)

0.55 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

51.4

/pkg/controller/clone/host-clone.go

package clone

import (
        "context"
        "fmt"
        "net/http"
        "strconv"
        "time"

        "github.com/go-logr/logr"

        corev1 "k8s.io/api/core/v1"
        k8serrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/tools/record"

        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/reconcile"

        cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
        cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
        metrics "kubevirt.io/containerized-data-importer/pkg/monitoring/metrics/cdi-cloner"
)

// HostClonePhaseName is the name of the host clone phase
const HostClonePhaseName = "HostClone"

// HostClonePhase creates and monitors a dumb clone operation
type HostClonePhase struct {
        Owner             client.Object
        Namespace         string
        SourceName        string
        DesiredClaim      *corev1.PersistentVolumeClaim
        ImmediateBind     bool
        OwnershipLabel    string
        Preallocation     bool
        PriorityClassName string
        Client            client.Client
        Log               logr.Logger
        Recorder          record.EventRecorder
}

var _ Phase = &HostClonePhase{}

var _ StatusReporter = &HostClonePhase{}

var httpClient *http.Client

func init() {
        httpClient = cc.BuildHTTPClient(httpClient)
}

// Name returns the name of the phase
func (p *HostClonePhase) Name() string {
        return HostClonePhaseName
}

// Status returns the phase status
func (p *HostClonePhase) Status(ctx context.Context) (*PhaseStatus, error) {
        result := &PhaseStatus{}
        pvc := &corev1.PersistentVolumeClaim{}
        exists, err := getResource(ctx, p.Client, p.Namespace, p.DesiredClaim.Name, pvc)
        if err != nil {
                return nil, err
        }

        if !exists {
                return result, nil
        }

        result.Annotations = pvc.Annotations

        podName := pvc.Annotations[cc.AnnCloneSourcePod]
        if podName == "" {
                return result, nil
        }

        args := &progressFromClaimArgs{
                Client:       p.Client,
                HTTPClient:   httpClient,
                Claim:        pvc,
                PodNamespace: p.Namespace,
                PodName:      podName,
                OwnerUID:     string(p.Owner.GetUID()),
        }

        progress, err := progressFromClaim(ctx, args)
        if err != nil {
                return nil, err
        }

        result.Progress = progress

        return result, nil
}

// progressFromClaimArgs are the args for ProgressFromClaim
type progressFromClaimArgs struct {
        Client       client.Client
        HTTPClient   *http.Client
        Claim        *corev1.PersistentVolumeClaim
        OwnerUID     string
        PodNamespace string
        PodName      string
}

// progressFromClaim returns the progres
func progressFromClaim(ctx context.Context, args *progressFromClaimArgs) (string, error) {
        // Just set 100.0% if pod is succeeded
        if args.Claim.Annotations[cc.AnnPodPhase] == string(corev1.PodSucceeded) {
                return cc.ProgressDone, nil
        }

        pod := &corev1.Pod{
                ObjectMeta: metav1.ObjectMeta{
                        Namespace: args.PodNamespace,
                        Name:      args.PodName,
                },
        }
        if err := args.Client.Get(ctx, client.ObjectKeyFromObject(pod), pod); err != nil {
                if k8serrors.IsNotFound(err) {
                        return "", nil
                }
                return "", err
        }

        // This will only work when the clone source pod is running
        if pod.Status.Phase != corev1.PodRunning {
                return "", nil
        }
        url, err := cc.GetMetricsURL(pod)
        if err != nil {
                return "", err
        }
        if url == "" {
                return "", nil
        }

        // We fetch the clone progress from the clone source pod metrics
        progressReport, err := cc.GetProgressReportFromURL(ctx, url, args.HTTPClient, metrics.CloneProgressMetricName, args.OwnerUID)
        if err != nil {
                return "", err
        }
        if progressReport != "" {
                if f, err := strconv.ParseFloat(progressReport, 64); err == nil {
                        return fmt.Sprintf("%.2f%%", f), nil
                }
        }

        return "", nil
}

// Reconcile creates the desired pvc and waits for the operation to complete
func (p *HostClonePhase) Reconcile(ctx context.Context) (*reconcile.Result, error) {
        actualClaim := &corev1.PersistentVolumeClaim{}
        exists, err := getResource(ctx, p.Client, p.Namespace, p.DesiredClaim.Name, actualClaim)
        if err != nil {
                return nil, err
        }

        if !exists {
                actualClaim, err = p.createClaim(ctx)
                if err != nil {
                        return nil, err
                }
        }

        targetPvc, err := cc.GetAnnotatedEventSource(ctx, p.Client, actualClaim)
        if err != nil {
                return nil, err
        }
        cc.CopyEvents(actualClaim, targetPvc, p.Client, p.Recorder)

        if !p.hostCloneComplete(actualClaim) {
                // requeue to update status
                return &reconcile.Result{RequeueAfter: 3 * time.Second}, nil
        }

        return nil, nil
}

func (p *HostClonePhase) createClaim(ctx context.Context) (*corev1.PersistentVolumeClaim, error) {
        claim := p.DesiredClaim.DeepCopy()

        claim.Namespace = p.Namespace
        cc.AddAnnotation(claim, cc.AnnPreallocationRequested, fmt.Sprintf("%t", p.Preallocation))
        cc.AddAnnotation(claim, cc.AnnOwnerUID, string(p.Owner.GetUID()))
        cc.AddAnnotation(claim, cc.AnnPodRestarts, "0")
        cc.AddAnnotation(claim, cc.AnnCloneRequest, fmt.Sprintf("%s/%s", p.Namespace, p.SourceName))
        cc.AddAnnotation(claim, cc.AnnPopulatorKind, cdiv1.VolumeCloneSourceRef)
        cc.AddAnnotation(claim, cc.AnnEventSourceKind, p.Owner.GetObjectKind().GroupVersionKind().Kind)
        cc.AddAnnotation(claim, cc.AnnEventSource, fmt.Sprintf("%s/%s", p.Owner.GetNamespace(), p.Owner.GetName()))
        if p.OwnershipLabel != "" {
                AddOwnershipLabel(p.OwnershipLabel, claim, p.Owner)
        }
        if p.ImmediateBind {
                cc.AddAnnotation(claim, cc.AnnImmediateBinding, "")
        }
        if p.PriorityClassName != "" {
                cc.AddAnnotation(claim, cc.AnnPriorityClassName, p.PriorityClassName)
        }
        cc.AddLabel(claim, cc.LabelExcludeFromVeleroBackup, "true")

        if err := p.MakeSureTargetPVCHasSufficientSpace(ctx, claim); err != nil {
                return nil, err
        }

        if err := p.Client.Create(ctx, claim); err != nil {
                checkQuotaExceeded(p.Recorder, p.Owner, err)
                return nil, err
        }

        return claim, nil
}

// The purpose of this check is to make sure the target PVC to have sufficient space
// before the cloning actually happens.
func (p *HostClonePhase) MakeSureTargetPVCHasSufficientSpace(ctx context.Context, targetPvc *corev1.PersistentVolumeClaim) error {
        sourcePvc := &corev1.PersistentVolumeClaim{}
        sourcePvcKey := client.ObjectKey{Namespace: p.Namespace, Name: p.SourceName}

        if err := p.Client.Get(ctx, sourcePvcKey, sourcePvc); err != nil {
                return err
        }

        if targetPvc.Spec.Resources.Requests == nil {
                return fmt.Errorf("no target resource request specified")
        }
        targetSize, ok := targetPvc.Spec.Resources.Requests[corev1.ResourceStorage]

        if !ok {
                return fmt.Errorf("no target size specified")
        }

        unInflatedSourceSize := sourcePvc.Spec.Resources.Requests[corev1.ResourceStorage]
        sourceVolumeMode := cc.GetVolumeMode(sourcePvc)
        targetVolumeMode := cc.GetVolumeMode(targetPvc)
        var err error

        // For filesystem source PVCs, we need to get the original DV size to account for
        if sourceVolumeMode == corev1.PersistentVolumeFilesystem {
                original, err := cc.GetHostCloneOriginalSourceDVSize(ctx, p.Client, sourcePvc)
                if err != nil {
                        return err
                }
                if !original.IsZero() {
                        unInflatedSourceSize = original
                }
        }

        targetSizeUpdated := false
        if unInflatedSourceSize.Cmp(targetSize) >= 0 {
                targetSizeUpdated = true
                targetSize = unInflatedSourceSize
        }

        // if target is filesystem, inflate the original size if target
        if targetVolumeMode == corev1.PersistentVolumeFilesystem && targetSizeUpdated {
                // when only sourc pvc size is available (snapshot case) we have no
                // way to trace back to its original DV size (because the DV probably
                // doesn't exist anymore), so we use source pvc size to just inflate
                // the target size with overhead
                targetSize, err = cc.InflateSizeWithOverhead(ctx, p.Client, unInflatedSourceSize.Value(), &targetPvc.Spec)
                if err != nil {
                        return err
                }
        }

        targetPvc.Spec.Resources.Requests[corev1.ResourceStorage] = targetSize
        return nil
}

func (p *HostClonePhase) hostCloneComplete(pvc *corev1.PersistentVolumeClaim) bool {
        // this is awfully lame
        // both the upload controller and clone controller update the PVC status to succeeded
        // but only the clone controller will set the preallocation annotation
        // so we have to wait for that
        if p.Preallocation && pvc.Annotations[cc.AnnPreallocationApplied] != "true" {
                return false
        }
        return pvc.Annotations[cc.AnnPodPhase] == string(cdiv1.Succeeded)
}

1	package clone
2
3	import (
4	"context"
5	"fmt"
6	"net/http"
7	"strconv"
8	"time"
9
10	"github.com/go-logr/logr"
11
12	corev1 "k8s.io/api/core/v1"
13	k8serrors "k8s.io/apimachinery/pkg/api/errors"
14	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15	"k8s.io/client-go/tools/record"
16
17	"sigs.k8s.io/controller-runtime/pkg/client"
18	"sigs.k8s.io/controller-runtime/pkg/reconcile"
19
20	cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
21	cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
22	metrics "kubevirt.io/containerized-data-importer/pkg/monitoring/metrics/cdi-cloner"
23	)
24
25	// HostClonePhaseName is the name of the host clone phase
26	const HostClonePhaseName = "HostClone"
27
28	// HostClonePhase creates and monitors a dumb clone operation
29	type HostClonePhase struct {
30	Owner client.Object
31	Namespace string
32	SourceName string
33	DesiredClaim *corev1.PersistentVolumeClaim
34	ImmediateBind bool
35	OwnershipLabel string
36	Preallocation bool
37	PriorityClassName string
38	Client client.Client
39	Log logr.Logger
40	Recorder record.EventRecorder
41	}
42
43	var _ Phase = &HostClonePhase{}
44
45	var _ StatusReporter = &HostClonePhase{}
46
47	var httpClient *http.Client
48
49	func init() {	1✔
50	httpClient = cc.BuildHTTPClient(httpClient)	1✔
51	}	1✔
52
53	// Name returns the name of the phase
54	func (p *HostClonePhase) Name() string {	×
55	return HostClonePhaseName	×
56	}	×
57
58	// Status returns the phase status
59	func (p HostClonePhase) Status(ctx context.Context) (PhaseStatus, error) {	×
60	result := &PhaseStatus{}	×
61	pvc := &corev1.PersistentVolumeClaim{}	×
62	exists, err := getResource(ctx, p.Client, p.Namespace, p.DesiredClaim.Name, pvc)	×
63	if err != nil {	×
64	return nil, err	×
65	}	×
66
67	if !exists {	×
68	return result, nil	×
69	}	×
70
71	result.Annotations = pvc.Annotations	×
72		×
73	podName := pvc.Annotations[cc.AnnCloneSourcePod]	×
74	if podName == "" {	×
75	return result, nil	×
76	}	×
77
78	args := &progressFromClaimArgs{	×
79	Client: p.Client,	×
80	HTTPClient: httpClient,	×
81	Claim: pvc,	×
82	PodNamespace: p.Namespace,	×
83	PodName: podName,	×
84	OwnerUID: string(p.Owner.GetUID()),	×
85	}	×
86		×
87	progress, err := progressFromClaim(ctx, args)	×
88	if err != nil {	×
89	return nil, err	×
90	}	×
91
92	result.Progress = progress	×
93		×
94	return result, nil	×
95	}
96
97	// progressFromClaimArgs are the args for ProgressFromClaim
98	type progressFromClaimArgs struct {
99	Client client.Client
100	HTTPClient *http.Client
101	Claim *corev1.PersistentVolumeClaim
102	OwnerUID string
103	PodNamespace string
104	PodName string
105	}
106
107	// progressFromClaim returns the progres
108	func progressFromClaim(ctx context.Context, args *progressFromClaimArgs) (string, error) {	×
109	// Just set 100.0% if pod is succeeded	×
110	if args.Claim.Annotations[cc.AnnPodPhase] == string(corev1.PodSucceeded) {	×
111	return cc.ProgressDone, nil	×
112	}	×
113
114	pod := &corev1.Pod{	×
115	ObjectMeta: metav1.ObjectMeta{	×
116	Namespace: args.PodNamespace,	×
117	Name: args.PodName,	×
118	},	×
119	}	×
120	if err := args.Client.Get(ctx, client.ObjectKeyFromObject(pod), pod); err != nil {	×
121	if k8serrors.IsNotFound(err) {	×
122	return "", nil	×
123	}	×
124	return "", err	×
125	}
126
127	// This will only work when the clone source pod is running
128	if pod.Status.Phase != corev1.PodRunning {	×
129	return "", nil	×
130	}	×
131	url, err := cc.GetMetricsURL(pod)	×
132	if err != nil {	×
133	return "", err	×
134	}	×
135	if url == "" {	×
136	return "", nil	×
137	}	×
138
139	// We fetch the clone progress from the clone source pod metrics
140	progressReport, err := cc.GetProgressReportFromURL(ctx, url, args.HTTPClient, metrics.CloneProgressMetricName, args.OwnerUID)	×
141	if err != nil {	×
142	return "", err	×
143	}	×
144	if progressReport != "" {	×
145	if f, err := strconv.ParseFloat(progressReport, 64); err == nil {	×
146	return fmt.Sprintf("%.2f%%", f), nil	×
147	}	×
148	}
149
150	return "", nil	×
151	}
152
153	// Reconcile creates the desired pvc and waits for the operation to complete
154	func (p HostClonePhase) Reconcile(ctx context.Context) (reconcile.Result, error) {	1✔
155	actualClaim := &corev1.PersistentVolumeClaim{}	1✔
156	exists, err := getResource(ctx, p.Client, p.Namespace, p.DesiredClaim.Name, actualClaim)	1✔
157	if err != nil {	1✔
158	return nil, err	×
159	}	×
160
161	if !exists {	2✔
162	actualClaim, err = p.createClaim(ctx)	1✔
163	if err != nil {	2✔
164	return nil, err	1✔
165	}	1✔
166	}
167
168	targetPvc, err := cc.GetAnnotatedEventSource(ctx, p.Client, actualClaim)	1✔
169	if err != nil {	1✔
170	return nil, err	×
171	}	×
172	cc.CopyEvents(actualClaim, targetPvc, p.Client, p.Recorder)	1✔
173		1✔
174	if !p.hostCloneComplete(actualClaim) {	2✔
175	// requeue to update status	1✔
176	return &reconcile.Result{RequeueAfter: 3 * time.Second}, nil	1✔
177	}	1✔
178
179	return nil, nil	1✔
180	}
181
182	func (p HostClonePhase) createClaim(ctx context.Context) (corev1.PersistentVolumeClaim, error) {	1✔
183	claim := p.DesiredClaim.DeepCopy()	1✔
184		1✔
185	claim.Namespace = p.Namespace	1✔
186	cc.AddAnnotation(claim, cc.AnnPreallocationRequested, fmt.Sprintf("%t", p.Preallocation))	1✔
187	cc.AddAnnotation(claim, cc.AnnOwnerUID, string(p.Owner.GetUID()))	1✔
188	cc.AddAnnotation(claim, cc.AnnPodRestarts, "0")	1✔
189	cc.AddAnnotation(claim, cc.AnnCloneRequest, fmt.Sprintf("%s/%s", p.Namespace, p.SourceName))	1✔
190	cc.AddAnnotation(claim, cc.AnnPopulatorKind, cdiv1.VolumeCloneSourceRef)	1✔
191	cc.AddAnnotation(claim, cc.AnnEventSourceKind, p.Owner.GetObjectKind().GroupVersionKind().Kind)	1✔
192	cc.AddAnnotation(claim, cc.AnnEventSource, fmt.Sprintf("%s/%s", p.Owner.GetNamespace(), p.Owner.GetName()))	1✔
193	if p.OwnershipLabel != "" {	2✔
194	AddOwnershipLabel(p.OwnershipLabel, claim, p.Owner)	1✔
195	}	1✔
196	if p.ImmediateBind {	2✔
197	cc.AddAnnotation(claim, cc.AnnImmediateBinding, "")	1✔
198	}	1✔
199	if p.PriorityClassName != "" {	2✔
200	cc.AddAnnotation(claim, cc.AnnPriorityClassName, p.PriorityClassName)	1✔
201	}	1✔
202	cc.AddLabel(claim, cc.LabelExcludeFromVeleroBackup, "true")	1✔
203		1✔
204	if err := p.MakeSureTargetPVCHasSufficientSpace(ctx, claim); err != nil {	2✔
205	return nil, err	1✔
206	}	1✔
207
208	if err := p.Client.Create(ctx, claim); err != nil {	1✔
209	checkQuotaExceeded(p.Recorder, p.Owner, err)	×
210	return nil, err	×
211	}	×
212
213	return claim, nil	1✔
214	}
215
216	// The purpose of this check is to make sure the target PVC to have sufficient space
217	// before the cloning actually happens.
218	func (p HostClonePhase) MakeSureTargetPVCHasSufficientSpace(ctx context.Context, targetPvc corev1.PersistentVolumeClaim) error {	1✔
219	sourcePvc := &corev1.PersistentVolumeClaim{}	1✔
220	sourcePvcKey := client.ObjectKey{Namespace: p.Namespace, Name: p.SourceName}	1✔
221		1✔
222	if err := p.Client.Get(ctx, sourcePvcKey, sourcePvc); err != nil {	1✔
NEW 223	return err	×
NEW 224	}	×
225
226	if targetPvc.Spec.Resources.Requests == nil {	2✔
227	return fmt.Errorf("no target resource request specified")	1✔
228	}	1✔
229	targetSize, ok := targetPvc.Spec.Resources.Requests[corev1.ResourceStorage]	1✔
230		1✔
231	if !ok {	1✔
NEW 232	return fmt.Errorf("no target size specified")	×
NEW 233	}	×
234
235	unInflatedSourceSize := sourcePvc.Spec.Resources.Requests[corev1.ResourceStorage]	1✔
236	sourceVolumeMode := cc.GetVolumeMode(sourcePvc)	1✔
237	targetVolumeMode := cc.GetVolumeMode(targetPvc)	1✔
238	var err error	1✔
239		1✔
240	// For filesystem source PVCs, we need to get the original DV size to account for	1✔
241	if sourceVolumeMode == corev1.PersistentVolumeFilesystem {	2✔
242	original, err := cc.GetHostCloneOriginalSourceDVSize(ctx, p.Client, sourcePvc)	1✔
243	if err != nil {	1✔
NEW 244	return err	×
NEW 245	}	×
246	if !original.IsZero() {	1✔
NEW 247	unInflatedSourceSize = original	×
NEW 248	}	×
249	}
250
251	targetSizeUpdated := false	1✔
252	if unInflatedSourceSize.Cmp(targetSize) >= 0 {	2✔
253	targetSizeUpdated = true	1✔
254	targetSize = unInflatedSourceSize	1✔
255	}	1✔
256
257	// if target is filesystem, inflate the original size if target
258	if targetVolumeMode == corev1.PersistentVolumeFilesystem && targetSizeUpdated {	2✔
259	// when only sourc pvc size is available (snapshot case) we have no	1✔
260	// way to trace back to its original DV size (because the DV probably	1✔
261	// doesn't exist anymore), so we use source pvc size to just inflate	1✔
262	// the target size with overhead	1✔
263	targetSize, err = cc.InflateSizeWithOverhead(ctx, p.Client, unInflatedSourceSize.Value(), &targetPvc.Spec)	1✔
264	if err != nil {	1✔
NEW 265	return err	×
NEW 266	}	×
267	}
268
269	targetPvc.Spec.Resources.Requests[corev1.ResourceStorage] = targetSize	1✔
270	return nil	1✔
271	}
272
273	func (p HostClonePhase) hostCloneComplete(pvc corev1.PersistentVolumeClaim) bool {	1✔
274	// this is awfully lame	1✔
275	// both the upload controller and clone controller update the PVC status to succeeded	1✔
276	// but only the clone controller will set the preallocation annotation	1✔
277	// so we have to wait for that	1✔
278	if p.Preallocation && pvc.Annotations[cc.AnnPreallocationApplied] != "true" {	2✔
279	return false	1✔
280	}	1✔
281	return pvc.Annotations[cc.AnnPodPhase] == string(cdiv1.Succeeded)	1✔
282	}

kubevirt / containerized-data-importer / #5755

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous