• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

kubernetes-sigs / blob-csi-driver / 14315335951

07 Apr 2025 05:14PM UTC coverage: 78.258%. Remained the same
14315335951

Pull #1931

github

web-flow
chore(deps): bump golang.org/x/sync from 0.12.0 to 0.13.0

Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.12.0 to 0.13.0.
- [Commits](https://github.com/golang/sync/compare/v0.12.0...v0.13.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sync
  dependency-version: 0.13.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Pull Request #1931: chore(deps): bump golang.org/x/sync from 0.12.0 to 0.13.0

2336 of 2985 relevant lines covered (78.26%)

7.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.75
/pkg/blob/blob.go
1
/*
2
Copyright 2019 The Kubernetes Authors.
3

4
Licensed under the Apache License, Version 2.0 (the "License");
5
you may not use this file except in compliance with the License.
6
You may obtain a copy of the License at
7

8
    http://www.apache.org/licenses/LICENSE-2.0
9

10
Unless required by applicable law or agreed to in writing, software
11
distributed under the License is distributed on an "AS IS" BASIS,
12
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
See the License for the specific language governing permissions and
14
limitations under the License.
15
*/
16

17
package blob
18

19
import (
20
        "context"
21
        "encoding/json"
22
        "errors"
23
        "flag"
24
        "fmt"
25
        "os"
26
        "strconv"
27
        "strings"
28
        "sync"
29
        "time"
30

31
        "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
32
        azstorage "github.com/Azure/azure-sdk-for-go/storage"
33
        "github.com/container-storage-interface/spec/lib/go/csi"
34
        grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
35
        "github.com/pborman/uuid"
36
        "google.golang.org/grpc"
37
        v1 "k8s.io/api/core/v1"
38
        apierror "k8s.io/apimachinery/pkg/api/errors"
39
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
40
        "k8s.io/client-go/kubernetes"
41
        "k8s.io/klog/v2"
42
        mount "k8s.io/mount-utils"
43
        utilexec "k8s.io/utils/exec"
44

45
        csicommon "sigs.k8s.io/blob-csi-driver/pkg/csi-common"
46
        "sigs.k8s.io/blob-csi-driver/pkg/util"
47
        "sigs.k8s.io/cloud-provider-azure/pkg/azclient"
48
        azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
49
        "sigs.k8s.io/cloud-provider-azure/pkg/provider/storage"
50
)
51

52
const (
53
        // DefaultDriverName holds the name of the csi-driver
54
        DefaultDriverName              = "blob.csi.azure.com"
55
        blobCSIDriverName              = "blob_csi_driver"
56
        separator                      = "#"
57
        volumeIDTemplate               = "%s#%s#%s#%s#%s#%s"
58
        secretNameTemplate             = "azure-storage-account-%s-secret"
59
        serverNameField                = "server"
60
        storageEndpointSuffixField     = "storageendpointsuffix"
61
        tagsField                      = "tags"
62
        matchTagsField                 = "matchtags"
63
        protocolField                  = "protocol"
64
        accountNameField               = "accountname"
65
        accountKeyField                = "accountkey"
66
        storageAccountField            = "storageaccount"
67
        storageAccountTypeField        = "storageaccounttype"
68
        skuNameField                   = "skuname"
69
        subscriptionIDField            = "subscriptionid"
70
        resourceGroupField             = "resourcegroup"
71
        locationField                  = "location"
72
        secretNameField                = "secretname"
73
        secretNamespaceField           = "secretnamespace"
74
        containerNameField             = "containername"
75
        containerNamePrefixField       = "containernameprefix"
76
        storeAccountKeyField           = "storeaccountkey"
77
        getLatestAccountKeyField       = "getlatestaccountkey"
78
        isHnsEnabledField              = "ishnsenabled"
79
        softDeleteBlobsField           = "softdeleteblobs"
80
        softDeleteContainersField      = "softdeletecontainers"
81
        enableBlobVersioningField      = "enableblobversioning"
82
        getAccountKeyFromSecretField   = "getaccountkeyfromsecret"
83
        storageSPNClientIDField        = "azurestoragespnclientid"
84
        storageSPNTenantIDField        = "azurestoragespntenantid"
85
        storageAuthTypeField           = "azurestorageauthtype"
86
        storageIdentityClientIDField   = "azurestorageidentityclientid"
87
        storageIdentityObjectIDField   = "azurestorageidentityobjectid"
88
        storageIdentityResourceIDField = "azurestorageidentityresourceid"
89
        msiEndpointField               = "msiendpoint"
90
        storageAADEndpointField        = "azurestorageaadendpoint"
91
        keyVaultURLField               = "keyvaulturl"
92
        keyVaultSecretNameField        = "keyvaultsecretname"
93
        keyVaultSecretVersionField     = "keyvaultsecretversion"
94
        storageAccountNameField        = "storageaccountname"
95
        allowBlobPublicAccessField     = "allowblobpublicaccess"
96
        allowSharedKeyAccessField      = "allowsharedkeyaccess"
97
        requireInfraEncryptionField    = "requireinfraencryption"
98
        ephemeralField                 = "csi.storage.k8s.io/ephemeral"
99
        podNamespaceField              = "csi.storage.k8s.io/pod.namespace"
100
        serviceAccountTokenField       = "csi.storage.k8s.io/serviceAccount.tokens"
101
        clientIDField                  = "clientID"
102
        tenantIDField                  = "tenantID"
103
        mountOptionsField              = "mountoptions"
104
        falseValue                     = "false"
105
        trueValue                      = "true"
106
        defaultSecretAccountName       = "azurestorageaccountname"
107
        defaultSecretAccountKey        = "azurestorageaccountkey"
108
        accountSasTokenField           = "azurestorageaccountsastoken"
109
        msiSecretField                 = "msisecret"
110
        storageSPNClientSecretField    = "azurestoragespnclientsecret"
111
        Fuse                           = "fuse"
112
        Fuse2                          = "fuse2"
113
        NFS                            = "nfs"
114
        AZNFS                          = "aznfs"
115
        NFSv3                          = "nfsv3"
116
        vnetResourceGroupField         = "vnetresourcegroup"
117
        vnetNameField                  = "vnetname"
118
        subnetNameField                = "subnetname"
119
        accessTierField                = "accesstier"
120
        networkEndpointTypeField       = "networkendpointtype"
121
        mountPermissionsField          = "mountpermissions"
122
        fsGroupChangePolicyField       = "fsgroupchangepolicy"
123
        useDataPlaneAPIField           = "usedataplaneapi"
124

125
        // See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names
126
        containerNameMinLength = 3
127
        containerNameMaxLength = 63
128

129
        accountNotProvisioned                   = "StorageAccountIsNotProvisioned"
130
        tooManyRequests                         = "TooManyRequests"
131
        clientThrottled                         = "client throttled"
132
        containerBeingDeletedDataplaneAPIError  = "ContainerBeingDeleted"
133
        containerBeingDeletedManagementAPIError = "container is being deleted"
134
        statusCodeNotFound                      = "StatusCode=404"
135
        httpCodeNotFound                        = "HTTPStatusCode: 404"
136

137
        // containerMaxSize is the max size of the blob container. See https://docs.microsoft.com/en-us/azure/storage/blobs/scalability-targets#scale-targets-for-blob-storage
138
        containerMaxSize = 100 * util.TiB
139

140
        subnetTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s"
141

142
        defaultNamespace = "default"
143

144
        pvcNameKey           = "csi.storage.k8s.io/pvc/name"
145
        pvcNamespaceKey      = "csi.storage.k8s.io/pvc/namespace"
146
        pvNameKey            = "csi.storage.k8s.io/pv/name"
147
        pvcNameMetadata      = "${pvc.metadata.name}"
148
        pvcNamespaceMetadata = "${pvc.metadata.namespace}"
149
        pvNameMetadata       = "${pv.metadata.name}"
150

151
        VolumeID = "volumeid"
152

153
        defaultStorageEndPointSuffix = "core.windows.net"
154

155
        FSGroupChangeNone = "None"
156
        // define tag value delimiter and default is comma
157
        tagValueDelimiterField = "tagvaluedelimiter"
158

159
        DefaultTokenAudience = "api://AzureADTokenExchange" //nolint:gosec // G101 ignore this!
160

161
)
162

163
var (
164
        supportedProtocolList            = []string{Fuse, Fuse2, NFS, AZNFS}
165
        retriableErrors                  = []string{accountNotProvisioned, tooManyRequests, statusCodeNotFound, containerBeingDeletedDataplaneAPIError, containerBeingDeletedManagementAPIError, clientThrottled}
166
        supportedFSGroupChangePolicyList = []string{FSGroupChangeNone, string(v1.FSGroupChangeAlways), string(v1.FSGroupChangeOnRootMismatch)}
167

168
        // azcopyCloneVolumeOptions used in volume cloning between different storage account and --check-length to false because volume data may be in changing state, copy volume is not same as current source volume,
169
        // set --s2s-preserve-access-tier=false to avoid BlobAccessTierNotSupportedForAccountType error in azcopy
170
        azcopyCloneVolumeOptions = []string{"--recursive", "--check-length=false", "--s2s-preserve-access-tier=false", "--log-level=ERROR"}
171
)
172

173
// DriverOptions defines driver parameters specified in driver deployment
174
type DriverOptions struct {
175
        NodeID                                 string
176
        DriverName                             string
177
        BlobfuseProxyEndpoint                  string
178
        EnableBlobfuseProxy                    bool
179
        BlobfuseProxyConnTimout                int
180
        EnableBlobMockMount                    bool
181
        AllowInlineVolumeKeyAccessWithIdentity bool
182
        EnableGetVolumeStats                   bool
183
        AppendTimeStampInCacheDir              bool
184
        AppendMountErrorHelpLink               bool
185
        MountPermissions                       uint64
186
        EnableAznfsMount                       bool
187
        VolStatsCacheExpireInMinutes           int
188
        SasTokenExpirationMinutes              int
189
        WaitForAzCopyTimeoutMinutes            int
190
        EnableVolumeMountGroup                 bool
191
        FSGroupChangePolicy                    string
192
}
193

194
func (option *DriverOptions) AddFlags() {
1✔
195
        flag.StringVar(&option.BlobfuseProxyEndpoint, "blobfuse-proxy-endpoint", "unix://tmp/blobfuse-proxy.sock", "blobfuse-proxy endpoint")
1✔
196
        flag.StringVar(&option.NodeID, "nodeid", "", "node id")
1✔
197
        flag.StringVar(&option.DriverName, "drivername", DefaultDriverName, "name of the driver")
1✔
198
        flag.BoolVar(&option.EnableBlobfuseProxy, "enable-blobfuse-proxy", false, "using blobfuse proxy for mounts")
1✔
199
        flag.IntVar(&option.BlobfuseProxyConnTimout, "blobfuse-proxy-connect-timeout", 5, "blobfuse proxy connection timeout(seconds)")
1✔
200
        flag.BoolVar(&option.EnableBlobMockMount, "enable-blob-mock-mount", false, "enable mock mount(only for testing)")
1✔
201
        flag.BoolVar(&option.EnableGetVolumeStats, "enable-get-volume-stats", false, "allow GET_VOLUME_STATS on agent node")
1✔
202
        flag.BoolVar(&option.AppendTimeStampInCacheDir, "append-timestamp-cache-dir", false, "append timestamp into cache directory on agent node")
1✔
203
        flag.Uint64Var(&option.MountPermissions, "mount-permissions", 0777, "mounted folder permissions")
1✔
204
        flag.BoolVar(&option.AllowInlineVolumeKeyAccessWithIdentity, "allow-inline-volume-key-access-with-idenitity", false, "allow accessing storage account key using cluster identity for inline volume")
1✔
205
        flag.BoolVar(&option.AppendMountErrorHelpLink, "append-mount-error-help-link", true, "Whether to include a link for help with mount errors when a mount error occurs.")
1✔
206
        flag.BoolVar(&option.EnableAznfsMount, "enable-aznfs-mount", false, "replace nfs mount with aznfs mount")
1✔
207
        flag.IntVar(&option.VolStatsCacheExpireInMinutes, "vol-stats-cache-expire-in-minutes", 10, "The cache expire time in minutes for volume stats cache")
1✔
208
        flag.IntVar(&option.SasTokenExpirationMinutes, "sas-token-expiration-minutes", 1440, "sas token expiration minutes during volume cloning")
1✔
209
        flag.IntVar(&option.WaitForAzCopyTimeoutMinutes, "wait-for-azcopy-timeout-minutes", 18, "timeout in minutes for waiting for azcopy to finish")
1✔
210
        flag.BoolVar(&option.EnableVolumeMountGroup, "enable-volume-mount-group", true, "indicates whether enabling VOLUME_MOUNT_GROUP")
1✔
211
        flag.StringVar(&option.FSGroupChangePolicy, "fsgroup-change-policy", "", "indicates how the volume's ownership will be changed by the driver, OnRootMismatch is the default value")
1✔
212
}
1✔
213

214
// Driver implements all interfaces of CSI drivers
215
type Driver struct {
216
        csicommon.CSIDriver
217
        // Embed UnimplementedXXXServer to ensure the driver returns Unimplemented for any
218
        // new RPC methods that might be introduced in future versions of the spec.
219
        csi.UnimplementedControllerServer
220
        csi.UnimplementedIdentityServer
221
        csi.UnimplementedNodeServer
222

223
        cloud                 *storage.AccountRepo
224
        clientFactory         azclient.ClientFactory
225
        networkClientFactory  azclient.ClientFactory
226
        KubeClient            kubernetes.Interface
227
        blobfuseProxyEndpoint string
228
        // enableBlobMockMount is only for testing, DO NOT set as true in non-testing scenario
229
        enableBlobMockMount                    bool
230
        enableBlobfuseProxy                    bool
231
        enableGetVolumeStats                   bool
232
        allowInlineVolumeKeyAccessWithIdentity bool
233
        appendTimeStampInCacheDir              bool
234
        appendMountErrorHelpLink               bool
235
        blobfuseProxyConnTimout                int
236
        mountPermissions                       uint64
237
        enableAznfsMount                       bool
238
        enableVolumeMountGroup                 bool
239
        fsGroupChangePolicy                    string
240
        mounter                                *mount.SafeFormatAndMount
241
        volLockMap                             *util.LockMap
242
        // A map storing all volumes with ongoing operations so that additional operations
243
        // for that same volume (as defined by VolumeID) return an Aborted error
244
        volumeLocks *volumeLocks
245
        // only for nfs feature
246
        subnetLockMap *util.LockMap
247
        // a map storing all volumes created by this driver <volumeName, accountName>
248
        volMap sync.Map
249
        // a timed cache storing all volumeIDs and storage accounts that are using data plane API
250
        dataPlaneAPIVolCache azcache.Resource
251
        // a timed cache storing account search history (solve account list throttling issue)
252
        accountSearchCache azcache.Resource
253
        // a timed cache storing volume stats <volumeID, volumeStats>
254
        volStatsCache azcache.Resource
255
        // a timed cache storing account which should use sastoken for azcopy based volume cloning
256
        azcopySasTokenCache azcache.Resource
257
        // a timed cache storing subnet operations
258
        subnetCache azcache.Resource
259
        // sas expiry time for azcopy in volume clone
260
        sasTokenExpirationMinutes int
261
        // timeout in minutes for waiting for azcopy to finish
262
        waitForAzCopyTimeoutMinutes int
263
        // azcopy for provide exec mock for ut
264
        azcopy *util.Azcopy
265
}
266

267
// NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version &
268
// does not support optional driver plugin info manifest field. Refer to CSI spec for more details.
269
func NewDriver(options *DriverOptions, kubeClient kubernetes.Interface, cloud *storage.AccountRepo) *Driver {
137✔
270
        d := Driver{
137✔
271
                volLockMap:                             util.NewLockMap(),
137✔
272
                subnetLockMap:                          util.NewLockMap(),
137✔
273
                volumeLocks:                            newVolumeLocks(),
137✔
274
                blobfuseProxyEndpoint:                  options.BlobfuseProxyEndpoint,
137✔
275
                enableBlobfuseProxy:                    options.EnableBlobfuseProxy,
137✔
276
                allowInlineVolumeKeyAccessWithIdentity: options.AllowInlineVolumeKeyAccessWithIdentity,
137✔
277
                blobfuseProxyConnTimout:                options.BlobfuseProxyConnTimout,
137✔
278
                enableBlobMockMount:                    options.EnableBlobMockMount,
137✔
279
                enableGetVolumeStats:                   options.EnableGetVolumeStats,
137✔
280
                enableVolumeMountGroup:                 options.EnableVolumeMountGroup,
137✔
281
                appendMountErrorHelpLink:               options.AppendMountErrorHelpLink,
137✔
282
                mountPermissions:                       options.MountPermissions,
137✔
283
                enableAznfsMount:                       options.EnableAznfsMount,
137✔
284
                sasTokenExpirationMinutes:              options.SasTokenExpirationMinutes,
137✔
285
                waitForAzCopyTimeoutMinutes:            options.WaitForAzCopyTimeoutMinutes,
137✔
286
                fsGroupChangePolicy:                    options.FSGroupChangePolicy,
137✔
287
                azcopy:                                 &util.Azcopy{ExecCmd: &util.ExecCommand{}},
137✔
288
                KubeClient:                             kubeClient,
137✔
289
                cloud:                                  cloud,
137✔
290
        }
137✔
291
        d.Name = options.DriverName
137✔
292
        d.Version = driverVersion
137✔
293
        d.NodeID = options.NodeID
137✔
294
        if d.cloud != nil {
274✔
295
                d.clientFactory = d.cloud.ComputeClientFactory
137✔
296
                d.networkClientFactory = d.cloud.NetworkClientFactory
137✔
297
                if d.networkClientFactory == nil {
274✔
298
                        d.networkClientFactory = d.cloud.ComputeClientFactory
137✔
299
                }
137✔
300
        }
301

302
        var err error
137✔
303
        getter := func(_ context.Context, _ string) (interface{}, error) { return nil, nil }
160✔
304
        if d.accountSearchCache, err = azcache.NewTimedCache(time.Minute, getter, false); err != nil {
137✔
305
                klog.Fatalf("%v", err)
×
306
        }
×
307
        if d.dataPlaneAPIVolCache, err = azcache.NewTimedCache(24*30*time.Hour, getter, false); err != nil {
137✔
308
                klog.Fatalf("%v", err)
×
309
        }
×
310
        if d.azcopySasTokenCache, err = azcache.NewTimedCache(15*time.Minute, getter, false); err != nil {
137✔
311
                klog.Fatalf("%v", err)
×
312
        }
×
313

314
        if options.VolStatsCacheExpireInMinutes <= 0 {
274✔
315
                options.VolStatsCacheExpireInMinutes = 10 // default expire in 10 minutes
137✔
316
        }
137✔
317
        if d.volStatsCache, err = azcache.NewTimedCache(time.Duration(options.VolStatsCacheExpireInMinutes)*time.Minute, getter, false); err != nil {
137✔
318
                klog.Fatalf("%v", err)
×
319
        }
×
320
        if d.subnetCache, err = azcache.NewTimedCache(10*time.Minute, getter, false); err != nil {
137✔
321
                klog.Fatalf("%v", err)
×
322
        }
×
323

324
        d.mounter = &mount.SafeFormatAndMount{
137✔
325
                Interface: mount.New(""),
137✔
326
                Exec:      utilexec.New(),
137✔
327
        }
137✔
328

137✔
329
        // Initialize default library driver
137✔
330
        d.AddControllerServiceCapabilities(
137✔
331
                []csi.ControllerServiceCapability_RPC_Type{
137✔
332
                        csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
137✔
333
                        //csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
137✔
334
                        //csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
137✔
335
                        csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
137✔
336
                        csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER,
137✔
337
                        csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
137✔
338
                })
137✔
339
        d.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
137✔
340
                csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
137✔
341
                csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY,
137✔
342
                csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER,
137✔
343
                csi.VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER,
137✔
344
                csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY,
137✔
345
                csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER,
137✔
346
                csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
137✔
347
        })
137✔
348

137✔
349
        nodeCap := []csi.NodeServiceCapability_RPC_Type{
137✔
350
                csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
137✔
351
                csi.NodeServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER,
137✔
352
        }
137✔
353
        if d.enableGetVolumeStats {
137✔
354
                nodeCap = append(nodeCap, csi.NodeServiceCapability_RPC_GET_VOLUME_STATS)
×
355
        }
×
356
        if d.enableVolumeMountGroup {
137✔
357
                nodeCap = append(nodeCap, csi.NodeServiceCapability_RPC_VOLUME_MOUNT_GROUP)
×
358
        }
×
359
        d.AddNodeServiceCapabilities(nodeCap)
137✔
360

137✔
361
        return &d
137✔
362
}
363

364
// Run driver initialization
365
func (d *Driver) Run(ctx context.Context, endpoint string) error {
2✔
366
        versionMeta, err := GetVersionYAML(d.Name)
2✔
367
        if err != nil {
2✔
368
                klog.Fatalf("%v", err)
×
369
        }
×
370
        klog.Infof("\nDRIVER INFORMATION:\n-------------------\n%s\n\nStreaming logs below:", versionMeta)
2✔
371
        opts := []grpc.ServerOption{
2✔
372
                grpc.ChainUnaryInterceptor(
2✔
373
                        grpcprom.NewServerMetrics().UnaryServerInterceptor(),
2✔
374
                        csicommon.LogGRPC,
2✔
375
                ),
2✔
376
        }
2✔
377
        s := grpc.NewServer(opts...)
2✔
378
        csi.RegisterIdentityServer(s, d)
2✔
379
        csi.RegisterControllerServer(s, d)
2✔
380
        csi.RegisterNodeServer(s, d)
2✔
381

2✔
382
        go func() {
4✔
383
                //graceful shutdown
2✔
384
                <-ctx.Done()
2✔
385
                s.GracefulStop()
2✔
386
        }()
2✔
387
        // Driver d act as IdentityServer, ControllerServer and NodeServer
388
        listener, err := csicommon.Listen(ctx, endpoint)
2✔
389
        if err != nil {
2✔
390
                klog.Fatalf("failed to listen to endpoint, error: %v", err)
×
391
        }
×
392
        err = s.Serve(listener)
2✔
393
        if errors.Is(err, grpc.ErrServerStopped) {
2✔
394
                klog.Infof("gRPC server stopped serving")
×
395
                return nil
×
396
        }
×
397
        return err
2✔
398
}
399

400
// GetContainerInfo get container info according to volume id
401
// the format of VolumeId is: rg#accountName#containerName#uuid#secretNamespace#subsID
402
//
403
// e.g.
404
// input: "rg#f5713de20cde511e8ba4900#containerName#uuid#"
405
// output: rg, f5713de20cde511e8ba4900, containerName, "" , ""
406
// input: "rg#f5713de20cde511e8ba4900#containerName#uuid#namespace#"
407
// output: rg, f5713de20cde511e8ba4900, containerName, namespace, ""
408
// input: "rg#f5713de20cde511e8ba4900#containerName#uuid#namespace#subsID"
409
// output: rg, f5713de20cde511e8ba4900, containerName, namespace, subsID
410
func GetContainerInfo(id string) (string, string, string, string, string, error) {
41✔
411
        segments := strings.Split(id, separator)
41✔
412
        if len(segments) < 3 {
51✔
413
                return "", "", "", "", "", fmt.Errorf("error parsing volume id: %q, should at least contain two #", id)
10✔
414
        }
10✔
415
        var secretNamespace, subsID string
31✔
416
        if len(segments) > 4 {
37✔
417
                secretNamespace = segments[4]
6✔
418
        }
6✔
419
        if len(segments) > 5 {
34✔
420
                subsID = segments[5]
3✔
421
        }
3✔
422
        return segments[0], segments[1], segments[2], secretNamespace, subsID, nil
31✔
423
}
424

425
// A container name must be a valid DNS name, conforming to the following naming rules:
426
//  1. Container names must start with a letter or number, and can contain only letters, numbers, and the dash (-) character.
427
//  2. Every dash (-) character must be immediately preceded and followed by a letter or number; consecutive dashes are not permitted in container names.
428
//  3. All letters in a container name must be lowercase.
429
//  4. Container names must be from 3 through 63 characters long.
430
//
431
// See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names
432
func getValidContainerName(volumeName, protocol string) string {
6✔
433
        containerName := strings.ToLower(volumeName)
6✔
434
        if len(containerName) > containerNameMaxLength {
7✔
435
                containerName = containerName[0:containerNameMaxLength]
1✔
436
        }
1✔
437
        if !checkContainerNameBeginAndEnd(containerName) || len(containerName) < containerNameMinLength {
6✔
438
                // now we set as 63 for maximum container name length
×
439
                // todo: get cluster name
×
440
                containerName = generateVolumeName(fmt.Sprintf("pvc-%s", protocol), uuid.NewUUID().String(), 63)
×
441
                klog.Warningf("requested volume name (%s) is invalid, regenerated as (%q)", volumeName, containerName)
×
442
        }
×
443
        return strings.Replace(containerName, "--", "-", -1)
6✔
444
}
445

446
func checkContainerNameBeginAndEnd(containerName string) bool {
12✔
447
        length := len(containerName)
12✔
448
        if (('a' <= containerName[0] && containerName[0] <= 'z') ||
12✔
449
                ('0' <= containerName[0] && containerName[0] <= '9')) &&
12✔
450
                (('a' <= containerName[length-1] && containerName[length-1] <= 'z') ||
12✔
451
                        ('0' <= containerName[length-1] && containerName[length-1] <= '9')) {
22✔
452
                return true
10✔
453
        }
10✔
454

455
        return false
2✔
456
}
457

458
// isSASToken checks if the key contains the patterns.
459
// SAS token format could refer to https://docs.microsoft.com/en-us/rest/api/eventhub/generate-sas-token
460
func isSASToken(key string) bool {
3✔
461
        return strings.HasPrefix(key, "?")
3✔
462
}
3✔
463

464
// GetAuthEnv return <accountName, containerName, authEnv, error>
465
func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attrib, secrets map[string]string) (string, string, string, string, []string, error) {
10✔
466
        rgName, accountName, containerName, secretNamespace, _, err := GetContainerInfo(volumeID)
10✔
467
        if err != nil {
12✔
468
                // ignore volumeID parsing error
2✔
469
                klog.V(2).Infof("parsing volumeID(%s) return with error: %v", volumeID, err)
2✔
470
                err = nil
2✔
471
        }
2✔
472

473
        var (
10✔
474
                subsID                  string
10✔
475
                accountKey              string
10✔
476
                accountSasToken         string
10✔
477
                msiSecret               string
10✔
478
                storageSPNClientSecret  string
10✔
479
                storageSPNClientID      string
10✔
480
                storageSPNTenantID      string
10✔
481
                secretName              string
10✔
482
                pvcNamespace            string
10✔
483
                keyVaultURL             string
10✔
484
                keyVaultSecretName      string
10✔
485
                keyVaultSecretVersion   string
10✔
486
                azureStorageAuthType    string
10✔
487
                authEnv                 []string
10✔
488
                getAccountKeyFromSecret bool
10✔
489
                getLatestAccountKey     bool
10✔
490
                clientID                string
10✔
491
                tenantID                string
10✔
492
                serviceAccountToken     string
10✔
493
        )
10✔
494

10✔
495
        for k, v := range attrib {
36✔
496
                switch strings.ToLower(k) {
26✔
497
                case subscriptionIDField:
1✔
498
                        subsID = v
1✔
499
                case resourceGroupField:
×
500
                        rgName = v
×
501
                case containerNameField:
3✔
502
                        containerName = v
3✔
503
                case keyVaultURLField:
1✔
504
                        keyVaultURL = v
1✔
505
                case keyVaultSecretNameField:
1✔
506
                        keyVaultSecretName = v
1✔
507
                case keyVaultSecretVersionField:
1✔
508
                        keyVaultSecretVersion = v
1✔
509
                case storageAccountField:
2✔
510
                        accountName = v
2✔
511
                case storageAccountNameField: // for compatibility
1✔
512
                        accountName = v
1✔
513
                case secretNameField:
1✔
514
                        secretName = v
1✔
515
                case secretNamespaceField:
1✔
516
                        secretNamespace = v
1✔
517
                case pvcNamespaceKey:
1✔
518
                        pvcNamespace = v
1✔
519
                case getAccountKeyFromSecretField:
1✔
520
                        getAccountKeyFromSecret = strings.EqualFold(v, trueValue)
1✔
521
                case storageAuthTypeField:
×
522
                        azureStorageAuthType = v
×
523
                        authEnv = append(authEnv, "AZURE_STORAGE_AUTH_TYPE="+v)
×
524
                case storageIdentityClientIDField:
1✔
525
                        authEnv = append(authEnv, "AZURE_STORAGE_IDENTITY_CLIENT_ID="+v)
1✔
526
                case storageIdentityObjectIDField:
1✔
527
                        authEnv = append(authEnv, "AZURE_STORAGE_IDENTITY_OBJECT_ID="+v)
1✔
528
                case storageIdentityResourceIDField:
1✔
529
                        authEnv = append(authEnv, "AZURE_STORAGE_IDENTITY_RESOURCE_ID="+v)
1✔
530
                case msiEndpointField:
1✔
531
                        authEnv = append(authEnv, "MSI_ENDPOINT="+v)
1✔
532
                case storageSPNClientIDField:
1✔
533
                        storageSPNClientID = v
1✔
534
                case storageSPNTenantIDField:
1✔
535
                        storageSPNTenantID = v
1✔
536
                case storageAADEndpointField:
1✔
537
                        authEnv = append(authEnv, "AZURE_STORAGE_AAD_ENDPOINT="+v)
1✔
538
                case getLatestAccountKeyField:
1✔
539
                        if getLatestAccountKey, err = strconv.ParseBool(v); err != nil {
2✔
540
                                return rgName, accountName, accountKey, containerName, authEnv, fmt.Errorf("invalid %s: %s in volume context", getLatestAccountKeyField, v)
1✔
541
                        }
1✔
542
                case strings.ToLower(clientIDField):
×
543
                        clientID = v
×
544
                case strings.ToLower(tenantIDField):
×
545
                        tenantID = v
×
546
                case strings.ToLower(serviceAccountTokenField):
×
547
                        serviceAccountToken = v
×
548
                }
549
        }
550
        klog.V(2).Infof("volumeID(%s) authEnv: %s", volumeID, authEnv)
9✔
551

9✔
552
        if protocol == NFS {
11✔
553
                // nfs protocol does not need account key, return directly
2✔
554
                return rgName, accountName, accountKey, containerName, authEnv, err
2✔
555
        }
2✔
556

557
        if secretNamespace == "" {
13✔
558
                if pvcNamespace == "" {
12✔
559
                        secretNamespace = defaultNamespace
6✔
560
                } else {
6✔
561
                        secretNamespace = pvcNamespace
×
562
                }
×
563
        }
564

565
        if rgName == "" {
8✔
566
                rgName = d.cloud.ResourceGroup
1✔
567
        }
1✔
568

569
        if tenantID == "" {
14✔
570
                tenantID = d.cloud.TenantID
7✔
571
        }
7✔
572

573
        // if client id is specified, we only use workload identity for blobfuse auth
574
        if clientID != "" {
7✔
575
                klog.V(2).Infof("clientID(%s) is specified, use workload identity for blobfuse auth", clientID)
×
576

×
577
                workloadIdentityToken, err := parseServiceAccountToken(serviceAccountToken)
×
578
                if err != nil {
×
579
                        return rgName, accountName, accountKey, containerName, authEnv, err
×
580
                }
×
581

582
                authEnv = append(authEnv, "AZURE_STORAGE_SPN_CLIENT_ID="+clientID)
×
583
                authEnv = append(authEnv, "AZURE_STORAGE_SPN_TENANT_ID="+tenantID)
×
584
                authEnv = append(authEnv, "WORKLOAD_IDENTITY_TOKEN="+workloadIdentityToken)
×
585

×
586
                return rgName, accountName, accountKey, containerName, authEnv, err
×
587
        }
588

589
        // 1. If keyVaultURL is not nil, preferentially use the key stored in key vault.
590
        // 2. Then if secrets map is not nil, use the key stored in the secrets map.
591
        // 3. Finally if both keyVaultURL and secrets map are nil, get the key from Azure.
592
        if keyVaultURL != "" {
8✔
593
                key, err := d.getKeyVaultSecretContent(ctx, keyVaultURL, keyVaultSecretName, keyVaultSecretVersion)
1✔
594
                if err != nil {
2✔
595
                        return rgName, accountName, accountKey, containerName, authEnv, err
1✔
596
                }
1✔
597
                if isSASToken(key) {
×
598
                        accountSasToken = key
×
599
                } else {
×
600
                        accountKey = key
×
601
                }
×
602
        } else {
6✔
603
                if len(secrets) == 0 {
11✔
604
                        if secretName == "" && accountName != "" {
9✔
605
                                secretName = fmt.Sprintf(secretNameTemplate, accountName)
4✔
606
                        }
4✔
607
                        if secretName != "" {
10✔
608
                                // read from k8s secret first
5✔
609
                                var name, spnClientID, spnTenantID string
5✔
610
                                name, accountKey, accountSasToken, msiSecret, storageSPNClientSecret, spnClientID, spnTenantID, err = d.GetInfoFromSecret(ctx, secretName, secretNamespace)
5✔
611
                                if name != "" {
5✔
612
                                        accountName = name
×
613
                                }
×
614
                                if spnClientID != "" {
5✔
615
                                        storageSPNClientID = spnClientID
×
616
                                }
×
617
                                if spnTenantID != "" {
5✔
618
                                        storageSPNTenantID = spnTenantID
×
619
                                }
×
620
                                if err != nil && strings.EqualFold(azureStorageAuthType, "msi") {
5✔
621
                                        klog.V(2).Infof("ignore error(%v) since secret is optional for auth type(%s)", err, azureStorageAuthType)
×
622
                                        err = nil
×
623
                                }
×
624
                                if err != nil && !getAccountKeyFromSecret && (azureStorageAuthType == "" || strings.EqualFold(azureStorageAuthType, "key")) {
10✔
625
                                        klog.V(2).Infof("get account(%s) key from secret(%s, %s) failed with error: %v, use cluster identity to get account key instead",
5✔
626
                                                accountName, secretNamespace, secretName, err)
5✔
627
                                        accountKey, err = d.GetStorageAccesskeyWithSubsID(ctx, subsID, accountName, rgName, getLatestAccountKey)
5✔
628
                                        if err != nil {
7✔
629
                                                return rgName, accountName, accountKey, containerName, authEnv, fmt.Errorf("no key for storage account(%s) under resource group(%s), err %w", accountName, rgName, err)
2✔
630
                                        }
2✔
631
                                }
632
                        }
633
                } else {
1✔
634
                        for k, v := range secrets {
8✔
635
                                v = strings.TrimSpace(v)
7✔
636
                                switch strings.ToLower(k) {
7✔
637
                                case accountNameField:
1✔
638
                                        accountName = v
1✔
639
                                case defaultSecretAccountName: // for compatibility with built-in blobfuse plugin
1✔
640
                                        accountName = v
1✔
641
                                case accountKeyField:
1✔
642
                                        accountKey = v
1✔
643
                                case defaultSecretAccountKey: // for compatibility with built-in blobfuse plugin
1✔
644
                                        accountKey = v
1✔
645
                                case accountSasTokenField:
1✔
646
                                        accountSasToken = v
1✔
647
                                case msiSecretField:
1✔
648
                                        msiSecret = v
1✔
649
                                case storageSPNClientSecretField:
1✔
650
                                        storageSPNClientSecret = v
1✔
651
                                case storageSPNClientIDField:
×
652
                                        storageSPNClientID = v
×
653
                                case storageSPNTenantIDField:
×
654
                                        storageSPNTenantID = v
×
655
                                }
656
                        }
657
                }
658
        }
659

660
        if containerName == "" {
4✔
661
                err = fmt.Errorf("could not find containerName from attributes(%v) or volumeID(%v)", attrib, volumeID)
×
662
        }
×
663

664
        if accountKey != "" {
8✔
665
                authEnv = append(authEnv, "AZURE_STORAGE_ACCESS_KEY="+accountKey)
4✔
666
        }
4✔
667

668
        if accountSasToken != "" {
5✔
669
                klog.V(2).Infof("accountSasToken is not empty, use it to access storage account(%s), container(%s)", accountName, containerName)
1✔
670
                authEnv = append(authEnv, "AZURE_STORAGE_SAS_TOKEN="+accountSasToken)
1✔
671
        }
1✔
672

673
        if msiSecret != "" {
5✔
674
                klog.V(2).Infof("msiSecret is not empty, use it to access storage account(%s), container(%s)", accountName, containerName)
1✔
675
                authEnv = append(authEnv, "MSI_SECRET="+msiSecret)
1✔
676
        }
1✔
677

678
        if storageSPNClientSecret != "" {
5✔
679
                klog.V(2).Infof("storageSPNClientSecret is not empty, use it to access storage account(%s), container(%s)", accountName, containerName)
1✔
680
                authEnv = append(authEnv, "AZURE_STORAGE_SPN_CLIENT_SECRET="+storageSPNClientSecret)
1✔
681
        }
1✔
682

683
        if storageSPNClientID != "" {
4✔
684
                klog.V(2).Infof("storageSPNClientID(%s) is not empty, use it to access storage account(%s), container(%s)", storageSPNClientID, accountName, containerName)
×
685
                authEnv = append(authEnv, "AZURE_STORAGE_SPN_CLIENT_ID="+storageSPNClientID)
×
686
        }
×
687

688
        if storageSPNTenantID != "" {
4✔
689
                klog.V(2).Infof("storageSPNTenantID(%s) is not empty, use it to access storage account(%s), container(%s)", storageSPNTenantID, accountName, containerName)
×
690
                authEnv = append(authEnv, "AZURE_STORAGE_SPN_TENANT_ID="+storageSPNTenantID)
×
691
        }
×
692

693
        return rgName, accountName, accountKey, containerName, authEnv, err
4✔
694
}
695

696
// GetStorageAccountAndContainer get storage account and container info
697
// returns <accountName, accountKey, accountSasToken, containerName>
698
// only for e2e testing
699
func (d *Driver) GetStorageAccountAndContainer(ctx context.Context, volumeID string, attrib, secrets map[string]string) (string, string, string, string, error) {
3✔
700
        var (
3✔
701
                subsID                string
3✔
702
                accountName           string
3✔
703
                accountKey            string
3✔
704
                accountSasToken       string
3✔
705
                containerName         string
3✔
706
                keyVaultURL           string
3✔
707
                keyVaultSecretName    string
3✔
708
                keyVaultSecretVersion string
3✔
709
                getLatestAccountKey   bool
3✔
710
                err                   error
3✔
711
        )
3✔
712

3✔
713
        for k, v := range attrib {
8✔
714
                switch strings.ToLower(k) {
5✔
715
                case subscriptionIDField:
×
716
                        subsID = v
×
717
                case containerNameField:
1✔
718
                        containerName = v
1✔
719
                case keyVaultURLField:
×
720
                        keyVaultURL = v
×
721
                case keyVaultSecretNameField:
1✔
722
                        keyVaultSecretName = v
1✔
723
                case keyVaultSecretVersionField:
1✔
724
                        keyVaultSecretVersion = v
1✔
725
                case storageAccountField:
×
726
                        accountName = v
×
727
                case storageAccountNameField: // for compatibility
1✔
728
                        accountName = v
1✔
729
                case getLatestAccountKeyField:
1✔
730
                        if getLatestAccountKey, err = strconv.ParseBool(v); err != nil {
2✔
731
                                return "", "", "", "", fmt.Errorf("invalid %s: %s in volume context", getLatestAccountKeyField, v)
1✔
732
                        }
1✔
733
                }
734
        }
735

736
        // 1. If keyVaultURL is not nil, preferentially use the key stored in key vault.
737
        // 2. Then if secrets map is not nil, use the key stored in the secrets map.
738
        // 3. Finally if both keyVaultURL and secrets map are nil, get the key from Azure.
739
        if keyVaultURL != "" {
2✔
740
                key, err := d.getKeyVaultSecretContent(ctx, keyVaultURL, keyVaultSecretName, keyVaultSecretVersion)
×
741
                if err != nil {
×
742
                        return "", "", "", "", err
×
743
                }
×
744
                if isSASToken(key) {
×
745
                        accountSasToken = key
×
746
                } else {
×
747
                        accountKey = key
×
748
                }
×
749
        } else {
2✔
750
                if len(secrets) == 0 {
4✔
751
                        var rgName string
2✔
752
                        rgName, accountName, containerName, _, _, err = GetContainerInfo(volumeID)
2✔
753
                        if err != nil {
2✔
754
                                return "", "", "", "", err
×
755
                        }
×
756

757
                        if rgName == "" {
2✔
758
                                rgName = d.cloud.ResourceGroup
×
759
                        }
×
760
                        accountKey, err = d.GetStorageAccesskeyWithSubsID(ctx, subsID, accountName, rgName, getLatestAccountKey)
2✔
761
                        if err != nil {
3✔
762
                                return "", "", "", "", fmt.Errorf("no key for storage account(%s) under resource group(%s), err %w", accountName, rgName, err)
1✔
763
                        }
1✔
764
                }
765
        }
766

767
        if containerName == "" {
1✔
768
                return "", "", "", "", fmt.Errorf("could not find containerName from attributes(%v) or volumeID(%v)", attrib, volumeID)
×
769
        }
×
770

771
        return accountName, accountKey, accountSasToken, containerName, nil
1✔
772
}
773

774
func IsCorruptedDir(dir string) bool {
4✔
775
        _, pathErr := mount.PathExists(dir)
4✔
776
        return pathErr != nil && mount.IsCorruptedMnt(pathErr)
4✔
777
}
4✔
778

779
func isRetriableError(err error) bool {
5✔
780
        if err != nil {
9✔
781
                for _, v := range retriableErrors {
19✔
782
                        if strings.Contains(strings.ToLower(err.Error()), strings.ToLower(v)) {
18✔
783
                                return true
3✔
784
                        }
3✔
785
                }
786
        }
787
        return false
2✔
788
}
789

790
func isSupportedProtocol(protocol string) bool {
19✔
791
        if protocol == "" {
20✔
792
                return true
1✔
793
        }
1✔
794
        for _, v := range supportedProtocolList {
46✔
795
                if protocol == v || protocol == NFSv3 {
44✔
796
                        return true
16✔
797
                }
16✔
798
        }
799
        return false
2✔
800
}
801

802
func isSupportedAccessTier(accessTier string) bool {
22✔
803
        if accessTier == "" {
37✔
804
                return true
15✔
805
        }
15✔
806
        for _, tier := range armstorage.PossibleAccessTierValues() {
32✔
807
                if accessTier == string(tier) {
28✔
808
                        return true
3✔
809
                }
3✔
810
        }
811
        return false
4✔
812
}
813

814
// container names can contain only lowercase letters, numbers, and hyphens,
815
// and must begin and end with a letter or a number
816
func isSupportedContainerNamePrefix(prefix string) bool {
21✔
817
        if prefix == "" {
34✔
818
                return true
13✔
819
        }
13✔
820
        if len(prefix) > 20 {
9✔
821
                return false
1✔
822
        }
1✔
823
        if prefix[0] == '-' {
8✔
824
                return false
1✔
825
        }
1✔
826
        for _, v := range prefix {
19✔
827
                if v != '-' && (v < '0' || v > '9') && (v < 'a' || v > 'z') {
17✔
828
                        return false
4✔
829
                }
4✔
830
        }
831
        return true
2✔
832
}
833

834
// isNFSProtocol checks if the protocol is NFS or AZNFS
835
func isNFSProtocol(protocol string) bool {
20✔
836
        protocol = strings.ToLower(protocol)
20✔
837
        return protocol == NFS || protocol == AZNFS || protocol == NFSv3
20✔
838
}
20✔
839

840
// get storage account from secrets map
841
func getStorageAccount(secrets map[string]string) (string, string, error) {
22✔
842
        if secrets == nil {
23✔
843
                return "", "", fmt.Errorf("unexpected: getStorageAccount secrets is nil")
1✔
844
        }
1✔
845

846
        var accountName, accountKey string
21✔
847
        for k, v := range secrets {
64✔
848
                v = strings.TrimSpace(v)
43✔
849
                switch strings.ToLower(k) {
43✔
850
                case accountNameField:
7✔
851
                        accountName = v
7✔
852
                case defaultSecretAccountName: // for compatibility with built-in azurefile plugin
13✔
853
                        accountName = v
13✔
854
                case accountKeyField:
7✔
855
                        accountKey = v
7✔
856
                case defaultSecretAccountKey: // for compatibility with built-in azurefile plugin
12✔
857
                        accountKey = v
12✔
858
                }
859
        }
860

861
        if accountName == "" {
25✔
862
                return accountName, accountKey, fmt.Errorf("could not find %s or %s field in secrets", accountNameField, defaultSecretAccountName)
4✔
863
        }
4✔
864
        if accountKey == "" {
21✔
865
                return accountName, accountKey, fmt.Errorf("could not find %s or %s field in secrets", accountKeyField, defaultSecretAccountKey)
4✔
866
        }
4✔
867

868
        accountName = strings.TrimSpace(accountName)
13✔
869
        klog.V(4).Infof("got storage account(%s) from secret", accountName)
13✔
870
        return accountName, accountKey, nil
13✔
871
}
872

873
func getContainerReference(containerName string, secrets map[string]string, storageEndpointSuffix string) (*azstorage.Container, error) {
9✔
874
        accountName, accountKey, rerr := getStorageAccount(secrets)
9✔
875
        if rerr != nil {
11✔
876
                return nil, rerr
2✔
877
        }
2✔
878
        client, err := azstorage.NewClient(accountName, accountKey, storageEndpointSuffix, azstorage.DefaultAPIVersion, true)
7✔
879
        if err != nil {
13✔
880
                return nil, err
6✔
881
        }
6✔
882
        blobClient := client.GetBlobService()
1✔
883
        container := blobClient.GetContainerReference(containerName)
1✔
884
        if container == nil {
1✔
885
                return nil, fmt.Errorf("ContainerReference of %s is nil", containerName)
×
886
        }
×
887
        return container, nil
1✔
888
}
889

890
func setAzureCredentials(ctx context.Context, kubeClient kubernetes.Interface, accountName, accountKey, secretNamespace string) (string, error) {
6✔
891
        if kubeClient == nil {
8✔
892
                klog.Warningf("could not create secret: kubeClient is nil")
2✔
893
                return "", nil
2✔
894
        }
2✔
895
        if accountName == "" || accountKey == "" {
6✔
896
                return "", fmt.Errorf("the account info is not enough, accountName(%v), accountKey(%v)", accountName, accountKey)
2✔
897
        }
2✔
898
        secretName := fmt.Sprintf(secretNameTemplate, accountName)
2✔
899
        secret := &v1.Secret{
2✔
900
                ObjectMeta: metav1.ObjectMeta{
2✔
901
                        Namespace: secretNamespace,
2✔
902
                        Name:      secretName,
2✔
903
                },
2✔
904
                Data: map[string][]byte{
2✔
905
                        defaultSecretAccountName: []byte(accountName),
2✔
906
                        defaultSecretAccountKey:  []byte(accountKey),
2✔
907
                },
2✔
908
                Type: "Opaque",
2✔
909
        }
2✔
910
        _, err := kubeClient.CoreV1().Secrets(secretNamespace).Create(ctx, secret, metav1.CreateOptions{})
2✔
911
        if apierror.IsAlreadyExists(err) {
3✔
912
                err = nil
1✔
913
        }
1✔
914
        if err != nil {
2✔
915
                return "", fmt.Errorf("couldn't create secret %w", err)
×
916
        }
×
917
        return secretName, err
2✔
918
}
919

920
// GetStorageAccesskey get Azure storage account key from
921
//  1. secrets (if not empty)
922
//  2. use k8s client identity to read from k8s secret
923
//  3. use cluster identity to get from storage account directly
924
func (d *Driver) GetStorageAccesskey(ctx context.Context, accountOptions *storage.AccountOptions, secrets map[string]string, secretName, secretNamespace string) (string, string, error) {
12✔
925
        if len(secrets) > 0 {
17✔
926
                return getStorageAccount(secrets)
5✔
927
        }
5✔
928

929
        // read from k8s secret first
930
        if secretName == "" {
12✔
931
                secretName = fmt.Sprintf(secretNameTemplate, accountOptions.Name)
5✔
932
        }
5✔
933
        _, accountKey, _, _, _, _, _, err := d.GetInfoFromSecret(ctx, secretName, secretNamespace) //nolint
7✔
934
        if err != nil && d.cloud != nil {
12✔
935
                klog.V(2).Infof("could not get account(%s) key from secret(%s) namespace(%s), error: %v, use cluster identity to get account key instead", accountOptions.Name, secretName, secretNamespace, err)
5✔
936
                accountKey, err = d.GetStorageAccesskeyWithSubsID(ctx, accountOptions.SubscriptionID, accountOptions.Name, accountOptions.ResourceGroup, accountOptions.GetLatestAccountKey)
5✔
937
        }
5✔
938
        return accountOptions.Name, accountKey, err
7✔
939
}
940

941
// GetStorageAccesskeyWithSubsID get Azure storage account key from storage account directly
942
func (d *Driver) GetStorageAccesskeyWithSubsID(ctx context.Context, subsID, account, resourceGroup string, getLatestAccountKey bool) (string, error) {
15✔
943
        if d.cloud == nil || d.cloud.ComputeClientFactory == nil {
17✔
944
                return "", fmt.Errorf("could not get account key: cloud or ComputeClientFactory is nil")
2✔
945
        }
2✔
946
        accountClient, err := d.cloud.ComputeClientFactory.GetAccountClientForSub(subsID)
13✔
947
        if err != nil {
13✔
948
                return "", err
×
949
        }
×
950
        return d.cloud.GetStorageAccesskey(ctx, accountClient, account, resourceGroup, getLatestAccountKey)
13✔
951
}
952

953
// GetInfoFromSecret get info from k8s secret
954
// return <accountName, accountKey, accountSasToken, msiSecret, spnClientSecret, spnClientID, spnTenantID, error>
955
func (d *Driver) GetInfoFromSecret(ctx context.Context, secretName, secretNamespace string) (string, string, string, string, string, string, string, error) {
16✔
956
        if d.KubeClient == nil {
26✔
957
                return "", "", "", "", "", "", "", fmt.Errorf("could not get account key from secret(%s): KubeClient is nil", secretName)
10✔
958
        }
10✔
959

960
        secret, err := d.KubeClient.CoreV1().Secrets(secretNamespace).Get(ctx, secretName, metav1.GetOptions{})
6✔
961
        if err != nil {
8✔
962
                return "", "", "", "", "", "", "", fmt.Errorf("could not get secret(%v): %w", secretName, err)
2✔
963
        }
2✔
964

965
        accountName := strings.TrimSpace(string(secret.Data[defaultSecretAccountName][:]))
4✔
966
        accountKey := strings.TrimSpace(string(secret.Data[defaultSecretAccountKey][:]))
4✔
967
        accountSasToken := strings.TrimSpace(string(secret.Data[accountSasTokenField][:]))
4✔
968
        msiSecret := strings.TrimSpace(string(secret.Data[msiSecretField][:]))
4✔
969
        spnClientSecret := strings.TrimSpace(string(secret.Data[storageSPNClientSecretField][:]))
4✔
970
        spnClientID := strings.TrimSpace(string(secret.Data[storageSPNClientIDField][:]))
4✔
971
        spnTenantID := strings.TrimSpace(string(secret.Data[storageSPNTenantIDField][:]))
4✔
972

4✔
973
        klog.V(4).Infof("got storage account(%s) from secret(%s) namespace(%s)", accountName, secretName, secretNamespace)
4✔
974
        return accountName, accountKey, accountSasToken, msiSecret, spnClientSecret, spnClientID, spnTenantID, nil
4✔
975
}
976

977
// getSubnetResourceID get default subnet resource ID from cloud provider config
978
func (d *Driver) getSubnetResourceID(vnetResourceGroup, vnetName, subnetName string) string {
6✔
979
        subsID := d.cloud.SubscriptionID
6✔
980
        if len(d.cloud.NetworkResourceSubscriptionID) > 0 {
10✔
981
                subsID = d.cloud.NetworkResourceSubscriptionID
4✔
982
        }
4✔
983

984
        if len(vnetResourceGroup) == 0 {
10✔
985
                vnetResourceGroup = d.cloud.ResourceGroup
4✔
986
                if len(d.cloud.VnetResourceGroup) > 0 {
7✔
987
                        vnetResourceGroup = d.cloud.VnetResourceGroup
3✔
988
                }
3✔
989
        }
990

991
        if len(vnetName) == 0 {
10✔
992
                vnetName = d.cloud.VnetName
4✔
993
        }
4✔
994

995
        if len(subnetName) == 0 {
10✔
996
                subnetName = d.cloud.SubnetName
4✔
997
        }
4✔
998
        return fmt.Sprintf(subnetTemplate, subsID, vnetResourceGroup, vnetName, subnetName)
6✔
999
}
1000

1001
func (d *Driver) useDataPlaneAPI(ctx context.Context, volumeID, accountName string) bool {
9✔
1002
        cache, err := d.dataPlaneAPIVolCache.Get(ctx, volumeID, azcache.CacheReadTypeDefault)
9✔
1003
        if err != nil {
9✔
1004
                klog.Errorf("get(%s) from dataPlaneAPIVolCache failed with error: %v", volumeID, err)
×
1005
        }
×
1006
        if cache != nil {
12✔
1007
                return true
3✔
1008
        }
3✔
1009
        cache, err = d.dataPlaneAPIVolCache.Get(ctx, accountName, azcache.CacheReadTypeDefault)
6✔
1010
        if err != nil {
6✔
1011
                klog.Errorf("get(%s) from dataPlaneAPIVolCache failed with error: %v", accountName, err)
×
1012
        }
×
1013
        if cache != nil {
6✔
1014
                return true
×
1015
        }
×
1016
        return false
6✔
1017
}
1018

1019
// appendDefaultMountOptions return mount options combined with mountOptions and defaultMountOptions
1020
func appendDefaultMountOptions(mountOptions []string, tmpPath, containerName string) []string {
4✔
1021
        var defaultMountOptions = map[string]string{
4✔
1022
                "--pre-mount-validate": "true",
4✔
1023
                "--use-https":          "true",
4✔
1024
                "--tmp-path":           tmpPath,
4✔
1025
                "--container-name":     containerName,
4✔
1026
                // prevent billing charges on mounting
4✔
1027
                "--cancel-list-on-mount-seconds": "10",
4✔
1028
                // allow remounting using a non-empty tmp-path
4✔
1029
                "--empty-dir-check": "false",
4✔
1030
        }
4✔
1031

4✔
1032
        // stores the mount options already included in mountOptions
4✔
1033
        included := make(map[string]bool)
4✔
1034

4✔
1035
        for _, mountOption := range mountOptions {
11✔
1036
                for k := range defaultMountOptions {
49✔
1037
                        if strings.HasPrefix(mountOption, k) {
46✔
1038
                                included[k] = true
4✔
1039
                        }
4✔
1040
                }
1041
        }
1042

1043
        allMountOptions := mountOptions
4✔
1044

4✔
1045
        for k, v := range defaultMountOptions {
28✔
1046
                if _, isIncluded := included[k]; !isIncluded {
44✔
1047
                        if v != "" {
40✔
1048
                                allMountOptions = append(allMountOptions, fmt.Sprintf("%s=%s", k, v))
20✔
1049
                        } else {
20✔
1050
                                allMountOptions = append(allMountOptions, k)
×
1051
                        }
×
1052
                }
1053
        }
1054

1055
        return allMountOptions
4✔
1056
}
1057

1058
// chmodIfPermissionMismatch only perform chmod when permission mismatches
1059
func chmodIfPermissionMismatch(targetPath string, mode os.FileMode) error {
3✔
1060
        info, err := os.Lstat(targetPath)
3✔
1061
        if err != nil {
4✔
1062
                return err
1✔
1063
        }
1✔
1064
        perm := info.Mode() & os.ModePerm
2✔
1065
        if perm != mode {
3✔
1066
                klog.V(2).Infof("chmod targetPath(%s, mode:0%o) with permissions(0%o)", targetPath, info.Mode(), mode)
1✔
1067
                if err := os.Chmod(targetPath, mode); err != nil {
1✔
1068
                        return err
×
1069
                }
×
1070
        } else {
1✔
1071
                klog.V(2).Infof("skip chmod on targetPath(%s) since mode is already 0%o)", targetPath, info.Mode())
1✔
1072
        }
1✔
1073
        return nil
2✔
1074
}
1075

1076
func createStorageAccountSecret(account, key string) map[string]string {
1✔
1077
        secret := make(map[string]string)
1✔
1078
        secret[defaultSecretAccountName] = account
1✔
1079
        secret[defaultSecretAccountKey] = key
1✔
1080
        return secret
1✔
1081
}
1✔
1082

1083
// setKeyValueInMap set key/value pair in map
1084
// key in the map is case insensitive, if key already exists, overwrite existing value
1085
func setKeyValueInMap(m map[string]string, key, value string) {
7✔
1086
        if m == nil {
8✔
1087
                return
1✔
1088
        }
1✔
1089
        for k := range m {
17✔
1090
                if strings.EqualFold(k, key) {
13✔
1091
                        m[k] = value
2✔
1092
                        return
2✔
1093
                }
2✔
1094
        }
1095
        m[key] = value
4✔
1096
}
1097

1098
// getValueInMap get value from map by key
1099
// key in the map is case insensitive
1100
func getValueInMap(m map[string]string, key string) string {
12✔
1101
        if m == nil {
13✔
1102
                return ""
1✔
1103
        }
1✔
1104
        for k, v := range m {
23✔
1105
                if strings.EqualFold(k, key) {
16✔
1106
                        return v
4✔
1107
                }
4✔
1108
        }
1109
        return ""
7✔
1110
}
1111

1112
// replaceWithMap replace key with value for str
1113
func replaceWithMap(str string, m map[string]string) string {
14✔
1114
        for k, v := range m {
19✔
1115
                if k != "" {
9✔
1116
                        str = strings.ReplaceAll(str, k, v)
4✔
1117
                }
4✔
1118
        }
1119
        return str
14✔
1120
}
1121

1122
func isSupportedFSGroupChangePolicy(policy string) bool {
27✔
1123
        if policy == "" {
47✔
1124
                return true
20✔
1125
        }
20✔
1126
        for _, v := range supportedFSGroupChangePolicyList {
25✔
1127
                if policy == v {
21✔
1128
                        return true
3✔
1129
                }
3✔
1130
        }
1131
        return false
4✔
1132
}
1133

1134
func isReadOnlyFromCapability(vc *csi.VolumeCapability) bool {
7✔
1135
        if vc.GetAccessMode() == nil {
9✔
1136
                return false
2✔
1137
        }
2✔
1138
        mode := vc.GetAccessMode().GetMode()
5✔
1139
        return (mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY ||
5✔
1140
                mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY)
5✔
1141
}
1142

1143
// generateVolumeName returns a PV name with clusterName prefix. The function
1144
// should be used to generate a name of GCE PD or Cinder volume. It basically
1145
// adds "<clusterName>-dynamic-" before the PV name, making sure the resulting
1146
// string fits given length and cuts "dynamic" if not.
1147
func generateVolumeName(clusterName, pvName string, maxLength int) string {
3✔
1148
        prefix := clusterName + "-dynamic"
3✔
1149
        pvLen := len(pvName)
3✔
1150
        // cut the "<clusterName>-dynamic" to fit full pvName into maxLength
3✔
1151
        // +1 for the '-' dash
3✔
1152
        if pvLen+1+len(prefix) > maxLength {
5✔
1153
                prefix = prefix[:maxLength-pvLen-1]
2✔
1154
        }
2✔
1155
        return prefix + "-" + pvName
3✔
1156
}
1157

1158
// serviceAccountToken represents the service account token sent from NodePublishVolume Request.
1159
// ref: https://kubernetes-csi.github.io/docs/token-requests.html
1160
type serviceAccountToken struct {
1161
        APIAzureADTokenExchange struct {
1162
                Token               string    `json:"token"`
1163
                ExpirationTimestamp time.Time `json:"expirationTimestamp"`
1164
        } `json:"api://AzureADTokenExchange"`
1165
}
1166

1167
// parseServiceAccountToken parses the bound service account token from the token passed from NodePublishVolume Request.
1168
// ref: https://kubernetes-csi.github.io/docs/token-requests.html
1169
func parseServiceAccountToken(tokenStr string) (string, error) {
5✔
1170
        if len(tokenStr) == 0 {
6✔
1171
                return "", fmt.Errorf("service account token is empty")
1✔
1172
        }
1✔
1173
        token := serviceAccountToken{}
4✔
1174
        if err := json.Unmarshal([]byte(tokenStr), &token); err != nil {
5✔
1175
                return "", fmt.Errorf("failed to unmarshal service account tokens, error: %w", err)
1✔
1176
        }
1✔
1177
        if token.APIAzureADTokenExchange.Token == "" {
5✔
1178
                return "", fmt.Errorf("token for audience %s not found", DefaultTokenAudience)
2✔
1179
        }
2✔
1180
        return token.APIAzureADTokenExchange.Token, nil
1✔
1181
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc