• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

kubernetes-sigs / blob-csi-driver / 4916492637

08 May 2023 03:09PM CUT coverage: 80.771%. Remained the same
4916492637

Pull #916

github

GitHub
chore(deps): bump github/codeql-action from 1 to 2
Pull Request #916: chore(deps): bump github/codeql-action from 1 to 2

1823 of 2257 relevant lines covered (80.77%)

5.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.38
/pkg/blob/blob.go
1
/*
2
Copyright 2019 The Kubernetes Authors.
3

4
Licensed under the Apache License, Version 2.0 (the "License");
5
you may not use this file except in compliance with the License.
6
You may obtain a copy of the License at
7

8
    http://www.apache.org/licenses/LICENSE-2.0
9

10
Unless required by applicable law or agreed to in writing, software
11
distributed under the License is distributed on an "AS IS" BASIS,
12
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
See the License for the specific language governing permissions and
14
limitations under the License.
15
*/
16

17
package blob
18

19
import (
20
        "fmt"
21
        "os"
22
        "strings"
23
        "sync"
24
        "time"
25

26
        "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2021-09-01/storage"
27
        azstorage "github.com/Azure/azure-sdk-for-go/storage"
28
        az "github.com/Azure/go-autorest/autorest/azure"
29
        "github.com/container-storage-interface/spec/lib/go/csi"
30
        "github.com/pborman/uuid"
31
        "golang.org/x/net/context"
32

33
        v1 "k8s.io/api/core/v1"
34
        "k8s.io/apimachinery/pkg/api/errors"
35
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
36
        "k8s.io/client-go/kubernetes"
37
        "k8s.io/klog/v2"
38
        k8sutil "k8s.io/kubernetes/pkg/volume/util"
39
        mount "k8s.io/mount-utils"
40
        utilexec "k8s.io/utils/exec"
41

42
        csicommon "sigs.k8s.io/blob-csi-driver/pkg/csi-common"
43
        "sigs.k8s.io/blob-csi-driver/pkg/util"
44
        azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
45
        azure "sigs.k8s.io/cloud-provider-azure/pkg/provider"
46
)
47

48
const (
49
        // DefaultDriverName holds the name of the csi-driver
50
        DefaultDriverName            = "blob.csi.azure.com"
51
        blobCSIDriverName            = "blob_csi_driver"
52
        separator                    = "#"
53
        volumeIDTemplate             = "%s#%s#%s#%s#%s#%s"
54
        secretNameTemplate           = "azure-storage-account-%s-secret"
55
        serverNameField              = "server"
56
        storageEndpointSuffixField   = "storageendpointsuffix"
57
        tagsField                    = "tags"
58
        matchTagsField               = "matchtags"
59
        protocolField                = "protocol"
60
        accountNameField             = "accountname"
61
        accountKeyField              = "accountkey"
62
        storageAccountField          = "storageaccount"
63
        storageAccountTypeField      = "storageaccounttype"
64
        skuNameField                 = "skuname"
65
        subscriptionIDField          = "subscriptionid"
66
        resourceGroupField           = "resourcegroup"
67
        locationField                = "location"
68
        secretNameField              = "secretname"
69
        secretNamespaceField         = "secretnamespace"
70
        containerNameField           = "containername"
71
        containerNamePrefixField     = "containernameprefix"
72
        storeAccountKeyField         = "storeaccountkey"
73
        isHnsEnabledField            = "ishnsenabled"
74
        softDeleteBlobsField         = "softdeleteblobs"
75
        softDeleteContainersField    = "softdeletecontainers"
76
        enableBlobVersioningField    = "enableblobversioning"
77
        getAccountKeyFromSecretField = "getaccountkeyfromsecret"
78
        keyVaultURLField             = "keyvaulturl"
79
        keyVaultSecretNameField      = "keyvaultsecretname"
80
        keyVaultSecretVersionField   = "keyvaultsecretversion"
81
        storageAccountNameField      = "storageaccountname"
82
        allowBlobPublicAccessField   = "allowblobpublicaccess"
83
        requireInfraEncryptionField  = "requireinfraencryption"
84
        ephemeralField               = "csi.storage.k8s.io/ephemeral"
85
        podNamespaceField            = "csi.storage.k8s.io/pod.namespace"
86
        mountOptionsField            = "mountoptions"
87
        falseValue                   = "false"
88
        trueValue                    = "true"
89
        defaultSecretAccountName     = "azurestorageaccountname"
90
        defaultSecretAccountKey      = "azurestorageaccountkey"
91
        accountSasTokenField         = "azurestorageaccountsastoken"
92
        msiSecretField               = "msisecret"
93
        storageSPNClientSecretField  = "azurestoragespnclientsecret"
94
        Fuse                         = "fuse"
95
        Fuse2                        = "fuse2"
96
        NFS                          = "nfs"
97
        vnetResourceGroupField       = "vnetresourcegroup"
98
        vnetNameField                = "vnetname"
99
        subnetNameField              = "subnetname"
100
        accessTierField              = "accesstier"
101
        networkEndpointTypeField     = "networkendpointtype"
102
        mountPermissionsField        = "mountpermissions"
103
        useDataPlaneAPIField         = "usedataplaneapi"
104

105
        // See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names
106
        containerNameMinLength = 3
107
        containerNameMaxLength = 63
108

109
        accountNotProvisioned                   = "StorageAccountIsNotProvisioned"
110
        tooManyRequests                         = "TooManyRequests"
111
        clientThrottled                         = "client throttled"
112
        containerBeingDeletedDataplaneAPIError  = "ContainerBeingDeleted"
113
        containerBeingDeletedManagementAPIError = "container is being deleted"
114
        statusCodeNotFound                      = "StatusCode=404"
115
        httpCodeNotFound                        = "HTTPStatusCode: 404"
116

117
        // containerMaxSize is the max size of the blob container. See https://docs.microsoft.com/en-us/azure/storage/blobs/scalability-targets#scale-targets-for-blob-storage
118
        containerMaxSize = 100 * util.TiB
119

120
        subnetTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s"
121

122
        defaultNamespace = "default"
123

124
        pvcNameKey           = "csi.storage.k8s.io/pvc/name"
125
        pvcNamespaceKey      = "csi.storage.k8s.io/pvc/namespace"
126
        pvNameKey            = "csi.storage.k8s.io/pv/name"
127
        pvcNameMetadata      = "${pvc.metadata.name}"
128
        pvcNamespaceMetadata = "${pvc.metadata.namespace}"
129
        pvNameMetadata       = "${pv.metadata.name}"
130

131
        VolumeID = "volumeid"
132

133
        defaultStorageEndPointSuffix = "core.windows.net"
134
)
135

136
var (
137
        supportedProtocolList = []string{Fuse, Fuse2, NFS}
138
        retriableErrors       = []string{accountNotProvisioned, tooManyRequests, statusCodeNotFound, containerBeingDeletedDataplaneAPIError, containerBeingDeletedManagementAPIError, clientThrottled}
139
)
140

141
// DriverOptions defines driver parameters specified in driver deployment
142
type DriverOptions struct {
143
        NodeID                                 string
144
        DriverName                             string
145
        CloudConfigSecretName                  string
146
        CloudConfigSecretNamespace             string
147
        CustomUserAgent                        string
148
        UserAgentSuffix                        string
149
        BlobfuseProxyEndpoint                  string
150
        EnableBlobfuseProxy                    bool
151
        BlobfuseProxyConnTimout                int
152
        EnableBlobMockMount                    bool
153
        AllowEmptyCloudConfig                  bool
154
        AllowInlineVolumeKeyAccessWithIdentity bool
155
        EnableGetVolumeStats                   bool
156
        AppendTimeStampInCacheDir              bool
157
        AppendMountErrorHelpLink               bool
158
        MountPermissions                       uint64
159
        KubeAPIQPS                             float64
160
        KubeAPIBurst                           int
161
}
162

163
// Driver implements all interfaces of CSI drivers
164
type Driver struct {
165
        csicommon.CSIDriver
166

167
        cloud                      *azure.Cloud
168
        cloudConfigSecretName      string
169
        cloudConfigSecretNamespace string
170
        customUserAgent            string
171
        userAgentSuffix            string
172
        blobfuseProxyEndpoint      string
173
        // enableBlobMockMount is only for testing, DO NOT set as true in non-testing scenario
174
        enableBlobMockMount                    bool
175
        enableBlobfuseProxy                    bool
176
        allowEmptyCloudConfig                  bool
177
        enableGetVolumeStats                   bool
178
        allowInlineVolumeKeyAccessWithIdentity bool
179
        appendTimeStampInCacheDir              bool
180
        appendMountErrorHelpLink               bool
181
        blobfuseProxyConnTimout                int
182
        mountPermissions                       uint64
183
        kubeAPIQPS                             float64
184
        kubeAPIBurst                           int
185
        mounter                                *mount.SafeFormatAndMount
186
        volLockMap                             *util.LockMap
187
        // A map storing all volumes with ongoing operations so that additional operations
188
        // for that same volume (as defined by VolumeID) return an Aborted error
189
        volumeLocks *volumeLocks
190
        // only for nfs feature
191
        subnetLockMap *util.LockMap
192
        // a map storing all volumes created by this driver <volumeName, accountName>
193
        volMap sync.Map
194
        // a timed cache storing all volumeIDs and storage accounts that are using data plane API
195
        dataPlaneAPIVolCache *azcache.TimedCache
196
        // a timed cache storing account search history (solve account list throttling issue)
197
        accountSearchCache *azcache.TimedCache
198
}
199

200
// NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version &
201
// does not support optional driver plugin info manifest field. Refer to CSI spec for more details.
202
func NewDriver(options *DriverOptions) *Driver {
203
        d := Driver{
204
                volLockMap:                             util.NewLockMap(),
205
                subnetLockMap:                          util.NewLockMap(),
206
                volumeLocks:                            newVolumeLocks(),
207
                cloudConfigSecretName:                  options.CloudConfigSecretName,
208
                cloudConfigSecretNamespace:             options.CloudConfigSecretNamespace,
209
                customUserAgent:                        options.CustomUserAgent,
210
                userAgentSuffix:                        options.UserAgentSuffix,
211
                blobfuseProxyEndpoint:                  options.BlobfuseProxyEndpoint,
212
                enableBlobfuseProxy:                    options.EnableBlobfuseProxy,
213
                allowInlineVolumeKeyAccessWithIdentity: options.AllowInlineVolumeKeyAccessWithIdentity,
214
                blobfuseProxyConnTimout:                options.BlobfuseProxyConnTimout,
215
                enableBlobMockMount:                    options.EnableBlobMockMount,
216
                allowEmptyCloudConfig:                  options.AllowEmptyCloudConfig,
217
                enableGetVolumeStats:                   options.EnableGetVolumeStats,
218
                appendMountErrorHelpLink:               options.AppendMountErrorHelpLink,
219
                mountPermissions:                       options.MountPermissions,
220
                kubeAPIQPS:                             options.KubeAPIQPS,
221
                kubeAPIBurst:                           options.KubeAPIBurst,
222
        }
223
        d.Name = options.DriverName
224
        d.Version = driverVersion
225
        d.NodeID = options.NodeID
226

227
        var err error
228
        getter := func(key string) (interface{}, error) { return nil, nil }
229
        if d.accountSearchCache, err = azcache.NewTimedcache(time.Minute, getter); err != nil {
230
                klog.Fatalf("%v", err)
231
        }
232
        if d.dataPlaneAPIVolCache, err = azcache.NewTimedcache(10*time.Minute, getter); err != nil {
233
                klog.Fatalf("%v", err)
234
        }
235
        return &d
236
}
237

238
// Run driver initialization
239
func (d *Driver) Run(endpoint, kubeconfig string, testBool bool) {
240
        versionMeta, err := GetVersionYAML(d.Name)
241
        if err != nil {
242
                klog.Fatalf("%v", err)
243
        }
244
        klog.Infof("\nDRIVER INFORMATION:\n-------------------\n%s\n\nStreaming logs below:", versionMeta)
245

246
        userAgent := GetUserAgent(d.Name, d.customUserAgent, d.userAgentSuffix)
247
        klog.V(2).Infof("driver userAgent: %s", userAgent)
248
        d.cloud, err = getCloudProvider(kubeconfig, d.NodeID, d.cloudConfigSecretName, d.cloudConfigSecretNamespace, userAgent, d.allowEmptyCloudConfig, d.kubeAPIQPS, d.kubeAPIBurst)
249
        if err != nil {
250
                klog.Fatalf("failed to get Azure Cloud Provider, error: %v", err)
251
        }
252
        klog.V(2).Infof("cloud: %s, location: %s, rg: %s, VnetName: %s, VnetResourceGroup: %s, SubnetName: %s", d.cloud.Cloud, d.cloud.Location, d.cloud.ResourceGroup, d.cloud.VnetName, d.cloud.VnetResourceGroup, d.cloud.SubnetName)
253

254
        d.mounter = &mount.SafeFormatAndMount{
255
                Interface: mount.New(""),
256
                Exec:      utilexec.New(),
257
        }
258

259
        // Initialize default library driver
260
        d.AddControllerServiceCapabilities(
261
                []csi.ControllerServiceCapability_RPC_Type{
262
                        csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
263
                        //csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
264
                        //csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
265
                        csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
266
                        csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER,
267
                })
268
        d.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
269
                csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
270
                csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY,
271
                csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER,
272
                csi.VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER,
273
                csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY,
274
                csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER,
275
                csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
276
        })
277

278
        nodeCap := []csi.NodeServiceCapability_RPC_Type{
279
                csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
280
                csi.NodeServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER,
281
        }
282
        if d.enableGetVolumeStats {
283
                nodeCap = append(nodeCap, csi.NodeServiceCapability_RPC_GET_VOLUME_STATS)
284
        }
285
        d.AddNodeServiceCapabilities(nodeCap)
286

287
        s := csicommon.NewNonBlockingGRPCServer()
288
        // Driver d act as IdentityServer, ControllerServer and NodeServer
289
        s.Start(endpoint, d, d, d, testBool)
290
        s.Wait()
291
}
292

293
// GetContainerInfo get container info according to volume id
294
// the format of VolumeId is: rg#accountName#containerName#uuid#secretNamespace#subsID
295
//
296
// e.g.
297
// input: "rg#f5713de20cde511e8ba4900#containerName#uuid#"
298
// output: rg, f5713de20cde511e8ba4900, containerName, "" , ""
299
// input: "rg#f5713de20cde511e8ba4900#containerName#uuid#namespace#"
300
// output: rg, f5713de20cde511e8ba4900, containerName, namespace, ""
301
// input: "rg#f5713de20cde511e8ba4900#containerName#uuid#namespace#subsID"
302
// output: rg, f5713de20cde511e8ba4900, containerName, namespace, subsID
303
func GetContainerInfo(id string) (string, string, string, string, string, error) {
304
        segments := strings.Split(id, separator)
305
        if len(segments) < 3 {
306
                return "", "", "", "", "", fmt.Errorf("error parsing volume id: %q, should at least contain two #", id)
307
        }
308
        var secretNamespace, subsID string
309
        if len(segments) > 4 {
310
                secretNamespace = segments[4]
311
        }
312
        if len(segments) > 5 {
313
                subsID = segments[5]
314
        }
315
        return segments[0], segments[1], segments[2], secretNamespace, subsID, nil
316
}
317

318
// A container name must be a valid DNS name, conforming to the following naming rules:
319
//  1. Container names must start with a letter or number, and can contain only letters, numbers, and the dash (-) character.
320
//  2. Every dash (-) character must be immediately preceded and followed by a letter or number; consecutive dashes are not permitted in container names.
321
//  3. All letters in a container name must be lowercase.
322
//  4. Container names must be from 3 through 63 characters long.
323
//
324
// See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names
325
func getValidContainerName(volumeName, protocol string) string {
326
        containerName := strings.ToLower(volumeName)
327
        if len(containerName) > containerNameMaxLength {
328
                containerName = containerName[0:containerNameMaxLength]
329
        }
330
        if !checkContainerNameBeginAndEnd(containerName) || len(containerName) < containerNameMinLength {
331
                // now we set as 63 for maximum container name length
332
                // todo: get cluster name
333
                containerName = k8sutil.GenerateVolumeName(fmt.Sprintf("pvc-%s", protocol), uuid.NewUUID().String(), 63)
334
                klog.Warningf("requested volume name (%s) is invalid, regenerated as (%q)", volumeName, containerName)
335
        }
336
        return strings.Replace(containerName, "--", "-", -1)
337
}
338

339
func checkContainerNameBeginAndEnd(containerName string) bool {
340
        length := len(containerName)
341
        if (('a' <= containerName[0] && containerName[0] <= 'z') ||
342
                ('0' <= containerName[0] && containerName[0] <= '9')) &&
343
                (('a' <= containerName[length-1] && containerName[length-1] <= 'z') ||
344
                        ('0' <= containerName[length-1] && containerName[length-1] <= '9')) {
345
                return true
346
        }
347

348
        return false
349
}
350

351
// isSASToken checks if the key contains the patterns.
352
// SAS token format could refer to https://docs.microsoft.com/en-us/rest/api/eventhub/generate-sas-token
353
func isSASToken(key string) bool {
354
        return strings.HasPrefix(key, "?")
355
}
356

357
// GetAuthEnv return <accountName, containerName, authEnv, error>
358
func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attrib, secrets map[string]string) (string, string, string, string, []string, error) {
359
        rgName, accountName, containerName, secretNamespace, _, err := GetContainerInfo(volumeID)
360
        if err != nil {
361
                // ignore volumeID parsing error
362
                klog.V(2).Infof("parsing volumeID(%s) return with error: %v", volumeID, err)
363
                err = nil
364
        }
365

366
        var (
367
                subsID                  string
368
                accountKey              string
369
                accountSasToken         string
370
                msiSecret               string
371
                storageSPNClientSecret  string
372
                secretName              string
373
                pvcNamespace            string
374
                keyVaultURL             string
375
                keyVaultSecretName      string
376
                keyVaultSecretVersion   string
377
                azureStorageAuthType    string
378
                authEnv                 []string
379
                getAccountKeyFromSecret bool
380
        )
381

382
        for k, v := range attrib {
383
                switch strings.ToLower(k) {
384
                case subscriptionIDField:
385
                        subsID = v
386
                case resourceGroupField:
387
                        rgName = v
388
                case containerNameField:
389
                        containerName = v
390
                case keyVaultURLField:
391
                        keyVaultURL = v
392
                case keyVaultSecretNameField:
393
                        keyVaultSecretName = v
394
                case keyVaultSecretVersionField:
395
                        keyVaultSecretVersion = v
396
                case storageAccountField:
397
                        accountName = v
398
                case storageAccountNameField: // for compatibility
399
                        accountName = v
400
                case secretNameField:
401
                        secretName = v
402
                case secretNamespaceField:
403
                        secretNamespace = v
404
                case pvcNamespaceKey:
405
                        pvcNamespace = v
406
                case getAccountKeyFromSecretField:
407
                        getAccountKeyFromSecret = strings.EqualFold(v, trueValue)
408
                case "azurestorageauthtype":
409
                        azureStorageAuthType = v
410
                        authEnv = append(authEnv, "AZURE_STORAGE_AUTH_TYPE="+v)
411
                case "azurestorageidentityclientid":
412
                        authEnv = append(authEnv, "AZURE_STORAGE_IDENTITY_CLIENT_ID="+v)
413
                case "azurestorageidentityobjectid":
414
                        authEnv = append(authEnv, "AZURE_STORAGE_IDENTITY_OBJECT_ID="+v)
415
                case "azurestorageidentityresourceid":
416
                        authEnv = append(authEnv, "AZURE_STORAGE_IDENTITY_RESOURCE_ID="+v)
417
                case "msiendpoint":
418
                        authEnv = append(authEnv, "MSI_ENDPOINT="+v)
419
                case "azurestoragespnclientid":
420
                        authEnv = append(authEnv, "AZURE_STORAGE_SPN_CLIENT_ID="+v)
421
                case "azurestoragespntenantid":
422
                        authEnv = append(authEnv, "AZURE_STORAGE_SPN_TENANT_ID="+v)
423
                case "azurestorageaadendpoint":
424
                        authEnv = append(authEnv, "AZURE_STORAGE_AAD_ENDPOINT="+v)
425
                }
426
        }
427
        klog.V(2).Infof("volumeID(%s) authEnv: %s", volumeID, authEnv)
428

429
        if protocol == NFS {
430
                // nfs protocol does not need account key, return directly
431
                return rgName, accountName, accountKey, containerName, authEnv, err
432
        }
433

434
        if secretNamespace == "" {
435
                if pvcNamespace == "" {
436
                        secretNamespace = defaultNamespace
437
                } else {
438
                        secretNamespace = pvcNamespace
439
                }
440
        }
441

442
        if rgName == "" {
443
                rgName = d.cloud.ResourceGroup
444
        }
445

446
        // 1. If keyVaultURL is not nil, preferentially use the key stored in key vault.
447
        // 2. Then if secrets map is not nil, use the key stored in the secrets map.
448
        // 3. Finally if both keyVaultURL and secrets map are nil, get the key from Azure.
449
        if keyVaultURL != "" {
450
                key, err := d.getKeyVaultSecretContent(ctx, keyVaultURL, keyVaultSecretName, keyVaultSecretVersion)
451
                if err != nil {
452
                        return rgName, accountName, accountKey, containerName, authEnv, err
453
                }
454
                if isSASToken(key) {
455
                        accountSasToken = key
456
                } else {
457
                        accountKey = key
458
                }
459
        } else {
460
                if len(secrets) == 0 {
461
                        if secretName == "" && accountName != "" {
462
                                secretName = fmt.Sprintf(secretNameTemplate, accountName)
463
                        }
464
                        if secretName != "" {
465
                                // read from k8s secret first
466
                                var name string
467
                                name, accountKey, accountSasToken, msiSecret, storageSPNClientSecret, err = d.GetInfoFromSecret(ctx, secretName, secretNamespace)
468
                                if name != "" {
469
                                        accountName = name
470
                                }
471
                                if err != nil && !getAccountKeyFromSecret && (azureStorageAuthType == "" || strings.EqualFold(azureStorageAuthType, "key")) {
472
                                        klog.V(2).Infof("get account(%s) key from secret(%s, %s) failed with error: %v, use cluster identity to get account key instead",
473
                                                accountName, secretNamespace, secretName, err)
474
                                        accountKey, err = d.cloud.GetStorageAccesskey(ctx, subsID, accountName, rgName)
475
                                        if err != nil {
476
                                                return rgName, accountName, accountKey, containerName, authEnv, fmt.Errorf("no key for storage account(%s) under resource group(%s), err %w", accountName, rgName, err)
477
                                        }
478
                                }
479
                        }
480
                } else {
481
                        for k, v := range secrets {
482
                                v = strings.TrimSpace(v)
483
                                switch strings.ToLower(k) {
484
                                case accountNameField:
485
                                        accountName = v
486
                                case defaultSecretAccountName: // for compatibility with built-in blobfuse plugin
487
                                        accountName = v
488
                                case accountKeyField:
489
                                        accountKey = v
490
                                case defaultSecretAccountKey: // for compatibility with built-in blobfuse plugin
491
                                        accountKey = v
492
                                case accountSasTokenField:
493
                                        accountSasToken = v
494
                                case msiSecretField:
495
                                        msiSecret = v
496
                                case storageSPNClientSecretField:
497
                                        storageSPNClientSecret = v
498
                                }
499
                        }
500
                }
501
        }
502

503
        if containerName == "" {
504
                err = fmt.Errorf("could not find containerName from attributes(%v) or volumeID(%v)", attrib, volumeID)
505
        }
506

507
        if accountKey != "" {
508
                authEnv = append(authEnv, "AZURE_STORAGE_ACCESS_KEY="+accountKey)
509
        }
510

511
        if accountSasToken != "" {
512
                klog.V(2).Infof("accountSasToken is not empty, use it to access storage account(%s), container(%s)", accountName, containerName)
513
                authEnv = append(authEnv, "AZURE_STORAGE_SAS_TOKEN="+accountSasToken)
514
        }
515

516
        if msiSecret != "" {
517
                klog.V(2).Infof("msiSecret is not empty, use it to access storage account(%s), container(%s)", accountName, containerName)
518
                authEnv = append(authEnv, "MSI_SECRET="+msiSecret)
519
        }
520

521
        if storageSPNClientSecret != "" {
522
                klog.V(2).Infof("storageSPNClientSecret is not empty, use it to access storage account(%s), container(%s)", accountName, containerName)
523
                authEnv = append(authEnv, "AZURE_STORAGE_SPN_CLIENT_SECRET="+storageSPNClientSecret)
524
        }
525

526
        return rgName, accountName, accountKey, containerName, authEnv, err
527
}
528

529
// GetStorageAccountAndContainer get storage account and container info
530
// returns <accountName, accountKey, accountSasToken, containerName>
531
// only for e2e testing
532
func (d *Driver) GetStorageAccountAndContainer(ctx context.Context, volumeID string, attrib, secrets map[string]string) (string, string, string, string, error) {
533
        var (
534
                subsID                string
535
                accountName           string
536
                accountKey            string
537
                accountSasToken       string
538
                containerName         string
539
                keyVaultURL           string
540
                keyVaultSecretName    string
541
                keyVaultSecretVersion string
542
                err                   error
543
        )
544

545
        for k, v := range attrib {
546
                switch strings.ToLower(k) {
547
                case subscriptionIDField:
548
                        subsID = v
549
                case containerNameField:
550
                        containerName = v
551
                case keyVaultURLField:
552
                        keyVaultURL = v
553
                case keyVaultSecretNameField:
554
                        keyVaultSecretName = v
555
                case keyVaultSecretVersionField:
556
                        keyVaultSecretVersion = v
557
                case storageAccountField:
558
                        accountName = v
559
                case storageAccountNameField: // for compatibility
560
                        accountName = v
561
                }
562
        }
563

564
        // 1. If keyVaultURL is not nil, preferentially use the key stored in key vault.
565
        // 2. Then if secrets map is not nil, use the key stored in the secrets map.
566
        // 3. Finally if both keyVaultURL and secrets map are nil, get the key from Azure.
567
        if keyVaultURL != "" {
568
                key, err := d.getKeyVaultSecretContent(ctx, keyVaultURL, keyVaultSecretName, keyVaultSecretVersion)
569
                if err != nil {
570
                        return "", "", "", "", err
571
                }
572
                if isSASToken(key) {
573
                        accountSasToken = key
574
                } else {
575
                        accountKey = key
576
                }
577
        } else {
578
                if len(secrets) == 0 {
579
                        var rgName string
580
                        rgName, accountName, containerName, _, _, err = GetContainerInfo(volumeID)
581
                        if err != nil {
582
                                return "", "", "", "", err
583
                        }
584

585
                        if rgName == "" {
586
                                rgName = d.cloud.ResourceGroup
587
                        }
588

589
                        accountKey, err = d.cloud.GetStorageAccesskey(ctx, subsID, accountName, rgName)
590
                        if err != nil {
591
                                return "", "", "", "", fmt.Errorf("no key for storage account(%s) under resource group(%s), err %w", accountName, rgName, err)
592
                        }
593
                }
594
        }
595

596
        if containerName == "" {
597
                return "", "", "", "", fmt.Errorf("could not find containerName from attributes(%v) or volumeID(%v)", attrib, volumeID)
598
        }
599

600
        return accountName, accountKey, accountSasToken, containerName, nil
601
}
602

603
func IsCorruptedDir(dir string) bool {
604
        _, pathErr := mount.PathExists(dir)
605
        return pathErr != nil && mount.IsCorruptedMnt(pathErr)
606
}
607

608
func isRetriableError(err error) bool {
609
        if err != nil {
610
                for _, v := range retriableErrors {
611
                        if strings.Contains(strings.ToLower(err.Error()), strings.ToLower(v)) {
612
                                return true
613
                        }
614
                }
615
        }
616
        return false
617
}
618

619
func isSupportedProtocol(protocol string) bool {
620
        if protocol == "" {
621
                return true
622
        }
623
        for _, v := range supportedProtocolList {
624
                if protocol == v {
625
                        return true
626
                }
627
        }
628
        return false
629
}
630

631
func isSupportedAccessTier(accessTier string) bool {
632
        if accessTier == "" {
633
                return true
634
        }
635
        for _, tier := range storage.PossibleAccessTierValues() {
636
                if accessTier == string(tier) {
637
                        return true
638
                }
639
        }
640
        return false
641
}
642

643
// container names can contain only lowercase letters, numbers, and hyphens,
644
// and must begin and end with a letter or a number
645
func isSupportedContainerNamePrefix(prefix string) bool {
646
        if prefix == "" {
647
                return true
648
        }
649
        if len(prefix) > 20 {
650
                return false
651
        }
652
        if prefix[0] == '-' {
653
                return false
654
        }
655
        for _, v := range prefix {
656
                if v != '-' && (v < '0' || v > '9') && (v < 'a' || v > 'z') {
657
                        return false
658
                }
659
        }
660
        return true
661
}
662

663
// get storage account from secrets map
664
func getStorageAccount(secrets map[string]string) (string, string, error) {
665
        if secrets == nil {
666
                return "", "", fmt.Errorf("unexpected: getStorageAccount secrets is nil")
667
        }
668

669
        var accountName, accountKey string
670
        for k, v := range secrets {
671
                v = strings.TrimSpace(v)
672
                switch strings.ToLower(k) {
673
                case accountNameField:
674
                        accountName = v
675
                case defaultSecretAccountName: // for compatibility with built-in azurefile plugin
676
                        accountName = v
677
                case accountKeyField:
678
                        accountKey = v
679
                case defaultSecretAccountKey: // for compatibility with built-in azurefile plugin
680
                        accountKey = v
681
                }
682
        }
683

684
        if accountName == "" {
685
                return accountName, accountKey, fmt.Errorf("could not find %s or %s field secrets(%v)", accountNameField, defaultSecretAccountName, secrets)
686
        }
687
        if accountKey == "" {
688
                return accountName, accountKey, fmt.Errorf("could not find %s or %s field in secrets(%v)", accountKeyField, defaultSecretAccountKey, secrets)
689
        }
690

691
        accountName = strings.TrimSpace(accountName)
692
        klog.V(4).Infof("got storage account(%s) from secret", accountName)
693
        return accountName, accountKey, nil
694
}
695

696
func getContainerReference(containerName string, secrets map[string]string, env az.Environment) (*azstorage.Container, error) {
697
        accountName, accountKey, rerr := getStorageAccount(secrets)
698
        if rerr != nil {
699
                return nil, rerr
700
        }
701
        client, err := azstorage.NewBasicClientOnSovereignCloud(accountName, accountKey, env)
702
        if err != nil {
703
                return nil, err
704
        }
705
        blobClient := client.GetBlobService()
706
        container := blobClient.GetContainerReference(containerName)
707
        if container == nil {
708
                return nil, fmt.Errorf("ContainerReference of %s is nil", containerName)
709
        }
710
        return container, nil
711
}
712

713
func setAzureCredentials(ctx context.Context, kubeClient kubernetes.Interface, accountName, accountKey, secretNamespace string) (string, error) {
714
        if kubeClient == nil {
715
                klog.Warningf("could not create secret: kubeClient is nil")
716
                return "", nil
717
        }
718
        if accountName == "" || accountKey == "" {
719
                return "", fmt.Errorf("the account info is not enough, accountName(%v), accountKey(%v)", accountName, accountKey)
720
        }
721
        secretName := fmt.Sprintf(secretNameTemplate, accountName)
722
        secret := &v1.Secret{
723
                ObjectMeta: metav1.ObjectMeta{
724
                        Namespace: secretNamespace,
725
                        Name:      secretName,
726
                },
727
                Data: map[string][]byte{
728
                        defaultSecretAccountName: []byte(accountName),
729
                        defaultSecretAccountKey:  []byte(accountKey),
730
                },
731
                Type: "Opaque",
732
        }
733
        _, err := kubeClient.CoreV1().Secrets(secretNamespace).Create(ctx, secret, metav1.CreateOptions{})
734
        if errors.IsAlreadyExists(err) {
735
                err = nil
736
        }
737
        if err != nil {
738
                return "", fmt.Errorf("couldn't create secret %w", err)
739
        }
740
        return secretName, err
741
}
742

743
// GetStorageAccesskey get Azure storage account key from
744
//  1. secrets (if not empty)
745
//  2. use k8s client identity to read from k8s secret
746
//  3. use cluster identity to get from storage account directly
747
func (d *Driver) GetStorageAccesskey(ctx context.Context, accountOptions *azure.AccountOptions, secrets map[string]string, secretName, secretNamespace string) (string, string, error) {
748
        if len(secrets) > 0 {
749
                return getStorageAccount(secrets)
750
        }
751

752
        // read from k8s secret first
753
        if secretName == "" {
754
                secretName = fmt.Sprintf(secretNameTemplate, accountOptions.Name)
755
        }
756
        _, accountKey, _, _, _, err := d.GetInfoFromSecret(ctx, secretName, secretNamespace) //nolint
757
        if err != nil {
758
                klog.V(2).Infof("could not get account(%s) key from secret(%s) namespace(%s), error: %v, use cluster identity to get account key instead", accountOptions.Name, secretName, secretNamespace, err)
759
                accountKey, err = d.cloud.GetStorageAccesskey(ctx, accountOptions.SubscriptionID, accountOptions.Name, accountOptions.ResourceGroup)
760
        }
761
        return accountOptions.Name, accountKey, err
762
}
763

764
// GetInfoFromSecret get info from k8s secret
765
// return <accountName, accountKey, accountSasToken, msiSecret, spnClientSecret, error>
766
func (d *Driver) GetInfoFromSecret(ctx context.Context, secretName, secretNamespace string) (string, string, string, string, string, error) {
767
        if d.cloud.KubeClient == nil {
768
                return "", "", "", "", "", fmt.Errorf("could not get account key from secret(%s): KubeClient is nil", secretName)
769
        }
770

771
        secret, err := d.cloud.KubeClient.CoreV1().Secrets(secretNamespace).Get(ctx, secretName, metav1.GetOptions{})
772
        if err != nil {
773
                return "", "", "", "", "", fmt.Errorf("could not get secret(%v): %w", secretName, err)
774
        }
775

776
        accountName := strings.TrimSpace(string(secret.Data[defaultSecretAccountName][:]))
777
        accountKey := strings.TrimSpace(string(secret.Data[defaultSecretAccountKey][:]))
778
        accountSasToken := strings.TrimSpace(string(secret.Data[accountSasTokenField][:]))
779
        msiSecret := strings.TrimSpace(string(secret.Data[msiSecretField][:]))
780
        spnClientSecret := strings.TrimSpace(string(secret.Data[storageSPNClientSecretField][:]))
781

782
        klog.V(4).Infof("got storage account(%s) from secret", accountName)
783
        return accountName, accountKey, accountSasToken, msiSecret, spnClientSecret, nil
784
}
785

786
// getSubnetResourceID get default subnet resource ID from cloud provider config
787
func (d *Driver) getSubnetResourceID(vnetResourceGroup, vnetName, subnetName string) string {
788
        subsID := d.cloud.SubscriptionID
789
        if len(d.cloud.NetworkResourceSubscriptionID) > 0 {
790
                subsID = d.cloud.NetworkResourceSubscriptionID
791
        }
792

793
        if len(vnetResourceGroup) == 0 {
794
                vnetResourceGroup = d.cloud.ResourceGroup
795
                if len(d.cloud.VnetResourceGroup) > 0 {
796
                        vnetResourceGroup = d.cloud.VnetResourceGroup
797
                }
798
        }
799

800
        if len(vnetName) == 0 {
801
                vnetName = d.cloud.VnetName
802
        }
803

804
        if len(subnetName) == 0 {
805
                subnetName = d.cloud.SubnetName
806
        }
807
        return fmt.Sprintf(subnetTemplate, subsID, vnetResourceGroup, vnetName, subnetName)
808
}
809

810
func (d *Driver) useDataPlaneAPI(volumeID, accountName string) bool {
811
        cache, err := d.dataPlaneAPIVolCache.Get(volumeID, azcache.CacheReadTypeDefault)
812
        if err != nil {
813
                klog.Errorf("get(%s) from dataPlaneAPIVolCache failed with error: %v", volumeID, err)
814
        }
815
        if cache != nil {
816
                return true
817
        }
818
        cache, err = d.dataPlaneAPIVolCache.Get(accountName, azcache.CacheReadTypeDefault)
819
        if err != nil {
820
                klog.Errorf("get(%s) from dataPlaneAPIVolCache failed with error: %v", accountName, err)
821
        }
822
        if cache != nil {
823
                return true
824
        }
825
        return false
826
}
827

828
// appendDefaultMountOptions return mount options combined with mountOptions and defaultMountOptions
829
func appendDefaultMountOptions(mountOptions []string, tmpPath, containerName string) []string {
830
        var defaultMountOptions = map[string]string{
831
                "--pre-mount-validate": "true",
832
                "--use-https":          "true",
833
                "--tmp-path":           tmpPath,
834
                "--container-name":     containerName,
835
                // prevent billing charges on mounting
836
                "--cancel-list-on-mount-seconds": "10",
837
                // allow remounting using a non-empty tmp-path
838
                "--empty-dir-check": "false",
839
        }
840

841
        // stores the mount options already included in mountOptions
842
        included := make(map[string]bool)
843

844
        for _, mountOption := range mountOptions {
845
                for k := range defaultMountOptions {
846
                        if strings.HasPrefix(mountOption, k) {
847
                                included[k] = true
848
                        }
849
                }
850
        }
851

852
        allMountOptions := mountOptions
853

854
        for k, v := range defaultMountOptions {
855
                if _, isIncluded := included[k]; !isIncluded {
856
                        if v != "" {
857
                                allMountOptions = append(allMountOptions, fmt.Sprintf("%s=%s", k, v))
858
                        } else {
859
                                allMountOptions = append(allMountOptions, k)
860
                        }
861
                }
862
        }
863

864
        return allMountOptions
865
}
866

867
// chmodIfPermissionMismatch only perform chmod when permission mismatches
868
func chmodIfPermissionMismatch(targetPath string, mode os.FileMode) error {
869
        info, err := os.Lstat(targetPath)
870
        if err != nil {
871
                return err
872
        }
873
        perm := info.Mode() & os.ModePerm
874
        if perm != mode {
875
                klog.V(2).Infof("chmod targetPath(%s, mode:0%o) with permissions(0%o)", targetPath, info.Mode(), mode)
876
                if err := os.Chmod(targetPath, mode); err != nil {
877
                        return err
878
                }
879
        } else {
880
                klog.V(2).Infof("skip chmod on targetPath(%s) since mode is already 0%o)", targetPath, info.Mode())
881
        }
882
        return nil
883
}
884

885
func createStorageAccountSecret(account, key string) map[string]string {
886
        secret := make(map[string]string)
887
        secret[defaultSecretAccountName] = account
888
        secret[defaultSecretAccountKey] = key
889
        return secret
890
}
891

892
// setKeyValueInMap set key/value pair in map
893
// key in the map is case insensitive, if key already exists, overwrite existing value
894
func setKeyValueInMap(m map[string]string, key, value string) {
895
        if m == nil {
896
                return
897
        }
898
        for k := range m {
899
                if strings.EqualFold(k, key) {
900
                        m[k] = value
901
                        return
902
                }
903
        }
904
        m[key] = value
905
}
906

907
// replaceWithMap replace key with value for str
908
func replaceWithMap(str string, m map[string]string) string {
909
        for k, v := range m {
910
                if k != "" {
911
                        str = strings.ReplaceAll(str, k, v)
912
                }
913
        }
914
        return str
915
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc