7961349851

Committed 19 Feb 2024 02:47PM CUT coverage: 62.707%. First build

Build # 7961349851

Build Type

Pull #14164

github

Committed by

web-flow

Commit Message

Merge 3cff1f821 into 033847426

Pull Request Pull Request #14164: release/530-release-candidate

Run Details

8964 of 14295 relevant lines covered (62.71%)

0.63 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/scala/com/johnsnowlabs/client/CloudManager.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.johnsnowlabs.client

import com.johnsnowlabs.client.aws.AWSClient
import com.johnsnowlabs.client.azure.AzureClient
import com.johnsnowlabs.client.gcp.GCPClient
import com.johnsnowlabs.client.util.CloudHelper
import com.johnsnowlabs.nlp.util.io.CloudStorageType
import com.johnsnowlabs.util.{ConfigHelper, ConfigLoader}
import org.slf4j.{Logger, LoggerFactory}

class CloudManager(parameters: Map[String, String] = Map.empty) {

  private val logger: Logger = LoggerFactory.getLogger(this.getClass.toString)

  def getClientInstance(uri: String): CloudClient = {
    CloudHelper.cloudType(uri) match {
      case CloudStorageType.S3 =>
        new AWSClient(parameters)
      case CloudStorageType.GCP => {
        new GCPClient(parameters)
      }
      case CloudStorageType.Azure => {
        val storageAccountName = CloudHelper.getAccountNameFromAzureBlobURI(uri)
        new AzureClient(Map("storageAccountName" -> storageAccountName))
      }
      case _ =>
        throw new IllegalArgumentException(s"Unsupported URI scheme: $uri")
    }
  }

  def getClientInstanceFromConfigurationParams(uri: String): CloudClient = {
    CloudHelper.cloudType(uri) match {
      case CloudStorageType.S3 =>
        val (accessKey, secretKey, sessionToken) = ConfigHelper.getHadoopS3Config
        val region = ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalRegion)
        val isS3Defined =
          accessKey != null && secretKey != null && sessionToken != null && region.nonEmpty

        val configParameters = if (isS3Defined) {
          Map(
            "accessKeyId" -> accessKey,
            "secretAccessKey" -> secretKey,
            "sessionToken" -> sessionToken,
            "region" -> region)
        } else {
          if (accessKey != null || secretKey != null || sessionToken != null)
            logger.info(
              "Not all configs set for private S3 access. Defaulting to public downloader.")
          Map("credentialsType" -> "public")
        }
        new AWSClient(configParameters)
      case CloudStorageType.GCP => {
        val projectId = ConfigLoader.getConfigStringValue(ConfigHelper.gcpProjectId)
        val configParameters = Map("projectId" -> projectId)

        new GCPClient(configParameters)
      }
      case CloudStorageType.Azure => {
        val storageAccountName = CloudHelper.getAccountNameFromAzureBlobURI(uri)
        val configParameters = Map("storageAccountName" -> storageAccountName)

        new AzureClient(configParameters)
      }
      case _ =>
        throw new IllegalArgumentException(s"Unsupported URI scheme: $uri")
    }
  }

}

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16	package com.johnsnowlabs.client
17
18	import com.johnsnowlabs.client.aws.AWSClient
19	import com.johnsnowlabs.client.azure.AzureClient
20	import com.johnsnowlabs.client.gcp.GCPClient
21	import com.johnsnowlabs.client.util.CloudHelper
22	import com.johnsnowlabs.nlp.util.io.CloudStorageType
23	import com.johnsnowlabs.util.{ConfigHelper, ConfigLoader}
24	import org.slf4j.{Logger, LoggerFactory}
25
26	class CloudManager(parameters: Map[String, String] = Map.empty) {
27
28	private val logger: Logger = LoggerFactory.getLogger(this.getClass.toString)	×
29
30	def getClientInstance(uri: String): CloudClient = {
31	CloudHelper.cloudType(uri) match {	×
32	case CloudStorageType.S3 =>
33	new AWSClient(parameters)	×
34	case CloudStorageType.GCP => {
35	new GCPClient(parameters)	×
36	}
37	case CloudStorageType.Azure => {
38	val storageAccountName = CloudHelper.getAccountNameFromAzureBlobURI(uri)	×
39	new AzureClient(Map("storageAccountName" -> storageAccountName))	×
40	}
41	case _ =>
42	throw new IllegalArgumentException(s"Unsupported URI scheme: $uri")	×
43	}
44	}
45
46	def getClientInstanceFromConfigurationParams(uri: String): CloudClient = {
47	CloudHelper.cloudType(uri) match {	×
48	case CloudStorageType.S3 =>
49	val (accessKey, secretKey, sessionToken) = ConfigHelper.getHadoopS3Config	×
50	val region = ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalRegion)	×
51	val isS3Defined =
52	accessKey != null && secretKey != null && sessionToken != null && region.nonEmpty	×
53
54	val configParameters = if (isS3Defined) {
55	Map(	×
56	"accessKeyId" -> accessKey,	×
57	"secretAccessKey" -> secretKey,	×
58	"sessionToken" -> sessionToken,	×
59	"region" -> region)	×
60	} else {	×
61	if (accessKey != null \|\| secretKey != null \|\| sessionToken != null)	×
62	logger.info(	×
63	"Not all configs set for private S3 access. Defaulting to public downloader.")
64	Map("credentialsType" -> "public")	×
65	}
66	new AWSClient(configParameters)	×
67	case CloudStorageType.GCP => {
68	val projectId = ConfigLoader.getConfigStringValue(ConfigHelper.gcpProjectId)	×
69	val configParameters = Map("projectId" -> projectId)	×
70
71	new GCPClient(configParameters)	×
72	}
73	case CloudStorageType.Azure => {
74	val storageAccountName = CloudHelper.getAccountNameFromAzureBlobURI(uri)	×
75	val configParameters = Map("storageAccountName" -> storageAccountName)	×
76
77	new AzureClient(configParameters)	×
78	}
79	case _ =>
80	throw new IllegalArgumentException(s"Unsupported URI scheme: $uri")	×
81	}
82	}
83
84	}

JohnSnowLabs / spark-nlp / 7961349851

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous