7961349851

Committed 19 Feb 2024 02:47PM CUT coverage: 62.707%. First build

Build # 7961349851

Build Type

Pull #14164

github

Committed by

web-flow

Commit Message

Merge 3cff1f821 into 033847426

Pull Request Pull Request #14164: release/530-release-candidate

Run Details

8964 of 14295 relevant lines covered (62.71%)

0.63 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/scala/com/johnsnowlabs/client/aws/AWSClient.scala

/*
 * Copyright 2017-2023 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.johnsnowlabs.client.aws

import com.johnsnowlabs.client.{CloudClient, CloudStorage}
import com.johnsnowlabs.util.{ConfigHelper, ConfigLoader}

class AWSClient(parameters: Map[String, String] = Map.empty) extends CloudClient {

  private lazy val awsStorageConnection = cloudConnect()

  override protected def cloudConnect(): CloudStorage = {
    val accessKeyId = parameters.getOrElse(
      "accessKeyId",
      ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalAccessKeyId))
    val secretAccessKey = parameters.getOrElse(
      "secretAccessKey",
      ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalSecretAccessKey))
    val sessionToken = parameters.getOrElse(
      "sessionToken",
      ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalSessionToken))
    val awsProfile = parameters.getOrElse(
      "awsProfile",
      ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalProfileName))
    val region = parameters.getOrElse(
      "region",
      ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalRegion))
    val credentialsType = parameters.getOrElse("credentialsType", "private")

    if (parameters.isEmpty) {
      getAWSClientInstanceFromHadoopConfig
    } else {
      new AWSGateway(
        accessKeyId,
        secretAccessKey,
        sessionToken,
        awsProfile,
        region,
        credentialsType)
    }
  }

  private def getAWSClientInstanceFromHadoopConfig: AWSGateway = {
    var (accessKeyId, secretKey, sessionToken) = ConfigHelper.getHadoopS3Config
    if (accessKeyId == null) accessKeyId = ""
    if (secretKey == null) secretKey = ""
    if (sessionToken == null) sessionToken = ""

    if (accessKeyId == "" && secretKey == "") {
      throw new IllegalAccessException(
        "Empty access.key and secret.key hadoop configuration and parameters.")
    }
    val awsDestinationGateway = new AWSGateway(accessKeyId, secretKey, sessionToken)
    awsDestinationGateway
  }

  override def doesBucketPathExist(bucketName: String, filePath: String): Boolean = {
    awsStorageConnection.doesBucketPathExist(bucketName, filePath)
  }

  override def copyInputStreamToBucket(
      bucketName: String,
      filePath: String,
      sourceFilePath: String): Unit = {
    awsStorageConnection.copyInputStreamToBucket(bucketName, filePath, sourceFilePath)
  }

  override def downloadFilesFromBucketToDirectory(
      bucketName: String,
      filePath: String,
      directoryPath: String,
      isIndex: Boolean): Unit = {
    awsStorageConnection.downloadFilesFromBucketToDirectory(
      bucketName,
      filePath,
      directoryPath,
      isIndex)
  }

}

1	/*
2	* Copyright 2017-2023 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16	package com.johnsnowlabs.client.aws
17
18	import com.johnsnowlabs.client.{CloudClient, CloudStorage}
19	import com.johnsnowlabs.util.{ConfigHelper, ConfigLoader}
20
21	class AWSClient(parameters: Map[String, String] = Map.empty) extends CloudClient {
22
23	private lazy val awsStorageConnection = cloudConnect()
24
25	override protected def cloudConnect(): CloudStorage = {
26	val accessKeyId = parameters.getOrElse(	×
27	"accessKeyId",	×
28	ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalAccessKeyId))	×
29	val secretAccessKey = parameters.getOrElse(	×
30	"secretAccessKey",	×
31	ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalSecretAccessKey))	×
32	val sessionToken = parameters.getOrElse(	×
33	"sessionToken",	×
34	ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalSessionToken))	×
35	val awsProfile = parameters.getOrElse(	×
36	"awsProfile",	×
37	ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalProfileName))	×
38	val region = parameters.getOrElse(	×
39	"region",	×
40	ConfigLoader.getConfigStringValue(ConfigHelper.awsExternalRegion))	×
41	val credentialsType = parameters.getOrElse("credentialsType", "private")	×
42
43	if (parameters.isEmpty) {	×
44	getAWSClientInstanceFromHadoopConfig	×
45	} else {
46	new AWSGateway(	×
47	accessKeyId,
48	secretAccessKey,
49	sessionToken,
50	awsProfile,
51	region,
52	credentialsType)
53	}
54	}
55
56	private def getAWSClientInstanceFromHadoopConfig: AWSGateway = {
57	var (accessKeyId, secretKey, sessionToken) = ConfigHelper.getHadoopS3Config	×
58	if (accessKeyId == null) accessKeyId = ""	×
59	if (secretKey == null) secretKey = ""	×
60	if (sessionToken == null) sessionToken = ""	×
61
62	if (accessKeyId == "" && secretKey == "") {	×
63	throw new IllegalAccessException(	×
64	"Empty access.key and secret.key hadoop configuration and parameters.")
65	}
66	val awsDestinationGateway = new AWSGateway(accessKeyId, secretKey, sessionToken)	×
67	awsDestinationGateway
68	}
69
70	override def doesBucketPathExist(bucketName: String, filePath: String): Boolean = {
71	awsStorageConnection.doesBucketPathExist(bucketName, filePath)	×
72	}
73
74	override def copyInputStreamToBucket(
75	bucketName: String,
76	filePath: String,
77	sourceFilePath: String): Unit = {
78	awsStorageConnection.copyInputStreamToBucket(bucketName, filePath, sourceFilePath)	×
79	}
80
81	override def downloadFilesFromBucketToDirectory(
82	bucketName: String,
83	filePath: String,
84	directoryPath: String,
85	isIndex: Boolean): Unit = {
86	awsStorageConnection.downloadFilesFromBucketToDirectory(	×
87	bucketName,
88	filePath,
89	directoryPath,
90	isIndex)
91	}
92
93	}

JohnSnowLabs / spark-nlp / 7961349851

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous