4413868535

Build Type

push

github

Committed by GitHub

Commit Message

SPARKNLP-746: Handle empty validation sets (#13615)

Run Details

8597 of 12936 relevant lines covered (66.46%)

0.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/scala/com/johnsnowlabs/client/gcp/GCPGateway.scala

package com.johnsnowlabs.client.gcp

import com.google.cloud.storage.{BlobId, BlobInfo, Storage, StorageOptions}
import com.johnsnowlabs.util.{ConfigHelper, ConfigLoader}
import org.sparkproject.guava.collect.Iterables

import java.io.InputStream

class GCPGateway(
    projectId: String = ConfigLoader.getConfigStringValue(ConfigHelper.gcpProjectId)) {

  private lazy val storageClient: Storage = {
    if (projectId == null || projectId.isEmpty) {
      throw new UnsupportedOperationException(
        "projectId argument is mandatory to create GCP Storage client.")
    }

    StorageOptions.newBuilder.setProjectId(projectId).build.getService
  }

  def copyFileToGCPStorage(
      bucket: String,
      destinationGCPStoragePath: String,
      inputStream: InputStream): Unit = {

    val blobId = BlobId.of(bucket, destinationGCPStoragePath)
    val blobInfo = BlobInfo.newBuilder(blobId).build

    storageClient.createFrom(blobInfo, inputStream)
  }

  def doesFolderExist(bucket: String, prefix: String): Boolean = {
    val storage = StorageOptions.newBuilder.setProjectId(projectId).build.getService
    val blobs = storage.list(
      bucket,
      Storage.BlobListOption.prefix(prefix),
      Storage.BlobListOption.currentDirectory)

    val blobsSize = Iterables.size(blobs.iterateAll())
    blobsSize > 0
  }

}

1	package com.johnsnowlabs.client.gcp
2
3	import com.google.cloud.storage.{BlobId, BlobInfo, Storage, StorageOptions}
4	import com.johnsnowlabs.util.{ConfigHelper, ConfigLoader}
5	import org.sparkproject.guava.collect.Iterables
6
7	import java.io.InputStream
8
9	class GCPGateway(
10	projectId: String = ConfigLoader.getConfigStringValue(ConfigHelper.gcpProjectId)) {
11
12	private lazy val storageClient: Storage = {
13	if (projectId == null \|\| projectId.isEmpty) {
14	throw new UnsupportedOperationException(
15	"projectId argument is mandatory to create GCP Storage client.")
16	}
17
18	StorageOptions.newBuilder.setProjectId(projectId).build.getService
19	}
20
21	def copyFileToGCPStorage(
22	bucket: String,
23	destinationGCPStoragePath: String,
24	inputStream: InputStream): Unit = {
25
26	val blobId = BlobId.of(bucket, destinationGCPStoragePath)	×
27	val blobInfo = BlobInfo.newBuilder(blobId).build	×
28
29	storageClient.createFrom(blobInfo, inputStream)	×
30	}
31
32	def doesFolderExist(bucket: String, prefix: String): Boolean = {
33	val storage = StorageOptions.newBuilder.setProjectId(projectId).build.getService	×
34	val blobs = storage.list(	×
35	bucket,
36	Storage.BlobListOption.prefix(prefix),	×
37	Storage.BlobListOption.currentDirectory)	×
38
39	val blobsSize = Iterables.size(blobs.iterateAll())	×
40	blobsSize > 0	×
41	}
42
43	}

JohnSnowLabs / spark-nlp / 4413868535

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous