#5658

Committed 14 Apr 2025 03:23PM UTC coverage: 79.137% (+0.004%) from 79.133%

Build # #5658

Build Type

push

Committed by

Commit Message

[DT-1493] Add mapping from institution to domains (#2478)

Coverage Stats

16 of 19 new or added lines in 4 files covered. (84.21%)

10291 of 13004 relevant lines covered (79.14%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.88

/src/main/java/org/broadinstitute/consent/http/cloudstore/GCSService.java

package org.broadinstitute.consent.http.cloudstore;

import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Bucket;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import com.google.inject.Inject;
import jakarta.ws.rs.NotFoundException;
import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import org.apache.commons.io.IOUtils;
import org.broadinstitute.consent.http.configurations.StoreConfiguration;
import org.broadinstitute.consent.http.util.ConsentLogger;
import org.broadinstitute.consent.http.util.gson.GsonUtil;

public class GCSService implements ConsentLogger {

  private static final Gson GSON = GsonUtil.gsonBuilderWithAdapters().create();

  private StoreConfiguration config;
  private Storage storage;

  public GCSService() {
  }

  @Inject
  public GCSService(StoreConfiguration config) {
    this.config = config;
    try {
      ServiceAccountCredentials credentials = ServiceAccountCredentials.
          fromStream(new FileInputStream(config.getPassword()));
      Storage storage = StorageOptions.newBuilder().
          setProjectId(credentials.getProjectId()).
          setCredentials(credentials).
          build().
          getService();
      this.setStorage(storage);
    } catch (Exception e) {
      logException("Exception initializing GCSService: " + e.getMessage(), e);
    }
  }

  @VisibleForTesting
  public void setStorage(Storage storage) {
    this.storage = storage;
  }

  @VisibleForTesting
  public void setConfig(StoreConfiguration config) {
    this.config = config;
  }

  /**
   * Get the root bucket configured for this environment. Returns a Bucket with all possible
   * metadata values.
   *
   * @return Bucket
   */
  public Bucket getRootBucketWithMetadata() {
    return storage.get(config.getBucket(),
        Storage.BucketGetOption.fields(Storage.BucketField.values()));
  }

  /**
   * Store an input stream as a Blob
   *
   * @param content   InputStream content
   * @param mediaType String media type
   * @param id        String UUID of the file
   * @return BlobId of the stored document
   * @throws IOException Exception when storing document
   */
  public BlobId storeDocument(InputStream content, String mediaType, UUID id)
      throws IOException {
    byte[] bytes = IOUtils.toByteArray(content);
    BlobId blobId = BlobId.of(config.getBucket(), id.toString());
    BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType(mediaType).build();
    Blob blob = storage.create(blobInfo, bytes);
    return blob.getBlobId();
  }


  /**
   * Delete a document by Blob Id Name
   *
   * @param blobIdName String value of the document blob id name
   * @return True if document was deleted, false otherwise.
   */
  public boolean deleteDocument(String blobIdName) {
    Optional<Blob> blobOptional = getBlobFromUrl(blobIdName);
    return blobOptional
        .map(blob -> storage.delete(blob.getBlobId()))
        .orElse(false);
  }

  /**
   * Retrieve a document by Blob Id Name
   *
   * @param blobIdName String value of the document blob id name
   * @return InputStream of the document
   * @throws NotFoundException Returned when no document found
   */
  public InputStream getDocument(String blobIdName) throws NotFoundException {
    Optional<Blob> blobOptional = getBlobFromUrl(blobIdName);
    if (blobOptional.isPresent()) {
      return new ByteArrayInputStream(blobOptional.get().getContent());
    } else {
      throw new NotFoundException("Document Not Found: " + blobIdName);
    }
  }

  public InputStream getDocument(BlobId blobId) throws NotFoundException {
    Optional<Blob> blobOptional = getBlobFromBlobId(blobId);
    if (blobOptional.isPresent()) {
      return new ByteArrayInputStream(blobOptional.get().getContent());
    } else {
      throw new NotFoundException("Document Not Found: " + blobId.toString());
    }
  }

  public Map<BlobId, InputStream> getDocuments(List<BlobId> blobIds) throws NotFoundException {
    Optional<List<Blob>> blobOptional = getBlobsFromBlobIds(blobIds);
    if (blobOptional.isPresent()) {
      List<Blob> blobs = blobOptional.get();
      Map<BlobId, InputStream> output = new HashMap<>();
      blobs.forEach((b) -> output.put(b.getBlobId(), new ByteArrayInputStream(b.getContent())));
      return output;
    } else {
      throw new NotFoundException("Document Not Found: " + blobIds.toString());
    }
  }

  /**
   * Read JSON file from GCS bucket
   *
   * @param blobIdName String value of the document blob id name
   * @param valueType  Class type of the object to be read
   * @return Object of the specified type
   */
  public <T> T readJsonFileFromBucket(String blobIdName, Class<T> valueType) throws IOException {
    try (InputStream is = getDocument(blobIdName)) {
      InputStreamReader reader = new InputStreamReader(is);
      return GSON.fromJson(reader, valueType);
    } catch (Exception e) {
      logException("Error reading JSON file from bucket: " + e.getMessage(), e);
      throw new IOException("Failed to read JSON file from bucket: " + blobIdName, e);
    }
  }

  /**
   * Find a blob in the current storage bucket.
   *
   * @param blobIdName String value of the document blob id name
   * @return Optional<Blob>
   */
  private Optional<Blob> getBlobFromUrl(String blobIdName) {
    Blob blob = storage.get(BlobId.of(config.getBucket(), blobIdName));
    return Optional.of(blob);
  }

  /**
   * Find a blob in the current storage bucket.
   *
   * @param blobId Bucket and blob id
   * @return Optional<Blob>
   */
  private Optional<Blob> getBlobFromBlobId(BlobId blobId) {
    Blob blob = storage.get(blobId);
    return Optional.of(blob);
  }

  private Optional<List<Blob>> getBlobsFromBlobIds(List<BlobId> blobIds) {
    List<Blob> blobs = storage.get(blobIds);
    return Optional.of(blobs);
  }
}

1	package org.broadinstitute.consent.http.cloudstore;
2
3	import com.google.auth.oauth2.ServiceAccountCredentials;
4	import com.google.cloud.storage.Blob;
5	import com.google.cloud.storage.BlobId;
6	import com.google.cloud.storage.BlobInfo;
7	import com.google.cloud.storage.Bucket;
8	import com.google.cloud.storage.Storage;
9	import com.google.cloud.storage.StorageOptions;
10	import com.google.common.annotations.VisibleForTesting;
11	import com.google.gson.Gson;
12	import com.google.inject.Inject;
13	import jakarta.ws.rs.NotFoundException;
14	import java.io.ByteArrayInputStream;
15	import java.io.FileInputStream;
16	import java.io.IOException;
17	import java.io.InputStream;
18	import java.io.InputStreamReader;
19	import java.util.HashMap;
20	import java.util.List;
21	import java.util.Map;
22	import java.util.Optional;
23	import java.util.UUID;
24	import org.apache.commons.io.IOUtils;
25	import org.broadinstitute.consent.http.configurations.StoreConfiguration;
26	import org.broadinstitute.consent.http.util.ConsentLogger;
27	import org.broadinstitute.consent.http.util.gson.GsonUtil;
28
29	public class GCSService implements ConsentLogger {
30
31	private static final Gson GSON = GsonUtil.gsonBuilderWithAdapters().create();	1✔
32
33	private StoreConfiguration config;
34	private Storage storage;
35
36	public GCSService() {	1✔
37	}	1✔
38
39	@Inject
40	public GCSService(StoreConfiguration config) {	1✔
41	this.config = config;	1✔
42	try {
43	ServiceAccountCredentials credentials = ServiceAccountCredentials.	1✔
44	fromStream(new FileInputStream(config.getPassword()));	×
45	Storage storage = StorageOptions.newBuilder().	×
46	setProjectId(credentials.getProjectId()).	×
47	setCredentials(credentials).	×
48	build().	×
49	getService();	×
50	this.setStorage(storage);	×
51	} catch (Exception e) {	1✔
52	logException("Exception initializing GCSService: " + e.getMessage(), e);	1✔
53	}	×
54	}	1✔
55
56	@VisibleForTesting
57	public void setStorage(Storage storage) {
58	this.storage = storage;	1✔
59	}	1✔
60
61	@VisibleForTesting
62	public void setConfig(StoreConfiguration config) {
63	this.config = config;	1✔
64	}	1✔
65
66	/**
67	* Get the root bucket configured for this environment. Returns a Bucket with all possible
68	* metadata values.
69	*
70	* @return Bucket
71	*/
72	public Bucket getRootBucketWithMetadata() {
73	return storage.get(config.getBucket(),	×
74	Storage.BucketGetOption.fields(Storage.BucketField.values()));	×
75	}
76
77	/**
78	* Store an input stream as a Blob
79	*
80	* @param content InputStream content
81	* @param mediaType String media type
82	* @param id String UUID of the file
83	* @return BlobId of the stored document
84	* @throws IOException Exception when storing document
85	*/
86	public BlobId storeDocument(InputStream content, String mediaType, UUID id)
87	throws IOException {
88	byte[] bytes = IOUtils.toByteArray(content);	1✔
89	BlobId blobId = BlobId.of(config.getBucket(), id.toString());	1✔
90	BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType(mediaType).build();	1✔
91	Blob blob = storage.create(blobInfo, bytes);	1✔
92	return blob.getBlobId();	1✔
93	}
94
95
96	/**
97	* Delete a document by Blob Id Name
98	*
99	* @param blobIdName String value of the document blob id name
100	* @return True if document was deleted, false otherwise.
101	*/
102	public boolean deleteDocument(String blobIdName) {
103	Optional<Blob> blobOptional = getBlobFromUrl(blobIdName);	1✔
104	return blobOptional	1✔
105	.map(blob -> storage.delete(blob.getBlobId()))	1✔
106	.orElse(false);	1✔
107	}
108
109	/**
110	* Retrieve a document by Blob Id Name
111	*
112	* @param blobIdName String value of the document blob id name
113	* @return InputStream of the document
114	* @throws NotFoundException Returned when no document found
115	*/
116	public InputStream getDocument(String blobIdName) throws NotFoundException {
117	Optional<Blob> blobOptional = getBlobFromUrl(blobIdName);	1✔
118	if (blobOptional.isPresent()) {	1✔
119	return new ByteArrayInputStream(blobOptional.get().getContent());	1✔
120	} else {
121	throw new NotFoundException("Document Not Found: " + blobIdName);	×
122	}
123	}
124
125	public InputStream getDocument(BlobId blobId) throws NotFoundException {
126	Optional<Blob> blobOptional = getBlobFromBlobId(blobId);	1✔
127	if (blobOptional.isPresent()) {	1✔
128	return new ByteArrayInputStream(blobOptional.get().getContent());	1✔
129	} else {
130	throw new NotFoundException("Document Not Found: " + blobId.toString());	×
131	}
132	}
133
134	public Map<BlobId, InputStream> getDocuments(List<BlobId> blobIds) throws NotFoundException {
135	Optional<List<Blob>> blobOptional = getBlobsFromBlobIds(blobIds);	1✔
136	if (blobOptional.isPresent()) {	1✔
137	List<Blob> blobs = blobOptional.get();	1✔
138	Map<BlobId, InputStream> output = new HashMap<>();	1✔
139	blobs.forEach((b) -> output.put(b.getBlobId(), new ByteArrayInputStream(b.getContent())));	1✔
140	return output;	1✔
141	} else {
142	throw new NotFoundException("Document Not Found: " + blobIds.toString());	×
143	}
144	}
145
146	/**
147	* Read JSON file from GCS bucket
148	*
149	* @param blobIdName String value of the document blob id name
150	* @param valueType Class type of the object to be read
151	* @return Object of the specified type
152	*/
153	public <T> T readJsonFileFromBucket(String blobIdName, Class<T> valueType) throws IOException {
154	try (InputStream is = getDocument(blobIdName)) {	1✔
155	InputStreamReader reader = new InputStreamReader(is);	1✔
156	return GSON.fromJson(reader, valueType);	1✔
NEW 157	} catch (Exception e) {	×
NEW 158	logException("Error reading JSON file from bucket: " + e.getMessage(), e);	×
NEW 159	throw new IOException("Failed to read JSON file from bucket: " + blobIdName, e);	×
160	}
161	}
162
163	/**
164	* Find a blob in the current storage bucket.
165	*
166	* @param blobIdName String value of the document blob id name
167	* @return Optional<Blob>
168	*/
169	private Optional<Blob> getBlobFromUrl(String blobIdName) {
170	Blob blob = storage.get(BlobId.of(config.getBucket(), blobIdName));	1✔
171	return Optional.of(blob);	1✔
172	}
173
174	/**
175	* Find a blob in the current storage bucket.
176	*
177	* @param blobId Bucket and blob id
178	* @return Optional<Blob>
179	*/
180	private Optional<Blob> getBlobFromBlobId(BlobId blobId) {
181	Blob blob = storage.get(blobId);	1✔
182	return Optional.of(blob);	1✔
183	}
184
185	private Optional<List<Blob>> getBlobsFromBlobIds(List<BlobId> blobIds) {
186	List<Blob> blobs = storage.get(blobIds);	1✔
187	return Optional.of(blobs);	1✔
188	}
189	}

DataBiosphere / consent / #5658

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous