• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

DataBiosphere / consent / #4594

26 Sep 2023 12:24PM UTC coverage: 76.158% (+0.1%) from 76.047%
#4594

push

web-flow
[DUOS-2651] Additional dataset index terms (#2148)

34 of 34 new or added lines in 6 files covered. (100.0%)

9385 of 12323 relevant lines covered (76.16%)

0.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.19
/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchService.java
1
package org.broadinstitute.consent.http.service;
2

3
import com.google.gson.JsonArray;
4
import jakarta.ws.rs.HttpMethod;
5
import jakarta.ws.rs.core.Response;
6
import java.io.IOException;
7
import java.nio.charset.StandardCharsets;
8
import java.util.ArrayList;
9
import java.util.Collection;
10
import java.util.List;
11
import java.util.Map;
12
import java.util.Objects;
13
import java.util.Optional;
14
import org.apache.http.entity.ContentType;
15
import org.apache.http.nio.entity.NStringEntity;
16
import org.broadinstitute.consent.http.configurations.ElasticSearchConfiguration;
17
import org.broadinstitute.consent.http.db.DacDAO;
18
import org.broadinstitute.consent.http.db.DataAccessRequestDAO;
19
import org.broadinstitute.consent.http.db.InstitutionDAO;
20
import org.broadinstitute.consent.http.db.UserDAO;
21
import org.broadinstitute.consent.http.models.Dac;
22
import org.broadinstitute.consent.http.models.Dataset;
23
import org.broadinstitute.consent.http.models.DatasetProperty;
24
import org.broadinstitute.consent.http.models.Institution;
25
import org.broadinstitute.consent.http.models.Study;
26
import org.broadinstitute.consent.http.models.StudyProperty;
27
import org.broadinstitute.consent.http.models.User;
28
import org.broadinstitute.consent.http.models.elastic_search.DacTerm;
29
import org.broadinstitute.consent.http.models.elastic_search.DatasetTerm;
30
import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchHits;
31
import org.broadinstitute.consent.http.models.elastic_search.InstitutionTerm;
32
import org.broadinstitute.consent.http.models.elastic_search.StudyTerm;
33
import org.broadinstitute.consent.http.models.elastic_search.UserTerm;
34
import org.broadinstitute.consent.http.util.ConsentLogger;
35
import org.broadinstitute.consent.http.util.gson.GsonUtil;
36
import org.elasticsearch.client.Request;
37
import org.elasticsearch.client.RestClient;
38

39
public class ElasticSearchService implements ConsentLogger {
40

41
  private final RestClient esClient;
42
  private final ElasticSearchConfiguration esConfig;
43
  private final DacDAO dacDAO;
44
  private final DataAccessRequestDAO dataAccessRequestDAO;
45
  private final UserDAO userDAO;
46
  private final OntologyService ontologyService;
47
  private final InstitutionDAO institutionDAO;
48

49
  public ElasticSearchService(
50
      RestClient esClient,
51
      ElasticSearchConfiguration esConfig,
52
      DacDAO dacDAO,
53
      DataAccessRequestDAO dataAccessRequestDAO,
54
      UserDAO userDao,
55
      OntologyService ontologyService,
56
      InstitutionDAO institutionDAO) {
1✔
57
    this.esClient = esClient;
1✔
58
    this.esConfig = esConfig;
1✔
59
    this.dacDAO = dacDAO;
1✔
60
    this.dataAccessRequestDAO = dataAccessRequestDAO;
1✔
61
    this.userDAO = userDao;
1✔
62
    this.ontologyService = ontologyService;
1✔
63
    this.institutionDAO = institutionDAO;
1✔
64
  }
1✔
65

66

67
  private static final String bulkHeader = """
68
      { "index": {"_type": "dataset", "_id": "%d"} }
69
      """;
70

71
  private static final String deleteQuery = """
72
      { "query": { "bool": { "must": [ { "match": { "_type": "dataset" } }, { "match": { "_id": "%d" } } ] } } }
73
      """;
74

75
  private Response performRequest(Request request) throws IOException {
76
    var response = esClient.performRequest(request);
1✔
77
    var status = response.getStatusLine().getStatusCode();
1✔
78
    if (status != 200) {
1✔
79
      throw new IOException("Invalid Elasticsearch query");
1✔
80
    }
81
    var body = new String(response.getEntity().getContent().readAllBytes(),
1✔
82
        StandardCharsets.UTF_8);
83
    return Response.status(status).entity(body).build();
1✔
84
  }
85

86
  public Response indexDatasetTerms(List<DatasetTerm> datasets) throws IOException {
87
    List<String> bulkApiCall = new ArrayList<>();
1✔
88

89
    datasets.forEach(dsTerm -> {
1✔
90
      bulkApiCall.add(bulkHeader.formatted(dsTerm.getDatasetId()));
1✔
91
      bulkApiCall.add(GsonUtil.getInstance().toJson(dsTerm) + "\n");
1✔
92
    });
1✔
93

94
    Request bulkRequest = new Request(
1✔
95
        HttpMethod.PUT,
96
        "/" + esConfig.getDatasetIndexName() + "/_bulk");
1✔
97

98
    bulkRequest.setEntity(new NStringEntity(
1✔
99
        String.join("", bulkApiCall) + "\n",
1✔
100
        ContentType.APPLICATION_JSON));
101

102
    return performRequest(bulkRequest);
1✔
103
  }
104

105
  public Response deleteIndex(Integer datasetId) throws IOException {
106
    Request deleteRequest = new Request(
×
107
        HttpMethod.POST,
108
        "/" + esConfig.getDatasetIndexName() + "/_delete_by_query");
×
109
    deleteRequest.setEntity(new NStringEntity(
×
110
        deleteQuery.formatted(datasetId),
×
111
        ContentType.APPLICATION_JSON));
112
    return performRequest(deleteRequest);
×
113
  }
114

115
  public boolean validateQuery(String query) throws IOException {
116
    // Remove `size` and `from` parameters from query, otherwise validation will fail
117
    var modifiedQuery = query
1✔
118
        .replaceAll("\"size\": ?\\d+,?", "")
1✔
119
        .replaceAll("\"from\": ?\\d+,?", "");
1✔
120

121
    Request validateRequest = new Request(
1✔
122
        HttpMethod.GET,
123
        "/" + esConfig.getDatasetIndexName() + "/_validate/query");
1✔
124
    validateRequest.setEntity(new NStringEntity(modifiedQuery, ContentType.APPLICATION_JSON));
1✔
125
    Response response = performRequest(validateRequest);
1✔
126

127
    var entity = response.getEntity().toString();
1✔
128
    var json = GsonUtil.getInstance().fromJson(entity, Map.class);
1✔
129

130
    return (boolean) json.get("valid");
1✔
131
  }
132

133
  public Response searchDatasets(String query) throws IOException {
134
    if (!validateQuery(query)) {
1✔
135
      throw new IOException("Invalid Elasticsearch query");
×
136
    }
137

138
    Request searchRequest = new Request(
1✔
139
        HttpMethod.GET,
140
        "/" + esConfig.getDatasetIndexName() + "/_search");
1✔
141
    searchRequest.setEntity(new NStringEntity(query, ContentType.APPLICATION_JSON));
1✔
142

143
    Response response = performRequest(searchRequest);
1✔
144

145
    var entity = response.getEntity().toString();
1✔
146
    var json = GsonUtil.getInstance().fromJson(entity, ElasticSearchHits.class);
1✔
147
    var hits = json.getHits();
1✔
148

149
    return Response.ok().entity(hits).build();
1✔
150
  }
151

152
  public StudyTerm toStudyTerm(Study study) {
153
    if (Objects.isNull(study)) {
1✔
154
      return null;
×
155
    }
156

157
    StudyTerm term = new StudyTerm();
1✔
158

159
    term.setDescription(study.getDescription());
1✔
160
    term.setStudyName(study.getName());
1✔
161
    term.setStudyId(study.getStudyId());
1✔
162
    term.setDataTypes(study.getDataTypes());
1✔
163
    term.setPiName(study.getPiName());
1✔
164
    term.setPublicVisibility(study.getPublicVisibility());
1✔
165

166
    findStudyProperty(
1✔
167
        study.getProperties(), "phenotypeIndication"
1✔
168
    ).ifPresent(
1✔
169
        prop -> term.setPhenotype(prop.getValue().toString())
1✔
170
    );
171

172
    findStudyProperty(
1✔
173
        study.getProperties(), "species"
1✔
174
    ).ifPresent(
1✔
175
        prop -> term.setSpecies(prop.getValue().toString())
1✔
176
    );
177

178
    findStudyProperty(
1✔
179
        study.getProperties(), "dataCustodianEmail"
1✔
180
    ).ifPresent(
1✔
181
        prop -> {
182
          JsonArray jsonArray = (JsonArray) prop.getValue();
1✔
183
          List<String> dataCustodianEmail = new ArrayList<>();
1✔
184
          jsonArray.forEach(email -> dataCustodianEmail.add(email.getAsString()));
1✔
185
          term.setDataCustodianEmail(dataCustodianEmail);
1✔
186
        }
1✔
187
    );
188

189
    if (Objects.nonNull(study.getCreateUserId())) {
1✔
190
      term.setDataSubmitterId(study.getCreateUserId());
1✔
191
      User user = userDAO.findUserById(study.getCreateUserId());
1✔
192
      if (Objects.nonNull(user)) {
1✔
193
        study.setCreateUserEmail(user.getEmail());
1✔
194
      }
195
    }
196

197
    if (Objects.nonNull(study.getCreateUserEmail())) {
1✔
198
      term.setDataSubmitterEmail(study.getCreateUserEmail());
1✔
199
    }
200

201
    return term;
1✔
202
  }
203

204
  public UserTerm toUserTerm(User user) {
205
    if (Objects.isNull(user)) {
1✔
206
      return null;
×
207
    }
208
    InstitutionTerm institution = (Objects.nonNull(user.getInstitutionId())) ?
1✔
209
      toInstitutionTerm(institutionDAO.findInstitutionById(user.getInstitutionId())) :
1✔
210
      null;
1✔
211
    return new UserTerm(user.getUserId(), user.getDisplayName(), institution);
1✔
212
  }
213

214
  public DacTerm toDacTerm(Dac dac) {
215
    if (Objects.isNull(dac)) {
1✔
216
      return null;
×
217
    }
218
    return new DacTerm(dac.getDacId(), dac.getName());
1✔
219
  }
220

221
  public InstitutionTerm toInstitutionTerm(Institution institution) {
222
    if (Objects.isNull(institution)) {
1✔
223
      return null;
1✔
224
    }
225
    return new InstitutionTerm(institution.getId(), institution.getName());
1✔
226
  }
227

228
  public Response indexDataset(Dataset dataset) throws IOException {
229
    return indexDatasetTerms(List.of(toDatasetTerm(dataset)));
×
230
  }
231

232
  public Response indexDatasets(List<Dataset> datasets) throws IOException {
233
    List<DatasetTerm> datasetTerms = datasets.stream().map(this::toDatasetTerm).toList();
×
234
    return indexDatasetTerms(datasetTerms);
×
235
  }
236

237
  public DatasetTerm toDatasetTerm(Dataset dataset) {
238
    if (Objects.isNull(dataset)) {
1✔
239
      return null;
×
240
    }
241

242
    DatasetTerm term = new DatasetTerm();
1✔
243

244
    term.setDatasetId(dataset.getDataSetId());
1✔
245
    Optional.ofNullable(dataset.getCreateUserId()).ifPresent(userId -> {
1✔
246
      User user = userDAO.findUserById(dataset.getCreateUserId());
1✔
247
      term.setCreateUserId(dataset.getCreateUserId());
1✔
248
      term.setCreateUserDisplayName(user.getDisplayName());
1✔
249
      term.setSubmitter(toUserTerm(user));
1✔
250
    });
1✔
251
    Optional.ofNullable(dataset.getUpdateUserId())
1✔
252
        .map(userDAO::findUserById)
1✔
253
        .map(this::toUserTerm)
1✔
254
        .ifPresent(term::setUpdateUser);
1✔
255
    term.setDatasetIdentifier(dataset.getDatasetIdentifier());
1✔
256
    term.setDatasetName(dataset.getName());
1✔
257

258
    if (Objects.nonNull(dataset.getStudy())) {
1✔
259
      term.setStudy(toStudyTerm(dataset.getStudy()));
1✔
260
    }
261

262
    Optional.ofNullable(dataset.getDacId()).ifPresent(dacId -> {
1✔
263
      Dac dac = dacDAO.findById(dataset.getDacId());
1✔
264
      term.setDacId(dataset.getDacId());
1✔
265
      term.setDacName(dac.getName());
1✔
266
      if (Objects.nonNull(dataset.getDacApproval())) {
1✔
267
        term.setDacApproval(dataset.getDacApproval());
1✔
268
      }
269
      term.setDac(toDacTerm(dac));
1✔
270
    });
1✔
271

272
    List<Integer> approvedUserIds =
1✔
273
        dataAccessRequestDAO.findAllUserIdsWithApprovedDARsByDatasetId(
1✔
274
            dataset.getDataSetId());
1✔
275

276
    if (!approvedUserIds.isEmpty()) {
1✔
277
      term.setApprovedUserIds(approvedUserIds);
1✔
278
    }
279

280
    if (Objects.nonNull(dataset.getDataUse())) {
1✔
281
      term.setDataUse(ontologyService.translateDataUseSummary(dataset.getDataUse()));
1✔
282
    }
283

284
    findDatasetProperty(
1✔
285
        dataset.getProperties(), "openAccess"
1✔
286
    ).ifPresent(
1✔
287
        datasetProperty -> term.setOpenAccess((Boolean) datasetProperty.getPropertyValue())
1✔
288
    );
289

290
    findDatasetProperty(
1✔
291
        dataset.getProperties(), "numberOfParticipants"
1✔
292
    ).ifPresent(
1✔
293
        datasetProperty -> term.setParticipantCount((Integer) datasetProperty.getPropertyValue())
1✔
294
    );
295

296
    findDatasetProperty(
1✔
297
        dataset.getProperties(), "url"
1✔
298
    ).ifPresent(
1✔
299
        datasetProperty -> term.setUrl(datasetProperty.getPropertyValueAsString())
1✔
300
    );
301

302
    findDatasetProperty(
1✔
303
        dataset.getProperties(), "dataLocation"
1✔
304
    ).ifPresent(
1✔
305
        datasetProperty -> term.setDataLocation(datasetProperty.getPropertyValueAsString())
1✔
306
    );
307

308
    return term;
1✔
309
  }
310

311
  Optional<DatasetProperty> findDatasetProperty(Collection<DatasetProperty> props,
312
      String schemaProp) {
313
    return
1✔
314
        props
315
            .stream()
1✔
316
            .filter(p -> Objects.nonNull(p.getSchemaProperty()))
1✔
317
            .filter(p -> p.getSchemaProperty().equals(schemaProp))
1✔
318
            .findFirst();
1✔
319
  }
320

321
  Optional<StudyProperty> findStudyProperty(Collection<StudyProperty> props, String key) {
322
    if (Objects.isNull(props)) {
1✔
323
      return Optional.empty();
×
324
    }
325

326
    return
1✔
327
        props
328
            .stream()
1✔
329
            .filter(p -> p.getKey().equals(key))
1✔
330
            .findFirst();
1✔
331
  }
332

333

334
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc