#5636

Committed 08 Apr 2025 02:19PM UTC coverage: 79.133% (-0.1%) from 79.273%

Build # #5636

Build Type

push

Committed by

Commit Message

DT-1424: Mark datasets as indexed/deindexed when indexing operations are called (#2470)

Run Details

58 of 83 new or added lines in 10 files covered. (69.88%)

10 existing lines in 3 files now uncovered.

10277 of 12987 relevant lines covered (79.13%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.62

/src/main/java/org/broadinstitute/consent/http/resources/StudyResource.java

package org.broadinstitute.consent.http.resources;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.google.inject.Inject;
import io.dropwizard.auth.Auth;
import jakarta.annotation.security.PermitAll;
import jakarta.annotation.security.RolesAllowed;
import jakarta.ws.rs.BadRequestException;
import jakarta.ws.rs.Consumes;
import jakarta.ws.rs.DELETE;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.NotFoundException;
import jakarta.ws.rs.PUT;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.Response.Status;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import org.apache.commons.validator.routines.EmailValidator;
import org.broadinstitute.consent.http.enumeration.UserRoles;
import org.broadinstitute.consent.http.models.AuthUser;
import org.broadinstitute.consent.http.models.Dataset;
import org.broadinstitute.consent.http.models.Study;
import org.broadinstitute.consent.http.models.StudyConversion;
import org.broadinstitute.consent.http.models.User;
import org.broadinstitute.consent.http.models.dataset_registration_v1.DatasetRegistrationSchemaV1;
import org.broadinstitute.consent.http.models.dataset_registration_v1.DatasetRegistrationSchemaV1UpdateValidator;
import org.broadinstitute.consent.http.models.dataset_registration_v1.builder.DatasetRegistrationSchemaV1Builder;
import org.broadinstitute.consent.http.service.DatasetRegistrationService;
import org.broadinstitute.consent.http.service.DatasetService;
import org.broadinstitute.consent.http.service.ElasticSearchService;
import org.broadinstitute.consent.http.service.UserService;
import org.broadinstitute.consent.http.util.gson.GsonUtil;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.FormDataParam;

@Path("api/dataset/study")
public class StudyResource extends Resource {

  private final DatasetService datasetService;
  private final DatasetRegistrationService datasetRegistrationService;
  private final UserService userService;
  private final ElasticSearchService elasticSearchService;


  @Inject
  public StudyResource(DatasetService datasetService, UserService userService,
      DatasetRegistrationService datasetRegistrationService,
      ElasticSearchService elasticSearchService) {
    this.datasetService = datasetService;
    this.userService = userService;
    this.datasetRegistrationService = datasetRegistrationService;
    this.elasticSearchService = elasticSearchService;
  }

  /**
   * This API creates a study for a provided dataset, or updates existing study/dataset information
   * with what is provided in the request body. It is intended to be a short-lived API that will be
   * removed once all production datasets have been migrated.
   */
  @PUT
  @Path("/convert/{datasetIdentifier}")
  @Consumes(MediaType.APPLICATION_JSON)
  @Produces(MediaType.APPLICATION_JSON)
  @RolesAllowed({ADMIN})
  public Response convertToStudy(@Auth AuthUser authUser,
    @PathParam("datasetIdentifier") String datasetIdentifier, String json) {
    try {
      User user = userService.findUserByEmail(authUser.getEmail());
      Dataset dataset = datasetService.findDatasetByIdentifier(datasetIdentifier);
      StudyConversion studyConversion = new Gson().fromJson(json, StudyConversion.class);
      Study study = datasetService.convertDatasetToStudy(user, dataset, studyConversion);
      return Response.ok(study).build();
    } catch (Exception e) {
      return createExceptionResponse(e);
    }
  }

  /**
   * This API adds/updates custodians for a study. The payload needs to be a JSON array of valid
   * email addresses
   */
  @PUT
  @Path("/{studyId}/custodians")
  @Consumes(MediaType.APPLICATION_JSON)
  @Produces(MediaType.APPLICATION_JSON)
  @RolesAllowed({ADMIN})
  public Response updateCustodians(@Auth AuthUser authUser,
    @PathParam("studyId") Integer studyId, String json) {
    try {
      User user = userService.findUserByEmail(authUser.getEmail());
      Gson gson = new Gson();
      Type listOfStringObject = new TypeToken<ArrayList<String>>() {}.getType();
      List<String> custodians = gson.fromJson(json, listOfStringObject);
      // Validate that the custodians are all valid email addresses:
      EmailValidator emailValidator = EmailValidator.getInstance();
      List<Boolean> valid = custodians.stream().map(emailValidator::isValid).toList();
      if (valid.contains(false)) {
        throw new BadRequestException(String.format("Invalid email address: %s", json));
      }
      Study study = datasetService.updateStudyCustodians(user, studyId, json);
      return Response.ok(study).build();
    } catch (Exception e) {
      return createExceptionResponse(e);
    }
  }

  @GET
  @Path("/{studyId}")
  @Produces(MediaType.APPLICATION_JSON)
  @PermitAll
  public Response getStudyById(@PathParam("studyId") Integer studyId) {
    try {
      Study study = datasetService.getStudyWithDatasetsById(studyId);
      return Response.ok(study).build();
    } catch (Exception e) {
      return createExceptionResponse(e);
    }
  }

  @DELETE
  @Path("/{studyId}")
  @Produces(MediaType.APPLICATION_JSON)
  @RolesAllowed({ADMIN, CHAIRPERSON, DATASUBMITTER})
  public Response deleteStudyById(@Auth AuthUser authUser, @PathParam("studyId") Integer studyId) {
    try {
      final User user = userService.findUserByEmail(authUser.getEmail());
      Study study = datasetService.getStudyWithDatasetsById(studyId);

      if (Objects.isNull(study)) {
        throw new NotFoundException("Study not found");
      }

      // If the user is not an admin, ensure that they are the study/dataset creator
      if (!user.hasUserRole(UserRoles.ADMIN) && (!Objects.equals(study.getCreateUserId(),
          user.getUserId()))) {
        throw new NotFoundException("Study not found");
      }

      boolean deletable = (study.getDatasets() == null || study.getDatasets().isEmpty()) || study.getDatasets()
          .stream()
          .allMatch(Dataset::getDeletable);
      if (!deletable) {
        throw new BadRequestException("Study has datasets that are in use and cannot be deleted.");
      }
      Set<Integer> studyDatasetIds = study.getDatasetIds();
      datasetService.deleteStudy(study, user);
      // Remove from ES index
      studyDatasetIds.forEach(id -> {
        try (Response indexResponse = elasticSearchService.deleteIndex(id, user.getUserId())) {
          if (indexResponse.getStatus() >= Status.BAD_REQUEST.getStatusCode()) {
            logWarn("Non-OK response when deleting index for dataset with id: " + id);
          }
        } catch (IOException e) {
          logException(e);
        }
      });
      return Response.ok().build();
    } catch (Exception e) {
      return createExceptionResponse(e);
    }
  }

  @GET
  @Path("/registration/{studyId}")
  @Produces(MediaType.APPLICATION_JSON)
  @PermitAll
  public Response getRegistrationFromStudy(@Auth AuthUser authUser,
      @PathParam("studyId") Integer studyId) {
    try {
      Study study = datasetService.getStudyWithDatasetsById(studyId);
      List<Dataset> datasets =
          Objects.nonNull(study.getDatasets()) ? study.getDatasets().stream().toList() : List.of();
      DatasetRegistrationSchemaV1 registration = new DatasetRegistrationSchemaV1Builder().build(
          study, datasets);
      String entity = GsonUtil.buildGsonNullSerializer().toJson(registration);
      return Response.ok().entity(entity).build();
    } catch (Exception e) {
      return createExceptionResponse(e);
    }
  }

  @PUT
  @Consumes({MediaType.MULTIPART_FORM_DATA})
  @Produces({MediaType.APPLICATION_JSON})
  @Path("/{studyId}")
  @RolesAllowed({ADMIN, CHAIRPERSON, DATASUBMITTER})
  /*
   * This endpoint accepts a json instance of a dataset-registration-schema_v1.json schema.
   * With that object, we can fully update the study/datasets from the provided values.
   */
  public Response updateStudyByRegistration(
      @Auth AuthUser authUser,
      FormDataMultiPart multipart,
      @PathParam("studyId") Integer studyId,
      @FormDataParam("dataset") String json) {
    try {
      User user = userService.findUserByEmail(authUser.getEmail());
      Study existingStudy = datasetRegistrationService.findStudyById(studyId);

      // Manually validate the schema from an editing context. Validation with the schema tools
      // enforces it in a creation context but doesn't work for editing purposes.
      DatasetRegistrationSchemaV1UpdateValidator updateValidator = new DatasetRegistrationSchemaV1UpdateValidator(datasetService);
      DatasetRegistrationSchemaV1 registration = updateValidator.deserializeRegistration(json);

      if (updateValidator.validate(existingStudy, registration)) {
        // Update study from registration
        Map<String, FormDataBodyPart> files = extractFilesFromMultiPart(multipart);
        Study updatedStudy = datasetRegistrationService.updateStudyFromRegistration(
            studyId,
            registration,
            user,
            files);
        try (Response indexResponse = elasticSearchService.indexStudy(studyId, user))  {
          if (indexResponse.getStatus() >= Status.BAD_REQUEST.getStatusCode()) {
            logWarn("Non-OK response when reindexing study with id: " + studyId);
          }
        } catch (Exception e) {
          logException("Exception re-indexing datasets from study id: " + studyId, e);
        }
        return Response.ok(updatedStudy).build();
      } else {
        return Response.status(Status.BAD_REQUEST).build();
      }
    } catch (Exception e) {
      return createExceptionResponse(e);
    }
  }
}

1	package org.broadinstitute.consent.http.resources;
2
3	import com.google.gson.Gson;
4	import com.google.gson.reflect.TypeToken;
5	import com.google.inject.Inject;
6	import io.dropwizard.auth.Auth;
7	import jakarta.annotation.security.PermitAll;
8	import jakarta.annotation.security.RolesAllowed;
9	import jakarta.ws.rs.BadRequestException;
10	import jakarta.ws.rs.Consumes;
11	import jakarta.ws.rs.DELETE;
12	import jakarta.ws.rs.GET;
13	import jakarta.ws.rs.NotFoundException;
14	import jakarta.ws.rs.PUT;
15	import jakarta.ws.rs.Path;
16	import jakarta.ws.rs.PathParam;
17	import jakarta.ws.rs.Produces;
18	import jakarta.ws.rs.core.MediaType;
19	import jakarta.ws.rs.core.Response;
20	import jakarta.ws.rs.core.Response.Status;
21	import java.io.IOException;
22	import java.lang.reflect.Type;
23	import java.util.ArrayList;
24	import java.util.List;
25	import java.util.Map;
26	import java.util.Objects;
27	import java.util.Set;
28	import org.apache.commons.validator.routines.EmailValidator;
29	import org.broadinstitute.consent.http.enumeration.UserRoles;
30	import org.broadinstitute.consent.http.models.AuthUser;
31	import org.broadinstitute.consent.http.models.Dataset;
32	import org.broadinstitute.consent.http.models.Study;
33	import org.broadinstitute.consent.http.models.StudyConversion;
34	import org.broadinstitute.consent.http.models.User;
35	import org.broadinstitute.consent.http.models.dataset_registration_v1.DatasetRegistrationSchemaV1;
36	import org.broadinstitute.consent.http.models.dataset_registration_v1.DatasetRegistrationSchemaV1UpdateValidator;
37	import org.broadinstitute.consent.http.models.dataset_registration_v1.builder.DatasetRegistrationSchemaV1Builder;
38	import org.broadinstitute.consent.http.service.DatasetRegistrationService;
39	import org.broadinstitute.consent.http.service.DatasetService;
40	import org.broadinstitute.consent.http.service.ElasticSearchService;
41	import org.broadinstitute.consent.http.service.UserService;
42	import org.broadinstitute.consent.http.util.gson.GsonUtil;
43	import org.glassfish.jersey.media.multipart.FormDataBodyPart;
44	import org.glassfish.jersey.media.multipart.FormDataMultiPart;
45	import org.glassfish.jersey.media.multipart.FormDataParam;
46
47	@Path("api/dataset/study")
48	public class StudyResource extends Resource {
49
50	private final DatasetService datasetService;
51	private final DatasetRegistrationService datasetRegistrationService;
52	private final UserService userService;
53	private final ElasticSearchService elasticSearchService;
54
55
56	@Inject
57	public StudyResource(DatasetService datasetService, UserService userService,
58	DatasetRegistrationService datasetRegistrationService,
59	ElasticSearchService elasticSearchService) {	1✔
60	this.datasetService = datasetService;	1✔
61	this.userService = userService;	1✔
62	this.datasetRegistrationService = datasetRegistrationService;	1✔
63	this.elasticSearchService = elasticSearchService;	1✔
64	}	1✔
65
66	/**
67	* This API creates a study for a provided dataset, or updates existing study/dataset information
68	* with what is provided in the request body. It is intended to be a short-lived API that will be
69	* removed once all production datasets have been migrated.
70	*/
71	@PUT
72	@Path("/convert/{datasetIdentifier}")
73	@Consumes(MediaType.APPLICATION_JSON)
74	@Produces(MediaType.APPLICATION_JSON)
75	@RolesAllowed({ADMIN})
76	public Response convertToStudy(@Auth AuthUser authUser,
77	@PathParam("datasetIdentifier") String datasetIdentifier, String json) {
78	try {
79	User user = userService.findUserByEmail(authUser.getEmail());	×
80	Dataset dataset = datasetService.findDatasetByIdentifier(datasetIdentifier);	×
81	StudyConversion studyConversion = new Gson().fromJson(json, StudyConversion.class);	×
82	Study study = datasetService.convertDatasetToStudy(user, dataset, studyConversion);	×
83	return Response.ok(study).build();	×
84	} catch (Exception e) {	×
85	return createExceptionResponse(e);	×
86	}
87	}
88
89	/**
90	* This API adds/updates custodians for a study. The payload needs to be a JSON array of valid
91	* email addresses
92	*/
93	@PUT
94	@Path("/{studyId}/custodians")
95	@Consumes(MediaType.APPLICATION_JSON)
96	@Produces(MediaType.APPLICATION_JSON)
97	@RolesAllowed({ADMIN})
98	public Response updateCustodians(@Auth AuthUser authUser,
99	@PathParam("studyId") Integer studyId, String json) {
100	try {
101	User user = userService.findUserByEmail(authUser.getEmail());	1✔
102	Gson gson = new Gson();	1✔
103	Type listOfStringObject = new TypeToken<ArrayList<String>>() {}.getType();	1✔
104	List<String> custodians = gson.fromJson(json, listOfStringObject);	1✔
105	// Validate that the custodians are all valid email addresses:
106	EmailValidator emailValidator = EmailValidator.getInstance();	1✔
107	List<Boolean> valid = custodians.stream().map(emailValidator::isValid).toList();	1✔
108	if (valid.contains(false)) {	1✔
109	throw new BadRequestException(String.format("Invalid email address: %s", json));	1✔
110	}
111	Study study = datasetService.updateStudyCustodians(user, studyId, json);	1✔
112	return Response.ok(study).build();	1✔
113	} catch (Exception e) {	1✔
114	return createExceptionResponse(e);	1✔
115	}
116	}
117
118	@GET
119	@Path("/{studyId}")
120	@Produces(MediaType.APPLICATION_JSON)
121	@PermitAll
122	public Response getStudyById(@PathParam("studyId") Integer studyId) {
123	try {
124	Study study = datasetService.getStudyWithDatasetsById(studyId);	1✔
125	return Response.ok(study).build();	1✔
126	} catch (Exception e) {	1✔
127	return createExceptionResponse(e);	1✔
128	}
129	}
130
131	@DELETE
132	@Path("/{studyId}")
133	@Produces(MediaType.APPLICATION_JSON)
134	@RolesAllowed({ADMIN, CHAIRPERSON, DATASUBMITTER})
135	public Response deleteStudyById(@Auth AuthUser authUser, @PathParam("studyId") Integer studyId) {
136	try {
137	final User user = userService.findUserByEmail(authUser.getEmail());	1✔
138	Study study = datasetService.getStudyWithDatasetsById(studyId);	1✔
139
140	if (Objects.isNull(study)) {	1✔
141	throw new NotFoundException("Study not found");	1✔
142	}
143
144	// If the user is not an admin, ensure that they are the study/dataset creator
145	if (!user.hasUserRole(UserRoles.ADMIN) && (!Objects.equals(study.getCreateUserId(),	1✔
146	user.getUserId()))) {	1✔
147	throw new NotFoundException("Study not found");	1✔
148	}
149
150	boolean deletable = (study.getDatasets() == null \|\| study.getDatasets().isEmpty()) \|\| study.getDatasets()	1✔
151	.stream()	1✔
152	.allMatch(Dataset::getDeletable);	1✔
153	if (!deletable) {	1✔
154	throw new BadRequestException("Study has datasets that are in use and cannot be deleted.");	1✔
155	}
156	Set<Integer> studyDatasetIds = study.getDatasetIds();	1✔
157	datasetService.deleteStudy(study, user);	1✔
158	// Remove from ES index
159	studyDatasetIds.forEach(id -> {	1✔
160	try (Response indexResponse = elasticSearchService.deleteIndex(id, user.getUserId())) {	1✔
161	if (indexResponse.getStatus() >= Status.BAD_REQUEST.getStatusCode()) {	1✔
NEW 162	logWarn("Non-OK response when deleting index for dataset with id: " + id);	×
163	}
164	} catch (IOException e) {	1✔
165	logException(e);	1✔
166	}	1✔
167	});	1✔
168	return Response.ok().build();	1✔
169	} catch (Exception e) {	1✔
170	return createExceptionResponse(e);	1✔
171	}
172	}
173
174	@GET
175	@Path("/registration/{studyId}")
176	@Produces(MediaType.APPLICATION_JSON)
177	@PermitAll
178	public Response getRegistrationFromStudy(@Auth AuthUser authUser,
179	@PathParam("studyId") Integer studyId) {
180	try {
181	Study study = datasetService.getStudyWithDatasetsById(studyId);	1✔
182	List<Dataset> datasets =
183	Objects.nonNull(study.getDatasets()) ? study.getDatasets().stream().toList() : List.of();	1✔
184	DatasetRegistrationSchemaV1 registration = new DatasetRegistrationSchemaV1Builder().build(	1✔
185	study, datasets);
186	String entity = GsonUtil.buildGsonNullSerializer().toJson(registration);	1✔
187	return Response.ok().entity(entity).build();	1✔
188	} catch (Exception e) {	1✔
189	return createExceptionResponse(e);	1✔
190	}
191	}
192
193	@PUT
194	@Consumes({MediaType.MULTIPART_FORM_DATA})
195	@Produces({MediaType.APPLICATION_JSON})
196	@Path("/{studyId}")
197	@RolesAllowed({ADMIN, CHAIRPERSON, DATASUBMITTER})
198	/*
199	* This endpoint accepts a json instance of a dataset-registration-schema_v1.json schema.
200	* With that object, we can fully update the study/datasets from the provided values.
201	*/
202	public Response updateStudyByRegistration(
203	@Auth AuthUser authUser,
204	FormDataMultiPart multipart,
205	@PathParam("studyId") Integer studyId,
206	@FormDataParam("dataset") String json) {
207	try {
208	User user = userService.findUserByEmail(authUser.getEmail());	1✔
209	Study existingStudy = datasetRegistrationService.findStudyById(studyId);	1✔
210
211	// Manually validate the schema from an editing context. Validation with the schema tools
212	// enforces it in a creation context but doesn't work for editing purposes.
213	DatasetRegistrationSchemaV1UpdateValidator updateValidator = new DatasetRegistrationSchemaV1UpdateValidator(datasetService);	1✔
214	DatasetRegistrationSchemaV1 registration = updateValidator.deserializeRegistration(json);	1✔
215
216	if (updateValidator.validate(existingStudy, registration)) {	1✔
217	// Update study from registration
218	Map<String, FormDataBodyPart> files = extractFilesFromMultiPart(multipart);	1✔
219	Study updatedStudy = datasetRegistrationService.updateStudyFromRegistration(	1✔
220	studyId,
221	registration,
222	user,
223	files);
224	try (Response indexResponse = elasticSearchService.indexStudy(studyId, user)) {	1✔
225	if (indexResponse.getStatus() >= Status.BAD_REQUEST.getStatusCode()) {	×
226	logWarn("Non-OK response when reindexing study with id: " + studyId);	×
227	}
228	} catch (Exception e) {	1✔
229	logException("Exception re-indexing datasets from study id: " + studyId, e);	1✔
230	}	×
231	return Response.ok(updatedStudy).build();	1✔
232	} else {
233	return Response.status(Status.BAD_REQUEST).build();	×
234	}
235	} catch (Exception e) {	1✔
236	return createExceptionResponse(e);	1✔
237	}
238	}
239	}

DataBiosphere / consent / #5636

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous