#22002

Committed 01 Apr 2024 07:56PM CUT coverage: 20.716% (+0.5%) from 20.173%

Build # #22002

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #10453 from IQSS/develop

Merge 6.2 into master

Run Details

704 of 2679 new or added lines in 152 files covered. (26.28%)

81 existing lines in 49 files now uncovered.

17160 of 82836 relevant lines covered (20.72%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java

package edu.harvard.iq.dataverse.api;

import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetServiceBean;
import edu.harvard.iq.dataverse.GlobalId;
import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitations;
import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitationsServiceBean;
import edu.harvard.iq.dataverse.makedatacount.DatasetMetrics;
import edu.harvard.iq.dataverse.makedatacount.DatasetMetricsServiceBean;
import edu.harvard.iq.dataverse.makedatacount.MakeDataCountProcessState;
import edu.harvard.iq.dataverse.makedatacount.MakeDataCountProcessStateServiceBean;
import edu.harvard.iq.dataverse.pidproviders.PidProvider;
import edu.harvard.iq.dataverse.pidproviders.PidUtil;
import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteDOIProvider;
import edu.harvard.iq.dataverse.settings.JvmSettings;
import edu.harvard.iq.dataverse.util.SystemConfig;
import edu.harvard.iq.dataverse.util.json.JsonUtil;

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import jakarta.ejb.EJB;
import jakarta.json.Json;
import jakarta.json.JsonArray;
import jakarta.json.JsonArrayBuilder;
import jakarta.json.JsonObject;
import jakarta.json.JsonObjectBuilder;
import jakarta.json.JsonValue;
import jakarta.ws.rs.DELETE;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.POST;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.Response.Status;

/**
 * Note that there are makeDataCount endpoints in Datasets.java as well.
 */
@Path("admin/makeDataCount")
public class MakeDataCountApi extends AbstractApiBean {

    private static final Logger logger = Logger.getLogger(MakeDataCountApi.class.getCanonicalName());

    @EJB
    DatasetMetricsServiceBean datasetMetricsService;
    @EJB
    MakeDataCountProcessStateServiceBean makeDataCountProcessStateService;
    @EJB
    DatasetExternalCitationsServiceBean datasetExternalCitationsService;
    @EJB
    DatasetServiceBean datasetService;
    @EJB
    SystemConfig systemConfig;

    /**
     * TODO: For each dataset, send the following:
     *
     * - views
     *
     * - downloads
     *
     * - citations (based on "Related Dataset"). "DataCite already supports
     * linking to publications in the DOI-related metadata that you submit with
     * your DOIs using the 'relatedIdentifier' element. See the DataCite
     * EventData Guide ( https://support.datacite.org/docs/eventdata-guide ).
     * These publication linkages are parsed and added to the EventData source.
     * So, the "hub" is used for reporting Investigations and Requests as
     * counts, whereas every individual citation event is reported in the DOI
     * metadata." mbjones at
     * https://github.com/IQSS/dataverse/issues/4821#issuecomment-440740205
     *
     * TODO: While we're working on sending citations for related *datasets* to
     * DataCite we should also strongly consider sending citations for related
     * *publications* (articles) as well. See
     * https://github.com/IQSS/dataverse/issues/2917 and
     * https://github.com/IQSS/dataverse/issues/2778
     */
    @POST
    @Path("sendToHub")
    public Response sendDataToHub() {
        String msg = "Data has been sent to Make Data Count";
        return ok(msg);
    }

    @POST
    @Path("{id}/addUsageMetricsFromSushiReport")
    public Response addUsageMetricsFromSushiReport(@PathParam("id") String id, @QueryParam("reportOnDisk") String reportOnDisk) {

        try {
            JsonObject report = JsonUtil.getJsonObjectFromFile(reportOnDisk);
            Dataset dataset = findDatasetOrDie(id);
            List<DatasetMetrics> datasetMetrics = datasetMetricsService.parseSushiReport(report, dataset);
            if (!datasetMetrics.isEmpty()) {
                for (DatasetMetrics dm : datasetMetrics) {
                    datasetMetricsService.save(dm);
                }
            }
        } catch (WrappedResponse ex) {
            logger.log(Level.SEVERE, null, ex);
            return error(Status.BAD_REQUEST, "Wrapped response: " + ex.getLocalizedMessage());

        } catch (IOException ex) {
            logger.log(Level.WARNING, ex.getMessage());
            return error(Status.BAD_REQUEST, "IOException: " + ex.getLocalizedMessage());
        }
        String msg = "Dummy Data has been added to dataset " + id;
        return ok(msg);
    }

    @POST
    @Path("/addUsageMetricsFromSushiReport")
    public Response addUsageMetricsFromSushiReportAll(@QueryParam("reportOnDisk") String reportOnDisk) {

        try {
            JsonObject report = JsonUtil.getJsonObjectFromFile(reportOnDisk);

            List<DatasetMetrics> datasetMetrics = datasetMetricsService.parseSushiReport(report, null);
            if (!datasetMetrics.isEmpty()) {
                for (DatasetMetrics dm : datasetMetrics) {
                    datasetMetricsService.save(dm);
                }
            }

        } catch (IOException ex) {
            logger.log(Level.WARNING, ex.getMessage());
            return error(Status.BAD_REQUEST, "IOException: " + ex.getLocalizedMessage());
        }
        String msg = "Usage Metrics Data has been added to all datasets from file  " + reportOnDisk;
        return ok(msg);
    }

    @POST
    @Path("{id}/updateCitationsForDataset")
    public Response updateCitationsForDataset(@PathParam("id") String id) throws IOException {
        try {
            Dataset dataset = findDatasetOrDie(id);
            GlobalId pid = dataset.getGlobalId();
            PidProvider pidProvider = PidUtil.getPidProvider(pid.getProviderId());
            // Only supported for DOIs and for DataCite DOI providers
            if(!DataCiteDOIProvider.TYPE.equals(pidProvider.getProviderType())) {
                return error(Status.BAD_REQUEST, "Only DataCite DOI providers are supported");
            }
            String persistentId = pid.toString();

            // DataCite wants "doi=", not "doi:".
            String authorityPlusIdentifier = persistentId.replaceFirst("doi:", "");
            // Request max page size and then loop to handle multiple pages
            URL url = new URL(JvmSettings.DATACITE_REST_API_URL.lookup() +
                              "/events?doi=" +
                              authorityPlusIdentifier +
                              "&source=crossref&page[size]=1000");
            logger.fine("Retrieving Citations from " + url.toString());
            boolean nextPage = true;
            JsonArrayBuilder dataBuilder = Json.createArrayBuilder();
            do {
                HttpURLConnection connection = (HttpURLConnection) url.openConnection();
                connection.setRequestMethod("GET");
                int status = connection.getResponseCode();
                if (status != 200) {
                    logger.warning("Failed to get citations from " + url.toString());
                    return error(Status.fromStatusCode(status), "Failed to get citations from " + url.toString());
                }
                JsonObject report;
                try (InputStream inStream = connection.getInputStream()) {
                    report = JsonUtil.getJsonObject(inStream);
                }
                JsonObject links = report.getJsonObject("links");
                JsonArray data = report.getJsonArray("data");
                Iterator<JsonValue> iter = data.iterator();
                while (iter.hasNext()) {
                    dataBuilder.add(iter.next());
                }
                if (links.containsKey("next")) {
                    url = new URL(links.getString("next"));
                } else {
                    nextPage = false;
                }
                logger.fine("body of citation response: " + report.toString());
            } while (nextPage == true);
            JsonArray allData = dataBuilder.build();
            List<DatasetExternalCitations> datasetExternalCitations = datasetExternalCitationsService.parseCitations(allData);
            /*
             * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopuate them.
             * As is, this call doesn't remove old citations if there are now none (legacy issue if we decide to stop counting certain types of citation
             * as we've done for 'hasPart').
             * If there are some, this call individually checks each one and if a matching item exists, it removes it and adds it back. Faster and better to delete all and
             * add the new ones.
             */
            if (!datasetExternalCitations.isEmpty()) {
                for (DatasetExternalCitations dm : datasetExternalCitations) {
                    datasetExternalCitationsService.save(dm);
                }
            }

            JsonObjectBuilder output = Json.createObjectBuilder();
            output.add("citationCount", datasetExternalCitations.size());
            return ok(output);
        } catch (WrappedResponse wr) {
            return wr.getResponse();
        }
    }
    @GET
    @Path("{yearMonth}/processingState")
    public Response getProcessingState(@PathParam("yearMonth") String yearMonth) {
        MakeDataCountProcessState mdcps;
        try {
            mdcps = makeDataCountProcessStateService.getMakeDataCountProcessState(yearMonth);
        } catch (IllegalArgumentException e) {
            return error(Status.BAD_REQUEST,e.getMessage());
        }
        if (mdcps != null) {
            JsonObjectBuilder output = Json.createObjectBuilder();
            output.add("yearMonth", mdcps.getYearMonth());
            output.add("state", mdcps.getState().name());
            output.add("stateChangeTimestamp", mdcps.getStateChangeTime().toString());
            return ok(output);
        } else {
            return error(Status.NOT_FOUND, "Could not find an existing process state for " + yearMonth);
        }
    }

    @POST
    @Path("{yearMonth}/processingState")
    public Response updateProcessingState(@PathParam("yearMonth") String yearMonth, @QueryParam("state") String state) {
        MakeDataCountProcessState mdcps;
        try {
            mdcps = makeDataCountProcessStateService.setMakeDataCountProcessState(yearMonth, state);
        } catch (Exception e) {
            return badRequest(e.getMessage());
        }

        JsonObjectBuilder output = Json.createObjectBuilder();
        output.add("yearMonth", mdcps.getYearMonth());
        output.add("state", mdcps.getState().name());
        output.add("stateChangeTimestamp", mdcps.getStateChangeTime().toString());
        return ok(output);
    }

    @DELETE
    @Path("{yearMonth}/processingState")
    public Response deleteProcessingState(@PathParam("yearMonth") String yearMonth) {
        boolean deleted = makeDataCountProcessStateService.deleteMakeDataCountProcessState(yearMonth);
        if (deleted) {
            return ok("Processing State deleted for " + yearMonth);
        } else {
            return notFound("Processing State not found for " + yearMonth);
        }
    }
}

1	package edu.harvard.iq.dataverse.api;
2
3	import edu.harvard.iq.dataverse.Dataset;
4	import edu.harvard.iq.dataverse.DatasetServiceBean;
5	import edu.harvard.iq.dataverse.GlobalId;
6	import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitations;
7	import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitationsServiceBean;
8	import edu.harvard.iq.dataverse.makedatacount.DatasetMetrics;
9	import edu.harvard.iq.dataverse.makedatacount.DatasetMetricsServiceBean;
10	import edu.harvard.iq.dataverse.makedatacount.MakeDataCountProcessState;
11	import edu.harvard.iq.dataverse.makedatacount.MakeDataCountProcessStateServiceBean;
12	import edu.harvard.iq.dataverse.pidproviders.PidProvider;
13	import edu.harvard.iq.dataverse.pidproviders.PidUtil;
14	import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteDOIProvider;
15	import edu.harvard.iq.dataverse.settings.JvmSettings;
16	import edu.harvard.iq.dataverse.util.SystemConfig;
17	import edu.harvard.iq.dataverse.util.json.JsonUtil;
18
19	import java.io.IOException;
20	import java.io.InputStream;
21	import java.net.HttpURLConnection;
22	import java.net.URL;
23	import java.util.Iterator;
24	import java.util.List;
25	import java.util.logging.Level;
26	import java.util.logging.Logger;
27	import jakarta.ejb.EJB;
28	import jakarta.json.Json;
29	import jakarta.json.JsonArray;
30	import jakarta.json.JsonArrayBuilder;
31	import jakarta.json.JsonObject;
32	import jakarta.json.JsonObjectBuilder;
33	import jakarta.json.JsonValue;
34	import jakarta.ws.rs.DELETE;
35	import jakarta.ws.rs.GET;
36	import jakarta.ws.rs.POST;
37	import jakarta.ws.rs.Path;
38	import jakarta.ws.rs.PathParam;
39	import jakarta.ws.rs.QueryParam;
40	import jakarta.ws.rs.core.Response;
41	import jakarta.ws.rs.core.Response.Status;
42
43	/**
44	* Note that there are makeDataCount endpoints in Datasets.java as well.
45	*/
46	@Path("admin/makeDataCount")
47	public class MakeDataCountApi extends AbstractApiBean {	×
48
49	private static final Logger logger = Logger.getLogger(MakeDataCountApi.class.getCanonicalName());	×
50
51	@EJB
52	DatasetMetricsServiceBean datasetMetricsService;
53	@EJB
54	MakeDataCountProcessStateServiceBean makeDataCountProcessStateService;
55	@EJB
56	DatasetExternalCitationsServiceBean datasetExternalCitationsService;
57	@EJB
58	DatasetServiceBean datasetService;
59	@EJB
60	SystemConfig systemConfig;
61
62	/**
63	* TODO: For each dataset, send the following:
64	*
65	* - views
66	*
67	* - downloads
68	*
69	* - citations (based on "Related Dataset"). "DataCite already supports
70	* linking to publications in the DOI-related metadata that you submit with
71	* your DOIs using the 'relatedIdentifier' element. See the DataCite
72	* EventData Guide ( https://support.datacite.org/docs/eventdata-guide ).
73	* These publication linkages are parsed and added to the EventData source.
74	* So, the "hub" is used for reporting Investigations and Requests as
75	* counts, whereas every individual citation event is reported in the DOI
76	* metadata." mbjones at
77	* https://github.com/IQSS/dataverse/issues/4821#issuecomment-440740205
78	*
79	* TODO: While we're working on sending citations for related datasets to
80	* DataCite we should also strongly consider sending citations for related
81	* publications (articles) as well. See
82	* https://github.com/IQSS/dataverse/issues/2917 and
83	* https://github.com/IQSS/dataverse/issues/2778
84	*/
85	@POST
86	@Path("sendToHub")
87	public Response sendDataToHub() {
88	String msg = "Data has been sent to Make Data Count";	×
89	return ok(msg);	×
90	}
91
92	@POST
93	@Path("{id}/addUsageMetricsFromSushiReport")
94	public Response addUsageMetricsFromSushiReport(@PathParam("id") String id, @QueryParam("reportOnDisk") String reportOnDisk) {
95
96	try {
97	JsonObject report = JsonUtil.getJsonObjectFromFile(reportOnDisk);	×
98	Dataset dataset = findDatasetOrDie(id);	×
99	List<DatasetMetrics> datasetMetrics = datasetMetricsService.parseSushiReport(report, dataset);	×
100	if (!datasetMetrics.isEmpty()) {	×
101	for (DatasetMetrics dm : datasetMetrics) {	×
102	datasetMetricsService.save(dm);	×
103	}	×
104	}
105	} catch (WrappedResponse ex) {	×
106	logger.log(Level.SEVERE, null, ex);	×
107	return error(Status.BAD_REQUEST, "Wrapped response: " + ex.getLocalizedMessage());	×
108
109	} catch (IOException ex) {	×
110	logger.log(Level.WARNING, ex.getMessage());	×
111	return error(Status.BAD_REQUEST, "IOException: " + ex.getLocalizedMessage());	×
112	}	×
113	String msg = "Dummy Data has been added to dataset " + id;	×
114	return ok(msg);	×
115	}
116
117	@POST
118	@Path("/addUsageMetricsFromSushiReport")
119	public Response addUsageMetricsFromSushiReportAll(@QueryParam("reportOnDisk") String reportOnDisk) {
120
121	try {
122	JsonObject report = JsonUtil.getJsonObjectFromFile(reportOnDisk);	×
123
124	List<DatasetMetrics> datasetMetrics = datasetMetricsService.parseSushiReport(report, null);	×
125	if (!datasetMetrics.isEmpty()) {	×
126	for (DatasetMetrics dm : datasetMetrics) {	×
127	datasetMetricsService.save(dm);	×
128	}	×
129	}
130
131	} catch (IOException ex) {	×
132	logger.log(Level.WARNING, ex.getMessage());	×
133	return error(Status.BAD_REQUEST, "IOException: " + ex.getLocalizedMessage());	×
134	}	×
135	String msg = "Usage Metrics Data has been added to all datasets from file " + reportOnDisk;	×
136	return ok(msg);	×
137	}
138
139	@POST
140	@Path("{id}/updateCitationsForDataset")
141	public Response updateCitationsForDataset(@PathParam("id") String id) throws IOException {
142	try {
143	Dataset dataset = findDatasetOrDie(id);	×
NEW 144	GlobalId pid = dataset.getGlobalId();	×
NEW 145	PidProvider pidProvider = PidUtil.getPidProvider(pid.getProviderId());	×
146	// Only supported for DOIs and for DataCite DOI providers
NEW 147	if(!DataCiteDOIProvider.TYPE.equals(pidProvider.getProviderType())) {	×
NEW 148	return error(Status.BAD_REQUEST, "Only DataCite DOI providers are supported");	×
149	}
NEW 150	String persistentId = pid.toString();	×
151
152	// DataCite wants "doi=", not "doi:".
153	String authorityPlusIdentifier = persistentId.replaceFirst("doi:", "");	×
154	// Request max page size and then loop to handle multiple pages
155	URL url = new URL(JvmSettings.DATACITE_REST_API_URL.lookup() +	×
156	"/events?doi=" +
157	authorityPlusIdentifier +
158	"&source=crossref&page[size]=1000");
159	logger.fine("Retrieving Citations from " + url.toString());	×
160	boolean nextPage = true;	×
161	JsonArrayBuilder dataBuilder = Json.createArrayBuilder();	×
162	do {
163	HttpURLConnection connection = (HttpURLConnection) url.openConnection();	×
164	connection.setRequestMethod("GET");	×
165	int status = connection.getResponseCode();	×
166	if (status != 200) {	×
167	logger.warning("Failed to get citations from " + url.toString());	×
168	return error(Status.fromStatusCode(status), "Failed to get citations from " + url.toString());	×
169	}
170	JsonObject report;
171	try (InputStream inStream = connection.getInputStream()) {	×
172	report = JsonUtil.getJsonObject(inStream);	×
173	}
174	JsonObject links = report.getJsonObject("links");	×
175	JsonArray data = report.getJsonArray("data");	×
176	Iterator<JsonValue> iter = data.iterator();	×
177	while (iter.hasNext()) {	×
178	dataBuilder.add(iter.next());	×
179	}
180	if (links.containsKey("next")) {	×
181	url = new URL(links.getString("next"));	×
182	} else {
183	nextPage = false;	×
184	}
185	logger.fine("body of citation response: " + report.toString());	×
186	} while (nextPage == true);	×
187	JsonArray allData = dataBuilder.build();	×
188	List<DatasetExternalCitations> datasetExternalCitations = datasetExternalCitationsService.parseCitations(allData);	×
189	/*
190	* ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopuate them.
191	* As is, this call doesn't remove old citations if there are now none (legacy issue if we decide to stop counting certain types of citation
192	* as we've done for 'hasPart').
193	* If there are some, this call individually checks each one and if a matching item exists, it removes it and adds it back. Faster and better to delete all and
194	* add the new ones.
195	*/
196	if (!datasetExternalCitations.isEmpty()) {	×
197	for (DatasetExternalCitations dm : datasetExternalCitations) {	×
198	datasetExternalCitationsService.save(dm);	×
199	}	×
200	}
201
202	JsonObjectBuilder output = Json.createObjectBuilder();	×
203	output.add("citationCount", datasetExternalCitations.size());	×
204	return ok(output);	×
205	} catch (WrappedResponse wr) {	×
206	return wr.getResponse();	×
207	}
208	}
209	@GET
210	@Path("{yearMonth}/processingState")
211	public Response getProcessingState(@PathParam("yearMonth") String yearMonth) {
212	MakeDataCountProcessState mdcps;
213	try {
NEW 214	mdcps = makeDataCountProcessStateService.getMakeDataCountProcessState(yearMonth);	×
NEW 215	} catch (IllegalArgumentException e) {	×
NEW 216	return error(Status.BAD_REQUEST,e.getMessage());	×
NEW 217	}	×
NEW 218	if (mdcps != null) {	×
NEW 219	JsonObjectBuilder output = Json.createObjectBuilder();	×
NEW 220	output.add("yearMonth", mdcps.getYearMonth());	×
NEW 221	output.add("state", mdcps.getState().name());	×
NEW 222	output.add("stateChangeTimestamp", mdcps.getStateChangeTime().toString());	×
NEW 223	return ok(output);	×
224	} else {
NEW 225	return error(Status.NOT_FOUND, "Could not find an existing process state for " + yearMonth);	×
226	}
227	}
228
229	@POST
230	@Path("{yearMonth}/processingState")
231	public Response updateProcessingState(@PathParam("yearMonth") String yearMonth, @QueryParam("state") String state) {
232	MakeDataCountProcessState mdcps;
233	try {
NEW 234	mdcps = makeDataCountProcessStateService.setMakeDataCountProcessState(yearMonth, state);	×
NEW 235	} catch (Exception e) {	×
NEW 236	return badRequest(e.getMessage());	×
NEW 237	}	×
238
NEW 239	JsonObjectBuilder output = Json.createObjectBuilder();	×
NEW 240	output.add("yearMonth", mdcps.getYearMonth());	×
NEW 241	output.add("state", mdcps.getState().name());	×
NEW 242	output.add("stateChangeTimestamp", mdcps.getStateChangeTime().toString());	×
NEW 243	return ok(output);	×
244	}
245
246	@DELETE
247	@Path("{yearMonth}/processingState")
248	public Response deleteProcessingState(@PathParam("yearMonth") String yearMonth) {
NEW 249	boolean deleted = makeDataCountProcessStateService.deleteMakeDataCountProcessState(yearMonth);	×
NEW 250	if (deleted) {	×
NEW 251	return ok("Processing State deleted for " + yearMonth);	×
252	} else {
NEW 253	return notFound("Processing State not found for " + yearMonth);	×
254	}
255	}
256	}

IQSS / dataverse / #22002

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous