#22987

Committed 23 Aug 2024 06:44PM CUT coverage: 20.61% (-0.2%) from 20.791%

Build # #22987

Build Type

Pull #10781

github

Committed by

landreev

Commit Message

added an upfront locks check to the /addGlobusFiles api #10623

Pull Request Pull Request #10781: Improved handling of Globus uploads

Run Details

4 of 417 new or added lines in 15 files covered. (0.96%)

4194 existing lines in 35 files now uncovered.

17388 of 84365 relevant lines covered (20.61%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java

package edu.harvard.iq.dataverse.engine.command.impl;

import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetVersion;
import edu.harvard.iq.dataverse.DatasetLock.Reason;
import edu.harvard.iq.dataverse.authorization.Permission;
import edu.harvard.iq.dataverse.authorization.users.ApiToken;
import edu.harvard.iq.dataverse.engine.command.Command;
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
import edu.harvard.iq.dataverse.workflow.step.Failure;
import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult;

import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.nio.charset.Charset;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Map;
import java.util.logging.Logger;

import jakarta.json.Json;
import jakarta.json.JsonObjectBuilder;

import org.apache.commons.codec.binary.Hex;
import org.duracloud.client.ContentStore;
import org.duracloud.client.ContentStoreManager;
import org.duracloud.client.ContentStoreManagerImpl;
import org.duracloud.common.model.Credential;
import org.duracloud.error.ContentStoreException;

@RequiredPermissions(Permission.PublishDataset)
public class DuraCloudSubmitToArchiveCommand extends AbstractSubmitToArchiveCommand implements Command<DatasetVersion> {

    private static final Logger logger = Logger.getLogger(DuraCloudSubmitToArchiveCommand.class.getName());
    private static final String DEFAULT_PORT = "443";
    private static final String DEFAULT_CONTEXT = "durastore";
    private static final String DURACLOUD_PORT = ":DuraCloudPort";
    private static final String DURACLOUD_HOST = ":DuraCloudHost";
    private static final String DURACLOUD_CONTEXT = ":DuraCloudContext";


    public DuraCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) {
        super(aRequest, version);
    }

    @Override
    public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken token,
            Map<String, String> requestedSettings) {

        String port = requestedSettings.get(DURACLOUD_PORT) != null ? requestedSettings.get(DURACLOUD_PORT)
                : DEFAULT_PORT;
        String dpnContext = requestedSettings.get(DURACLOUD_CONTEXT) != null ? requestedSettings.get(DURACLOUD_CONTEXT)
                : DEFAULT_CONTEXT;
        String host = requestedSettings.get(DURACLOUD_HOST);
        
        if (host != null) {
            Dataset dataset = dv.getDataset();
            // ToDo - change after HDC 3A changes to status reporting
            // This will make the archivalCopyLocation non-null after a failure which should
            // stop retries
            
            if (dataset.getLockFor(Reason.finalizePublication) == null
                    && dataset.getLockFor(Reason.FileValidationFailed) == null) {
                // Use Duracloud client classes to login
                ContentStoreManager storeManager = new ContentStoreManagerImpl(host, port, dpnContext);
                Credential credential = new Credential(System.getProperty("duracloud.username"),
                        System.getProperty("duracloud.password"));
                storeManager.login(credential);
                /*
                 * Aliases can contain upper case characters which are not allowed in space
                 * names. Similarly, aliases can contain '_' which isn't allowed in a space
                 * name. The line below replaces any upper case chars with lowercase and
                 * replaces any '_' with '.-' . The '-' after the dot assures we don't break the
                 * rule that
                 * "The last period in a aspace may not immediately be followed by a number".
                 * (Although we could check, it seems better to just add '.-' all the time.As
                 * written the replaceAll will also change any chars not valid in a spaceName to
                 * '.' which would avoid code breaking if the alias constraints change. That
                 * said, this line may map more than one alias to the same spaceName, e.g.
                 * "test" and "Test" aliases both map to the "test" space name. This does not
                 * break anything but does potentially put bags from more than one collection in
                 * the same space.
                 */
                String spaceName = dataset.getOwner().getAlias().toLowerCase().replaceAll("[^a-z0-9-]", ".dcsafe");
                String baseFileName = dataset.getGlobalId().asString().replace(':', '-').replace('/', '-')
                        .replace('.', '-').toLowerCase() + "_v" + dv.getFriendlyVersionNumber();

                ContentStore store;
                //Set a failure status that will be updated if we succeed
                JsonObjectBuilder statusObject = Json.createObjectBuilder();
                statusObject.add(DatasetVersion.ARCHIVAL_STATUS, DatasetVersion.ARCHIVAL_STATUS_FAILURE);
                statusObject.add(DatasetVersion.ARCHIVAL_STATUS_MESSAGE, "Bag not transferred");
                
                try {
                    /*
                     * If there is a failure in creating a space, it is likely that a prior version
                     * has not been fully processed (snapshot created, archiving completed and files
                     * and space deleted - currently manual operations done at the project's
                     * duracloud website)
                     */
                    store = storeManager.getPrimaryContentStore();
                    // Create space to copy archival files to
                    if (!store.spaceExists(spaceName)) {
                        store.createSpace(spaceName);
                    }
                    String dataciteXml = getDataCiteXml(dv);

                    MessageDigest messageDigest = MessageDigest.getInstance("MD5");
                    try (PipedInputStream dataciteIn = new PipedInputStream();
                            DigestInputStream digestInputStream = new DigestInputStream(dataciteIn, messageDigest)) {
                        // Add datacite.xml file

                        Thread dcThread = new Thread(new Runnable() {
                            public void run() {
                                try (PipedOutputStream dataciteOut = new PipedOutputStream(dataciteIn)) {

                                    dataciteOut.write(dataciteXml.getBytes(Charset.forName("utf-8")));
                                    dataciteOut.close();
                                    success=true;
                                } catch (Exception e) {
                                    logger.severe("Error creating datacite.xml: " + e.getMessage());
                                    // TODO Auto-generated catch block
                                    e.printStackTrace();
                                }
                            }
                        }); 
                        dcThread.start();
                        // Have seen Pipe Closed errors for other archivers when used as a workflow
                        // without this delay loop
                        int i = 0;
                        while (digestInputStream.available() <= 0 && i < 100) {
                            Thread.sleep(10);
                            i++;
                        }
                        String checksum = store.addContent(spaceName, baseFileName + "_datacite.xml", digestInputStream,
                                -1l, null, null, null);
                        logger.fine("Content: datacite.xml added with checksum: " + checksum);
                        dcThread.join();
                        String localchecksum = Hex.encodeHexString(digestInputStream.getMessageDigest().digest());
                        if (!success || !checksum.equals(localchecksum)) {
                            logger.severe("Failure on " + baseFileName);
                            logger.severe(success ? checksum + " not equal to " + localchecksum : "failed to transfer to DuraCloud");
                            try {
                                store.deleteContent(spaceName, baseFileName + "_datacite.xml");
                            } catch (ContentStoreException cse) {
                                logger.warning(cse.getMessage());
                            }
                            return new Failure("Error in transferring DataCite.xml file to DuraCloud",
                                    "DuraCloud Submission Failure: incomplete metadata transfer");
                        }

                        // Store BagIt file
                        success = false;
                        String fileName = baseFileName + ".zip";

                        // Add BagIt ZIP file
                        // Although DuraCloud uses SHA-256 internally, it's API uses MD5 to verify the
                        // transfer

                        messageDigest = MessageDigest.getInstance("MD5");
                        try (PipedInputStream in = new PipedInputStream(100000);
                                DigestInputStream digestInputStream2 = new DigestInputStream(in, messageDigest)) {
                            Thread bagThread = startBagThread(dv, in, digestInputStream2, dataciteXml, token);
                            checksum = store.addContent(spaceName, fileName, digestInputStream2, -1l, null, null, null);
                            bagThread.join();
                            if (success) {
                                logger.fine("Content: " + fileName + " added with checksum: " + checksum);
                                localchecksum = Hex.encodeHexString(digestInputStream2.getMessageDigest().digest());
                            }
                            if (!success || !checksum.equals(localchecksum)) {
                                logger.severe("Failure on " + fileName);
                                logger.severe(success ? checksum + " not equal to " + localchecksum : "failed to transfer to DuraCloud");
                                try {
                                    store.deleteContent(spaceName, fileName);
                                    store.deleteContent(spaceName, baseFileName + "_datacite.xml");
                                } catch (ContentStoreException cse) {
                                    logger.warning(cse.getMessage());
                                }
                                return new Failure("Error in transferring Zip file to DuraCloud",
                                        "DuraCloud Submission Failure: incomplete archive transfer");
                            }
                        }

                        logger.fine("DuraCloud Submission step: Content Transferred");

                        // Document the location of dataset archival copy location (actually the URL
                        // where you can
                        // view it as an admin)
                        StringBuffer sb = new StringBuffer("https://");
                        sb.append(host);
                        if (!port.equals("443")) {
                            sb.append(":" + port);
                        }
                        sb.append("/duradmin/spaces/sm/");
                        sb.append(store.getStoreId());
                        sb.append("/" + spaceName + "/" + fileName);
                        statusObject.add(DatasetVersion.ARCHIVAL_STATUS, DatasetVersion.ARCHIVAL_STATUS_SUCCESS);
                        statusObject.add(DatasetVersion.ARCHIVAL_STATUS_MESSAGE, sb.toString());
                        
                        logger.fine("DuraCloud Submission step complete: " + sb.toString());
                    } catch (ContentStoreException | IOException e) {
                        // TODO Auto-generated catch block
                        logger.warning(e.getMessage());
                        e.printStackTrace();
                        return new Failure("Error in transferring file to DuraCloud",
                                "DuraCloud Submission Failure: archive file not transferred");
                    } catch (InterruptedException e) {
                        logger.warning(e.getLocalizedMessage());
                        e.printStackTrace();
                    }
                } catch (ContentStoreException e) {
                    logger.warning(e.getMessage());
                    e.printStackTrace();
                    String mesg = "DuraCloud Submission Failure";
                    if (!(1 == dv.getVersion()) || !(0 == dv.getMinorVersionNumber())) {
                        mesg = mesg + ": Prior Version archiving not yet complete?";
                    }
                    return new Failure("Unable to create DuraCloud space with name: " + baseFileName, mesg);
                } catch (NoSuchAlgorithmException e) {
                    logger.severe("MD5 MessageDigest not available!");
                }
                finally {
                    dv.setArchivalCopyLocation(statusObject.build().toString());
                }
            } else {
                logger.warning(
                        "DuraCloud Submision Workflow aborted: Dataset locked for finalizePublication, or because file validation failed");
                return new Failure("Dataset locked");
            }
            return WorkflowStepResult.OK;
        } else {
            return new Failure("DuraCloud Submission not configured - no \":DuraCloudHost\".");
        }
    }
}

1	package edu.harvard.iq.dataverse.engine.command.impl;
2
3	import edu.harvard.iq.dataverse.Dataset;
4	import edu.harvard.iq.dataverse.DatasetVersion;
5	import edu.harvard.iq.dataverse.DatasetLock.Reason;
6	import edu.harvard.iq.dataverse.authorization.Permission;
7	import edu.harvard.iq.dataverse.authorization.users.ApiToken;
8	import edu.harvard.iq.dataverse.engine.command.Command;
9	import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
10	import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
11	import edu.harvard.iq.dataverse.workflow.step.Failure;
12	import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult;
13
14	import java.io.IOException;
15	import java.io.PipedInputStream;
16	import java.io.PipedOutputStream;
17	import java.nio.charset.Charset;
18	import java.security.DigestInputStream;
19	import java.security.MessageDigest;
20	import java.security.NoSuchAlgorithmException;
21	import java.util.Map;
22	import java.util.logging.Logger;
23
24	import jakarta.json.Json;
25	import jakarta.json.JsonObjectBuilder;
26
27	import org.apache.commons.codec.binary.Hex;
28	import org.duracloud.client.ContentStore;
29	import org.duracloud.client.ContentStoreManager;
30	import org.duracloud.client.ContentStoreManagerImpl;
31	import org.duracloud.common.model.Credential;
32	import org.duracloud.error.ContentStoreException;
33
34	@RequiredPermissions(Permission.PublishDataset)
35	public class DuraCloudSubmitToArchiveCommand extends AbstractSubmitToArchiveCommand implements Command<DatasetVersion> {
36
UNCOV 37	private static final Logger logger = Logger.getLogger(DuraCloudSubmitToArchiveCommand.class.getName());	×
38	private static final String DEFAULT_PORT = "443";
39	private static final String DEFAULT_CONTEXT = "durastore";
40	private static final String DURACLOUD_PORT = ":DuraCloudPort";
41	private static final String DURACLOUD_HOST = ":DuraCloudHost";
42	private static final String DURACLOUD_CONTEXT = ":DuraCloudContext";
43
44
45	public DuraCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) {
46	super(aRequest, version);	×
UNCOV 47	}	×
48
49	@Override
50	public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken token,
51	Map<String, String> requestedSettings) {
52
53	String port = requestedSettings.get(DURACLOUD_PORT) != null ? requestedSettings.get(DURACLOUD_PORT)	×
54	: DEFAULT_PORT;	×
55	String dpnContext = requestedSettings.get(DURACLOUD_CONTEXT) != null ? requestedSettings.get(DURACLOUD_CONTEXT)	×
56	: DEFAULT_CONTEXT;	×
UNCOV 57	String host = requestedSettings.get(DURACLOUD_HOST);	×
58
59	if (host != null) {	×
UNCOV 60	Dataset dataset = dv.getDataset();	×
61	// ToDo - change after HDC 3A changes to status reporting
62	// This will make the archivalCopyLocation non-null after a failure which should
63	// stop retries
64
65	if (dataset.getLockFor(Reason.finalizePublication) == null	×
UNCOV 66	&& dataset.getLockFor(Reason.FileValidationFailed) == null) {	×
67	// Use Duracloud client classes to login
68	ContentStoreManager storeManager = new ContentStoreManagerImpl(host, port, dpnContext);	×
69	Credential credential = new Credential(System.getProperty("duracloud.username"),	×
70	System.getProperty("duracloud.password"));	×
UNCOV 71	storeManager.login(credential);	×
72	/*
73	* Aliases can contain upper case characters which are not allowed in space
74	* names. Similarly, aliases can contain '_' which isn't allowed in a space
75	* name. The line below replaces any upper case chars with lowercase and
76	* replaces any '_' with '.-' . The '-' after the dot assures we don't break the
77	* rule that
78	* "The last period in a aspace may not immediately be followed by a number".
79	* (Although we could check, it seems better to just add '.-' all the time.As
80	* written the replaceAll will also change any chars not valid in a spaceName to
81	* '.' which would avoid code breaking if the alias constraints change. That
82	* said, this line may map more than one alias to the same spaceName, e.g.
83	* "test" and "Test" aliases both map to the "test" space name. This does not
84	* break anything but does potentially put bags from more than one collection in
85	* the same space.
86	*/
87	String spaceName = dataset.getOwner().getAlias().toLowerCase().replaceAll("[^a-z0-9-]", ".dcsafe");	×
88	String baseFileName = dataset.getGlobalId().asString().replace(':', '-').replace('/', '-')	×
UNCOV 89	.replace('.', '-').toLowerCase() + "_v" + dv.getFriendlyVersionNumber();	×
90
91	ContentStore store;
92	//Set a failure status that will be updated if we succeed
93	JsonObjectBuilder statusObject = Json.createObjectBuilder();	×
94	statusObject.add(DatasetVersion.ARCHIVAL_STATUS, DatasetVersion.ARCHIVAL_STATUS_FAILURE);	×
UNCOV 95	statusObject.add(DatasetVersion.ARCHIVAL_STATUS_MESSAGE, "Bag not transferred");	×
96
97	try {
98	/*
99	* If there is a failure in creating a space, it is likely that a prior version
100	* has not been fully processed (snapshot created, archiving completed and files
101	* and space deleted - currently manual operations done at the project's
102	* duracloud website)
103	*/
UNCOV 104	store = storeManager.getPrimaryContentStore();	×
105	// Create space to copy archival files to
106	if (!store.spaceExists(spaceName)) {	×
UNCOV 107	store.createSpace(spaceName);	×
108	}
UNCOV 109	String dataciteXml = getDataCiteXml(dv);	×
110
111	MessageDigest messageDigest = MessageDigest.getInstance("MD5");	×
112	try (PipedInputStream dataciteIn = new PipedInputStream();	×
UNCOV 113	DigestInputStream digestInputStream = new DigestInputStream(dataciteIn, messageDigest)) {	×
114	// Add datacite.xml file
115
UNCOV 116	Thread dcThread = new Thread(new Runnable() {	×
117	public void run() {
UNCOV 118	try (PipedOutputStream dataciteOut = new PipedOutputStream(dataciteIn)) {	×
119
120	dataciteOut.write(dataciteXml.getBytes(Charset.forName("utf-8")));	×
121	dataciteOut.close();	×
122	success=true;	×
123	} catch (Exception e) {	×
UNCOV 124	logger.severe("Error creating datacite.xml: " + e.getMessage());	×
125	// TODO Auto-generated catch block
126	e.printStackTrace();	×
127	}	×
UNCOV 128	}	×
129	});
UNCOV 130	dcThread.start();	×
131	// Have seen Pipe Closed errors for other archivers when used as a workflow
132	// without this delay loop
133	int i = 0;	×
134	while (digestInputStream.available() <= 0 && i < 100) {	×
135	Thread.sleep(10);	×
UNCOV 136	i++;	×
137	}
UNCOV 138	String checksum = store.addContent(spaceName, baseFileName + "_datacite.xml", digestInputStream,	×
139	-1l, null, null, null);
140	logger.fine("Content: datacite.xml added with checksum: " + checksum);	×
141	dcThread.join();	×
142	String localchecksum = Hex.encodeHexString(digestInputStream.getMessageDigest().digest());	×
143	if (!success \|\| !checksum.equals(localchecksum)) {	×
144	logger.severe("Failure on " + baseFileName);	×
UNCOV 145	logger.severe(success ? checksum + " not equal to " + localchecksum : "failed to transfer to DuraCloud");	×
146	try {
147	store.deleteContent(spaceName, baseFileName + "_datacite.xml");	×
148	} catch (ContentStoreException cse) {	×
149	logger.warning(cse.getMessage());	×
150	}	×
UNCOV 151	return new Failure("Error in transferring DataCite.xml file to DuraCloud",	×
152	"DuraCloud Submission Failure: incomplete metadata transfer");
153	}
154
155	// Store BagIt file
156	success = false;	×
UNCOV 157	String fileName = baseFileName + ".zip";	×
158
159	// Add BagIt ZIP file
160	// Although DuraCloud uses SHA-256 internally, it's API uses MD5 to verify the
161	// transfer
162
163	messageDigest = MessageDigest.getInstance("MD5");	×
164	try (PipedInputStream in = new PipedInputStream(100000);	×
165	DigestInputStream digestInputStream2 = new DigestInputStream(in, messageDigest)) {	×
166	Thread bagThread = startBagThread(dv, in, digestInputStream2, dataciteXml, token);	×
167	checksum = store.addContent(spaceName, fileName, digestInputStream2, -1l, null, null, null);	×
168	bagThread.join();	×
169	if (success) {	×
170	logger.fine("Content: " + fileName + " added with checksum: " + checksum);	×
UNCOV 171	localchecksum = Hex.encodeHexString(digestInputStream2.getMessageDigest().digest());	×
172	}
173	if (!success \|\| !checksum.equals(localchecksum)) {	×
174	logger.severe("Failure on " + fileName);	×
UNCOV 175	logger.severe(success ? checksum + " not equal to " + localchecksum : "failed to transfer to DuraCloud");	×
176	try {
177	store.deleteContent(spaceName, fileName);	×
178	store.deleteContent(spaceName, baseFileName + "_datacite.xml");	×
179	} catch (ContentStoreException cse) {	×
180	logger.warning(cse.getMessage());	×
181	}	×
UNCOV 182	return new Failure("Error in transferring Zip file to DuraCloud",	×
183	"DuraCloud Submission Failure: incomplete archive transfer");
184	}
UNCOV 185	}	×
186
UNCOV 187	logger.fine("DuraCloud Submission step: Content Transferred");	×
188
189	// Document the location of dataset archival copy location (actually the URL
190	// where you can
191	// view it as an admin)
192	StringBuffer sb = new StringBuffer("https://");	×
193	sb.append(host);	×
194	if (!port.equals("443")) {	×
UNCOV 195	sb.append(":" + port);	×
196	}
197	sb.append("/duradmin/spaces/sm/");	×
198	sb.append(store.getStoreId());	×
199	sb.append("/" + spaceName + "/" + fileName);	×
200	statusObject.add(DatasetVersion.ARCHIVAL_STATUS, DatasetVersion.ARCHIVAL_STATUS_SUCCESS);	×
UNCOV 201	statusObject.add(DatasetVersion.ARCHIVAL_STATUS_MESSAGE, sb.toString());	×
202
203	logger.fine("DuraCloud Submission step complete: " + sb.toString());	×
UNCOV 204	} catch (ContentStoreException \| IOException e) {	×
205	// TODO Auto-generated catch block
206	logger.warning(e.getMessage());	×
207	e.printStackTrace();	×
UNCOV 208	return new Failure("Error in transferring file to DuraCloud",	×
209	"DuraCloud Submission Failure: archive file not transferred");
210	} catch (InterruptedException e) {	×
211	logger.warning(e.getLocalizedMessage());	×
212	e.printStackTrace();	×
213	}	×
214	} catch (ContentStoreException e) {	×
215	logger.warning(e.getMessage());	×
216	e.printStackTrace();	×
217	String mesg = "DuraCloud Submission Failure";	×
218	if (!(1 == dv.getVersion()) \|\| !(0 == dv.getMinorVersionNumber())) {	×
UNCOV 219	mesg = mesg + ": Prior Version archiving not yet complete?";	×
220	}
221	return new Failure("Unable to create DuraCloud space with name: " + baseFileName, mesg);	×
222	} catch (NoSuchAlgorithmException e) {	×
UNCOV 223	logger.severe("MD5 MessageDigest not available!");	×
224	}
225	finally {
UNCOV 226	dv.setArchivalCopyLocation(statusObject.build().toString());	×
227	}
228	} else {	×
UNCOV 229	logger.warning(	×
230	"DuraCloud Submision Workflow aborted: Dataset locked for finalizePublication, or because file validation failed");
UNCOV 231	return new Failure("Dataset locked");	×
232	}
UNCOV 233	return WorkflowStepResult.OK;	×
234	} else {
UNCOV 235	return new Failure("DuraCloud Submission not configured - no \":DuraCloudHost\".");	×
236	}
237	}
238	}

IQSS / dataverse / #22987

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous