#22985

Committed 23 Aug 2024 06:32PM CUT coverage: 20.61% (-0.2%) from 20.791%

Build # #22985

Build Type

Pull #10781

github

Committed by

landreev

Commit Message

added an upfront locks check to the /addGlobusFiles api #10623

Pull Request Pull Request #10781: Improved handling of Globus uploads

Run Details

4 of 417 new or added lines in 15 files covered. (0.96%)

4194 existing lines in 35 files now uncovered.

17388 of 84365 relevant lines covered (20.61%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

37.23

/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractDatasetCommand.java

package edu.harvard.iq.dataverse.engine.command.impl;

import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetField;
import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
import edu.harvard.iq.dataverse.DatasetVersion;
import edu.harvard.iq.dataverse.DatasetVersionDifference;
import edu.harvard.iq.dataverse.DatasetVersionUser;
import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.MetadataBlock;
import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
import edu.harvard.iq.dataverse.engine.command.CommandContext;
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
import edu.harvard.iq.dataverse.pidproviders.PidProvider;
import edu.harvard.iq.dataverse.pidproviders.PidUtil;
import edu.harvard.iq.dataverse.util.BundleUtil;

import java.sql.Timestamp;
import java.util.Date;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import static java.util.stream.Collectors.joining;

import jakarta.ejb.EJB;
import jakarta.validation.ConstraintViolation;
import edu.harvard.iq.dataverse.settings.JvmSettings;

/**
 *
 * Base class for commands that deal with {@code Dataset}s.Mainly here as a code
 * re-use mechanism.
 *
 * @author michael
 * @param <T> The type of the command's result. Normally {@link Dataset}.
 */
public abstract class AbstractDatasetCommand<T> extends AbstractCommand<T> {

    private static final Logger logger = Logger.getLogger(AbstractDatasetCommand.class.getName());
    private static final int FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT = 2 ^ 8;
    private Dataset dataset;
    private final Timestamp timestamp = new Timestamp(new Date().getTime());

    public AbstractDatasetCommand(DataverseRequest aRequest, Dataset aDataset, Dataverse parent) {
        super(aRequest, parent);
        if (aDataset == null) {
            throw new IllegalArgumentException("aDataset cannot be null");
        }
        dataset = aDataset;
    }

    public AbstractDatasetCommand(DataverseRequest aRequest, Dataset aDataset) {
        super(aRequest, aDataset);
        if (aDataset == null) {
            throw new IllegalArgumentException("aDataset cannot be null");
        }
        dataset = aDataset;
    }

    /**
     * Creates/updates the {@link DatasetVersionUser} for our {@link #dataset}. After
     * calling this method, there is a {@link DatasetUser} object connecting
     * {@link #dataset} and the {@link AuthenticatedUser} who issued this
     * command, with the {@code lastUpdate} field containing {@link #timestamp}.
     *
     * @param ctxt The command context in which this command runs.
     */
    protected void updateDatasetUser(CommandContext ctxt) {
        DatasetVersionUser datasetDataverseUser = ctxt.datasets().getDatasetVersionUser(getDataset().getLatestVersion(), getUser());

        if (datasetDataverseUser != null) {
            // Update existing dataset-user
            datasetDataverseUser.setLastUpdateDate(getTimestamp());
            ctxt.em().merge(datasetDataverseUser);

        } else {
            // create a new dataset-user
            createDatasetUser(ctxt);
        }
    }
    
    protected void createDatasetUser(CommandContext ctxt) {
        DatasetVersionUser datasetDataverseUser = new DatasetVersionUser();
        datasetDataverseUser.setDatasetVersion(getDataset().getLatestVersion());
        datasetDataverseUser.setLastUpdateDate(getTimestamp());
        datasetDataverseUser.setAuthenticatedUser((AuthenticatedUser) getUser());
        ctxt.em().persist(datasetDataverseUser);
    }
    
    /**
     * Validates the fields of the {@link DatasetVersion} passed. Throws an
     * informational error if validation fails.
     *
     * @param dsv The dataset version whose fields we validate
     * @param lenient when {@code true}, invalid fields are populated with N/A
     * value.
     * @throws CommandException if and only if {@code lenient=false}, and field
     * validation failed.
     */
    protected void validateOrDie(DatasetVersion dsv, Boolean lenient) throws CommandException {
        Set<ConstraintViolation> constraintViolations = dsv.validate();
        if (!constraintViolations.isEmpty()) {
            if (lenient) {
                // populate invalid fields with N/A
                constraintViolations.stream()
                    .filter(cv -> cv.getRootBean() instanceof DatasetField)
                    .map(cv -> ((DatasetField) cv.getRootBean()))
                    .forEach(f -> f.setSingleValue(DatasetField.NA_VALUE));

            } else {
                // explode with a helpful message
                String validationMessage = constraintViolations.stream()
                    .map(cv -> cv.getMessage() + " (Invalid value:" + cv.getInvalidValue() + ")")
                    .collect(joining(", ", "Validation Failed: ", "."));
                
                validationMessage  += constraintViolations.stream()
                    .filter(cv -> cv.getRootBean() instanceof TermsOfUseAndAccess)
                    .map(cv -> cv.toString());
                
                for (ConstraintViolation cv : constraintViolations){
                    if (cv.getRootBean() instanceof TermsOfUseAndAccess){
                        throw new IllegalCommandException(validationMessage,  this);
                    }
                }

                throw new IllegalCommandException(validationMessage, this);
            }
        }
    }



    /**
     * Whether it's EZID or DataCite, if the registration is refused because the
     * identifier already exists, we'll generate another one and try to register
     * again... but only up to some reasonably high number of times - so that we
     * don't go into an infinite loop here, if EZID is giving us these duplicate
     * messages in error.
     *
     * (and we do want the limit to be a "reasonably high" number! true, if our
     * identifiers are randomly generated strings, then it is highly unlikely
     * that we'll ever run into a duplicate race condition repeatedly; but if
     * they are sequential numeric values, than it is entirely possible that a
     * large enough number of values will be legitimately registered by another
     * entity sharing the same authority...)
     *
     * @param theDataset
     * @param ctxt
     * @throws CommandException
     */
    protected void registerExternalIdentifier(Dataset theDataset, CommandContext ctxt, boolean retry) throws CommandException {
        if (!theDataset.isIdentifierRegistered()) {
            PidProvider pidProvider = PidUtil.getPidProvider(theDataset.getGlobalId().getProviderId());
            if ( pidProvider != null ) {
                try {
                    if (pidProvider.alreadyRegistered(theDataset)) {
                        int attempts = 0;
                        if(retry) {
                            do  {
                                pidProvider.generatePid(theDataset);
                                logger.log(Level.INFO, "Attempting to register external identifier for dataset {0} (trying: {1}).",
                                    new Object[]{theDataset.getId(), theDataset.getIdentifier()});
                                attempts++;
                            } while (pidProvider.alreadyRegistered(theDataset) && attempts <= FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT);
                        }
                        if(!retry) {
                            logger.warning("Reserving PID for: "  + getDataset().getId() + " during publication failed.");
                            throw new IllegalCommandException(BundleUtil.getStringFromBundle("publishDatasetCommand.pidNotReserved"), this);
                        }
                        if(attempts > FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT) {
                            //Didn't work - we existed the loop with too many tries
                            throw new CommandExecutionException("This dataset may not be published because its identifier is already in use by another dataset; "
                                + "gave up after " + attempts + " attempts. Current (last requested) identifier: " + theDataset.getIdentifier(), this);
                        }
                    }
                    // Invariant: Dataset identifier does not exist in the remote registry
                    try {
                        pidProvider.createIdentifier(theDataset);
                        theDataset.setGlobalIdCreateTime(getTimestamp());
                        theDataset.setIdentifierRegistered(true);
                    } catch (Throwable ex) {
                        logger.info("Call to globalIdServiceBean.createIdentifier failed: " + ex);
                    }

                } catch (Throwable e) {
                    throw new CommandException(BundleUtil.getStringFromBundle("dataset.publish.error", pidProvider.getProviderInformation()), this);
                }
            } else {
                throw new IllegalCommandException("This dataset may not be published because its id registry service is not supported.", this);
            }

        }
    }

    protected Dataset getDataset() {
        return dataset;
    }

    public void setDataset(Dataset dataset) {
        this.dataset = dataset;
    }

    /**
     * The time the command instance was created. Note: This is not the time the
     * command was submitted to the engine. If the difference can be large
     * enough, consider using another timestamping mechanism. This is a
     * convenience method fit for most cases.
     *
     * @return the time {@code this} command was created.
     */
    protected Timestamp getTimestamp() {
        return timestamp;
    }

    protected void checkSystemMetadataKeyIfNeeded(DatasetVersion newVersion, DatasetVersion persistedVersion) throws IllegalCommandException {
        Set<MetadataBlock> changedMDBs = DatasetVersionDifference.getBlocksWithChanges(newVersion, persistedVersion);
        for (MetadataBlock mdb : changedMDBs) {
            logger.fine(mdb.getName() + " has been changed");
            String smdbString = JvmSettings.MDB_SYSTEM_KEY_FOR.lookupOptional(mdb.getName())
                    .orElse(null);
            if (smdbString != null) {
                logger.fine("Found key: " + smdbString);
                String mdKey = getRequest().getSystemMetadataBlockKeyFor(mdb.getName());
                logger.fine("Found supplied key: " + mdKey);
                if (mdKey == null || !mdKey.equalsIgnoreCase(smdbString)) {
                    throw new IllegalCommandException("Updating system metadata in block " + mdb.getName() + " requires a valid key", this);
                }
            }
        }
    }

    protected void registerExternalVocabValuesIfAny(CommandContext ctxt, DatasetVersion newVersion) {
        for (DatasetField df : newVersion.getFlatDatasetFields()) {
            logger.fine("Found id: " + df.getDatasetFieldType().getId());
            if (ctxt.dsField().getCVocConf(true).containsKey(df.getDatasetFieldType().getId())) {
                ctxt.dsField().registerExternalVocabValues(df);
            }
        }
    }
}

1	package edu.harvard.iq.dataverse.engine.command.impl;
2
3	import edu.harvard.iq.dataverse.Dataset;
4	import edu.harvard.iq.dataverse.DatasetField;
5	import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
6	import edu.harvard.iq.dataverse.DatasetVersion;
7	import edu.harvard.iq.dataverse.DatasetVersionDifference;
8	import edu.harvard.iq.dataverse.DatasetVersionUser;
9	import edu.harvard.iq.dataverse.Dataverse;
10	import edu.harvard.iq.dataverse.MetadataBlock;
11	import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
12	import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
13	import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
14	import edu.harvard.iq.dataverse.engine.command.CommandContext;
15	import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
16	import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
17	import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
18	import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
19	import edu.harvard.iq.dataverse.pidproviders.PidProvider;
20	import edu.harvard.iq.dataverse.pidproviders.PidUtil;
21	import edu.harvard.iq.dataverse.util.BundleUtil;
22
23	import java.sql.Timestamp;
24	import java.util.Date;
25	import java.util.Set;
26	import java.util.logging.Level;
27	import java.util.logging.Logger;
28	import static java.util.stream.Collectors.joining;
29
30	import jakarta.ejb.EJB;
31	import jakarta.validation.ConstraintViolation;
32	import edu.harvard.iq.dataverse.settings.JvmSettings;
33
34	/**
35	*
36	* Base class for commands that deal with {@code Dataset}s.Mainly here as a code
37	* re-use mechanism.
38	*
39	* @author michael
40	* @param <T> The type of the command's result. Normally {@link Dataset}.
41	*/
42	public abstract class AbstractDatasetCommand<T> extends AbstractCommand<T> {
43
44	private static final Logger logger = Logger.getLogger(AbstractDatasetCommand.class.getName());	1✔
45	private static final int FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT = 2 ^ 8;
46	private Dataset dataset;
47	private final Timestamp timestamp = new Timestamp(new Date().getTime());	1✔
48
49	public AbstractDatasetCommand(DataverseRequest aRequest, Dataset aDataset, Dataverse parent) {
50	super(aRequest, parent);	1✔
51	if (aDataset == null) {	1✔
52	throw new IllegalArgumentException("aDataset cannot be null");	1✔
53	}
UNCOV 54	dataset = aDataset;	×
UNCOV 55	}	×
56
57	public AbstractDatasetCommand(DataverseRequest aRequest, Dataset aDataset) {
58	super(aRequest, aDataset);	1✔
59	if (aDataset == null) {	1✔
60	throw new IllegalArgumentException("aDataset cannot be null");	1✔
61	}
62	dataset = aDataset;	1✔
63	}	1✔
64
65	/**
66	* Creates/updates the {@link DatasetVersionUser} for our {@link #dataset}. After
67	* calling this method, there is a {@link DatasetUser} object connecting
68	* {@link #dataset} and the {@link AuthenticatedUser} who issued this
69	* command, with the {@code lastUpdate} field containing {@link #timestamp}.
70	*
71	* @param ctxt The command context in which this command runs.
72	*/
73	protected void updateDatasetUser(CommandContext ctxt) {
74	DatasetVersionUser datasetDataverseUser = ctxt.datasets().getDatasetVersionUser(getDataset().getLatestVersion(), getUser());	1✔
75
76	if (datasetDataverseUser != null) {	1✔
77	// Update existing dataset-user
UNCOV 78	datasetDataverseUser.setLastUpdateDate(getTimestamp());	×
UNCOV 79	ctxt.em().merge(datasetDataverseUser);	×
80
81	} else {
82	// create a new dataset-user
83	createDatasetUser(ctxt);	1✔
84	}
85	}	1✔
86
87	protected void createDatasetUser(CommandContext ctxt) {
88	DatasetVersionUser datasetDataverseUser = new DatasetVersionUser();	1✔
89	datasetDataverseUser.setDatasetVersion(getDataset().getLatestVersion());	1✔
90	datasetDataverseUser.setLastUpdateDate(getTimestamp());	1✔
91	datasetDataverseUser.setAuthenticatedUser((AuthenticatedUser) getUser());	1✔
92	ctxt.em().persist(datasetDataverseUser);	1✔
93	}	1✔
94
95	/**
96	* Validates the fields of the {@link DatasetVersion} passed. Throws an
97	* informational error if validation fails.
98	*
99	* @param dsv The dataset version whose fields we validate
100	* @param lenient when {@code true}, invalid fields are populated with N/A
101	* value.
102	* @throws CommandException if and only if {@code lenient=false}, and field
103	* validation failed.
104	*/
105	protected void validateOrDie(DatasetVersion dsv, Boolean lenient) throws CommandException {
106	Set<ConstraintViolation> constraintViolations = dsv.validate();	1✔
107	if (!constraintViolations.isEmpty()) {	1✔
UNCOV 108	if (lenient) {	×
109	// populate invalid fields with N/A
UNCOV 110	constraintViolations.stream()	×
111	.filter(cv -> cv.getRootBean() instanceof DatasetField)	×
UNCOV 112	.map(cv -> ((DatasetField) cv.getRootBean()))	×
113	.forEach(f -> f.setSingleValue(DatasetField.NA_VALUE));	×
114
115	} else {
116	// explode with a helpful message
UNCOV 117	String validationMessage = constraintViolations.stream()	×
UNCOV 118	.map(cv -> cv.getMessage() + " (Invalid value:" + cv.getInvalidValue() + ")")	×
UNCOV 119	.collect(joining(", ", "Validation Failed: ", "."));	×
120
121	validationMessage += constraintViolations.stream()	×
122	.filter(cv -> cv.getRootBean() instanceof TermsOfUseAndAccess)	×
UNCOV 123	.map(cv -> cv.toString());	×
124
125	for (ConstraintViolation cv : constraintViolations){	×
126	if (cv.getRootBean() instanceof TermsOfUseAndAccess){	×
UNCOV 127	throw new IllegalCommandException(validationMessage, this);	×
128	}
129	}	×
130
UNCOV 131	throw new IllegalCommandException(validationMessage, this);	×
132	}
133	}
134	}	1✔
135
136
137
138	/**
139	* Whether it's EZID or DataCite, if the registration is refused because the
140	* identifier already exists, we'll generate another one and try to register
141	* again... but only up to some reasonably high number of times - so that we
142	* don't go into an infinite loop here, if EZID is giving us these duplicate
143	* messages in error.
144	*
145	* (and we do want the limit to be a "reasonably high" number! true, if our
146	* identifiers are randomly generated strings, then it is highly unlikely
147	* that we'll ever run into a duplicate race condition repeatedly; but if
148	* they are sequential numeric values, than it is entirely possible that a
149	* large enough number of values will be legitimately registered by another
150	* entity sharing the same authority...)
151	*
152	* @param theDataset
153	* @param ctxt
154	* @throws CommandException
155	*/
156	protected void registerExternalIdentifier(Dataset theDataset, CommandContext ctxt, boolean retry) throws CommandException {
UNCOV 157	if (!theDataset.isIdentifierRegistered()) {	×
UNCOV 158	PidProvider pidProvider = PidUtil.getPidProvider(theDataset.getGlobalId().getProviderId());	×
UNCOV 159	if ( pidProvider != null ) {	×
160	try {
161	if (pidProvider.alreadyRegistered(theDataset)) {	×
162	int attempts = 0;	×
UNCOV 163	if(retry) {	×
164	do {
165	pidProvider.generatePid(theDataset);	×
166	logger.log(Level.INFO, "Attempting to register external identifier for dataset {0} (trying: {1}).",	×
UNCOV 167	new Object[]{theDataset.getId(), theDataset.getIdentifier()});	×
168	attempts++;	×
169	} while (pidProvider.alreadyRegistered(theDataset) && attempts <= FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT);	×
170	}
171	if(!retry) {	×
172	logger.warning("Reserving PID for: " + getDataset().getId() + " during publication failed.");	×
UNCOV 173	throw new IllegalCommandException(BundleUtil.getStringFromBundle("publishDatasetCommand.pidNotReserved"), this);	×
174	}
175	if(attempts > FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT) {	×
176	//Didn't work - we existed the loop with too many tries
UNCOV 177	throw new CommandExecutionException("This dataset may not be published because its identifier is already in use by another dataset; "	×
178	+ "gave up after " + attempts + " attempts. Current (last requested) identifier: " + theDataset.getIdentifier(), this);	×
179	}
180	}
181	// Invariant: Dataset identifier does not exist in the remote registry
182	try {
UNCOV 183	pidProvider.createIdentifier(theDataset);	×
UNCOV 184	theDataset.setGlobalIdCreateTime(getTimestamp());	×
185	theDataset.setIdentifierRegistered(true);	×
186	} catch (Throwable ex) {	×
187	logger.info("Call to globalIdServiceBean.createIdentifier failed: " + ex);	×
188	}	×
189
190	} catch (Throwable e) {	×
UNCOV 191	throw new CommandException(BundleUtil.getStringFromBundle("dataset.publish.error", pidProvider.getProviderInformation()), this);	×
192	}	×
193	} else {
194	throw new IllegalCommandException("This dataset may not be published because its id registry service is not supported.", this);	×
195	}
196
197	}
UNCOV 198	}	×
199
200	protected Dataset getDataset() {
201	return dataset;	1✔
202	}
203
204	public void setDataset(Dataset dataset) {
UNCOV 205	this.dataset = dataset;	×
UNCOV 206	}	×
207
208	/**
209	* The time the command instance was created. Note: This is not the time the
210	* command was submitted to the engine. If the difference can be large
211	* enough, consider using another timestamping mechanism. This is a
212	* convenience method fit for most cases.
213	*
214	* @return the time {@code this} command was created.
215	*/
216	protected Timestamp getTimestamp() {
217	return timestamp;	1✔
218	}
219
220	protected void checkSystemMetadataKeyIfNeeded(DatasetVersion newVersion, DatasetVersion persistedVersion) throws IllegalCommandException {
221	Set<MetadataBlock> changedMDBs = DatasetVersionDifference.getBlocksWithChanges(newVersion, persistedVersion);	1✔
222	for (MetadataBlock mdb : changedMDBs) {	1✔
223	logger.fine(mdb.getName() + " has been changed");	1✔
224	String smdbString = JvmSettings.MDB_SYSTEM_KEY_FOR.lookupOptional(mdb.getName())	1✔
225	.orElse(null);	1✔
226	if (smdbString != null) {	1✔
227	logger.fine("Found key: " + smdbString);	×
228	String mdKey = getRequest().getSystemMetadataBlockKeyFor(mdb.getName());	×
229	logger.fine("Found supplied key: " + mdKey);	×
230	if (mdKey == null \|\| !mdKey.equalsIgnoreCase(smdbString)) {	×
231	throw new IllegalCommandException("Updating system metadata in block " + mdb.getName() + " requires a valid key", this);	×
232	}
233	}
234	}	1✔
235	}	1✔
236
237	protected void registerExternalVocabValuesIfAny(CommandContext ctxt, DatasetVersion newVersion) {
238	for (DatasetField df : newVersion.getFlatDatasetFields()) {	1✔
UNCOV 239	logger.fine("Found id: " + df.getDatasetFieldType().getId());	×
240	if (ctxt.dsField().getCVocConf(true).containsKey(df.getDatasetFieldType().getId())) {	×
UNCOV 241	ctxt.dsField().registerExternalVocabValues(df);	×
242	}
UNCOV 243	}	×
244	}	1✔
245	}

IQSS / dataverse / #22985

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous