#22002

Committed 01 Apr 2024 07:56PM CUT coverage: 20.716% (+0.5%) from 20.173%

Build # #22002

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #10453 from IQSS/develop

Merge 6.2 into master

Run Details

704 of 2679 new or added lines in 152 files covered. (26.28%)

81 existing lines in 49 files now uncovered.

17160 of 82836 relevant lines covered (20.72%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package edu.harvard.iq.dataverse.engine.command.impl;

import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DataFileTag;
import edu.harvard.iq.dataverse.DataTable;
import edu.harvard.iq.dataverse.DatasetVersion;
import edu.harvard.iq.dataverse.FileMetadata;
import edu.harvard.iq.dataverse.authorization.Permission;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.dataaccess.DataAccess;
import edu.harvard.iq.dataverse.dataaccess.StorageIO;
import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand;
import edu.harvard.iq.dataverse.engine.command.CommandContext;
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
import edu.harvard.iq.dataverse.util.FileUtil;

import java.io.IOException;
import java.util.Collections;
import java.util.logging.Logger;
import jakarta.persistence.Query;

/**
 *
 * @author skraffmi
 * @author Leonid Andreev 
 */
@RequiredPermissions({})
public class UningestFileCommand extends AbstractVoidCommand  {

    private static final Logger logger = Logger.getLogger(UningestFileCommand.class.getCanonicalName());
    final DataFile uningest;
    
    public UningestFileCommand(DataverseRequest aRequest, DataFile uningest) {
        super(aRequest, uningest);
        this.uningest = uningest;
    }

    @Override
    protected void executeImpl(CommandContext ctxt) throws CommandException {
        
        // first check if user is a superuser
        if ((!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser())) {
            throw new PermissionException("Uningest File can only be called by Superusers.", this,
                    Collections.singleton(Permission.EditDataset), uningest);
        }
        
        // is this actually a tabular data file?
        if (!uningest.isTabularData()) {
            throw new IllegalCommandException("UningestFileCommand called on a non-tabular data file (id="+uningest.getId()+")", this);
        }

        String originalFileName = uningest.getOriginalFileName();

        StorageIO<DataFile> dataAccess = null;
        // size of the stored original:
        Long storedOriginalFileSize;

        // Try to open the main storageIO for the file; look up the AUX file for 
        // the saved original and check its size:
        try {
            dataAccess = DataAccess.getStorageIO(uningest);
            dataAccess.open();
            storedOriginalFileSize = dataAccess.getAuxObjectSize(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION);
        } catch (IOException ioex) {
            String errorMessage = "Failed to open StorageIO for " + uningest.getStorageIdentifier() + " attempting to revert tabular ingest" +  " aborting. (";
            if (ioex.getMessage() != null) {
                errorMessage += "(" + ioex.getMessage() + ")";
            } else {
                errorMessage += "(IOException caught; no further information is available)";
            }
            logger.warning(errorMessage);
            throw new CommandException(errorMessage, this);
            
        } 
          
        // Try to revert the backup-as-Aux:
        // (i.e., try to overwrite the current, tabular file, with the stored 
        // original file:
        // (if this fails, we definitely want to abort the whole thing and bail!)
        // -- L.A. May 2018
        
        try {
            dataAccess.revertBackupAsAux(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION);
        } catch (IOException ioex) {
            String errorMessage = "Failed to revert backup as Aux for " + uningest.getStorageIdentifier() + " attempting to revert tabular ingest" +  " aborting. (";
            if (ioex.getMessage() != null) {
                errorMessage += "(" + ioex.getMessage() + ")";
            } else {
                errorMessage += "(IOException caught; no further information is available)";
            }
            logger.warning(errorMessage);
            throw new CommandException(errorMessage, this);
        } 
        
        // OK, we have successfully reverted the backup - now let's change 
        // all the attribute of the file that are stored in the database: 
        
        // the file size: 
        long archivalFileSize = uningest.getFilesize();
        uningest.setFilesize(storedOriginalFileSize);
        
        // original file format:
        String originalFileFormat = uningest.getDataTable().getOriginalFileFormat();
        uningest.setContentType(originalFileFormat);
        
        
        // Delete the DataTable database object hierarchy that stores
        // all the tabular metadata - (DataTable, DataVariable, SummaryStatistics
        // *and more* sub-objects:
        
        //removeSummaryStatistics(uningest, ctxt);
        DataTable dataTable = ctxt.em().find(DataTable.class, uningest.getDataTable().getId());
        ctxt.em().remove(dataTable);
        uningest.setDataTable(null);

        // remove the IngestReport associated with this datafile: 
        // (this is a single table entry; ok to just issue an explicit 
        // DELETE query for it - as there's no complex cascade to resolve)
        resetIngestStats(uningest, ctxt);
        
        //probably unnecessary - why would you add tags to a file and then say "oops this shouldn't have been ingested"?
        DataFileTag tag;
        for (DataFileTag tagLoop: uningest.getTags()){
            tag = ctxt.em().find(DataFileTag.class, tagLoop.getId());
            ctxt.em().remove(tag);
        }
        uningest.setTags(null);        
        // Do the DB merge:
        ctxt.em().merge(uningest); 
        
        // Modify the file name - which is stored in FileMetadata, and there
        // could be more than one: 
        
       // String originalExtension = FileUtil.generateOriginalExtension(originalFileFormat);
        for (FileMetadata fm : uningest.getFileMetadatas()) {
            
            fm.setLabel(originalFileName);
            ctxt.em().merge(fm);
            
            /* 
            getOriginalFileName method replaces this code
            String filename = fm.getLabel();
            String extensionToRemove = StringUtil.substringIncludingLast(filename, ".");
            if (StringUtil.nonEmpty(extensionToRemove)) {
                String newFileName = filename.replace(extensionToRemove, originalExtension);
                fm.setLabel(newFileName);
                ctxt.em().merge(fm);
            }
             */
            
            DatasetVersion dv = fm.getDatasetVersion();
            
            // And, while we are here, recalculate the UNF for this DatasetVersion:
            dv.setUNF(null);
            ctxt.em().merge(dv);
            ctxt.datasetVersion().fixMissingUnf(dv.getId().toString(), true);
        }

        try{
            dataAccess.deleteAllAuxObjects();
        } catch (IOException e){
            logger.warning("Io Exception deleting all aux objects : " + uningest.getId());
        }
        
        // Finally, adjust the recorded storage use for the ancestral 
        // DvObjectContainers (the parent dataset + all the parent collections
        // up to the root):
        if (archivalFileSize > 0) {
            ctxt.storageUse().incrementStorageSizeRecursively(uningest.getOwner().getId(), (0L - archivalFileSize));
        }
        
    }
    
    @Override
    public boolean onSuccess(CommandContext ctxt, Object r) {
        
        return true; 
    }
    
    private void resetIngestStats(DataFile uningest, CommandContext ctxt){
        
        Long fileid = uningest.getId();        
        Query query = ctxt.em().createQuery("DELETE from IngestReport as o where o.dataFile.id  =:fileid");
        query.setParameter("fileid", fileid);
        query.executeUpdate();       
        uningest.setIngestStatus("A".charAt(0));
        
    }  

}

1	/*
2	* To change this license header, choose License Headers in Project Properties.
3	* To change this template file, choose Tools \| Templates
4	* and open the template in the editor.
5	*/
6	package edu.harvard.iq.dataverse.engine.command.impl;
7
8	import edu.harvard.iq.dataverse.DataFile;
9	import edu.harvard.iq.dataverse.DataFileTag;
10	import edu.harvard.iq.dataverse.DataTable;
11	import edu.harvard.iq.dataverse.DatasetVersion;
12	import edu.harvard.iq.dataverse.FileMetadata;
13	import edu.harvard.iq.dataverse.authorization.Permission;
14	import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
15	import edu.harvard.iq.dataverse.dataaccess.DataAccess;
16	import edu.harvard.iq.dataverse.dataaccess.StorageIO;
17	import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand;
18	import edu.harvard.iq.dataverse.engine.command.CommandContext;
19	import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
20	import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
21	import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
22	import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
23	import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
24	import edu.harvard.iq.dataverse.util.FileUtil;
25
26	import java.io.IOException;
27	import java.util.Collections;
28	import java.util.logging.Logger;
29	import jakarta.persistence.Query;
30
31	/**
32	*
33	* @author skraffmi
34	* @author Leonid Andreev
35	*/
36	@RequiredPermissions({})
37	public class UningestFileCommand extends AbstractVoidCommand {
38
39	private static final Logger logger = Logger.getLogger(UningestFileCommand.class.getCanonicalName());	×
40	final DataFile uningest;
41
42	public UningestFileCommand(DataverseRequest aRequest, DataFile uningest) {
43	super(aRequest, uningest);	×
44	this.uningest = uningest;	×
45	}	×
46
47	@Override
48	protected void executeImpl(CommandContext ctxt) throws CommandException {
49
50	// first check if user is a superuser
NEW 51	if ((!(getUser() instanceof AuthenticatedUser) \|\| !getUser().isSuperuser())) {	×
NEW 52	throw new PermissionException("Uningest File can only be called by Superusers.", this,	×
NEW 53	Collections.singleton(Permission.EditDataset), uningest);	×
54	}
55
56	// is this actually a tabular data file?
57	if (!uningest.isTabularData()) {	×
58	throw new IllegalCommandException("UningestFileCommand called on a non-tabular data file (id="+uningest.getId()+")", this);	×
59	}
60
61	String originalFileName = uningest.getOriginalFileName();	×
62
63	StorageIO<DataFile> dataAccess = null;	×
64	// size of the stored original:
65	Long storedOriginalFileSize;
66
67	// Try to open the main storageIO for the file; look up the AUX file for
68	// the saved original and check its size:
69	try {
70	dataAccess = DataAccess.getStorageIO(uningest);	×
71	dataAccess.open();	×
72	storedOriginalFileSize = dataAccess.getAuxObjectSize(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION);	×
73	} catch (IOException ioex) {	×
74	String errorMessage = "Failed to open StorageIO for " + uningest.getStorageIdentifier() + " attempting to revert tabular ingest" + " aborting. (";	×
75	if (ioex.getMessage() != null) {	×
76	errorMessage += "(" + ioex.getMessage() + ")";	×
77	} else {
78	errorMessage += "(IOException caught; no further information is available)";	×
79	}
80	logger.warning(errorMessage);	×
81	throw new CommandException(errorMessage, this);	×
82
83	}	×
84
85	// Try to revert the backup-as-Aux:
86	// (i.e., try to overwrite the current, tabular file, with the stored
87	// original file:
88	// (if this fails, we definitely want to abort the whole thing and bail!)
89	// -- L.A. May 2018
90
91	try {
92	dataAccess.revertBackupAsAux(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION);	×
93	} catch (IOException ioex) {	×
94	String errorMessage = "Failed to revert backup as Aux for " + uningest.getStorageIdentifier() + " attempting to revert tabular ingest" + " aborting. (";	×
95	if (ioex.getMessage() != null) {	×
96	errorMessage += "(" + ioex.getMessage() + ")";	×
97	} else {
98	errorMessage += "(IOException caught; no further information is available)";	×
99	}
100	logger.warning(errorMessage);	×
101	throw new CommandException(errorMessage, this);	×
102	}	×
103
104	// OK, we have successfully reverted the backup - now let's change
105	// all the attribute of the file that are stored in the database:
106
107	// the file size:
108	long archivalFileSize = uningest.getFilesize();	×
109	uningest.setFilesize(storedOriginalFileSize);	×
110
111	// original file format:
112	String originalFileFormat = uningest.getDataTable().getOriginalFileFormat();	×
113	uningest.setContentType(originalFileFormat);	×
114
115
116	// Delete the DataTable database object hierarchy that stores
117	// all the tabular metadata - (DataTable, DataVariable, SummaryStatistics
118	// and more sub-objects:
119
120	//removeSummaryStatistics(uningest, ctxt);
121	DataTable dataTable = ctxt.em().find(DataTable.class, uningest.getDataTable().getId());	×
122	ctxt.em().remove(dataTable);	×
123	uningest.setDataTable(null);	×
124
125	// remove the IngestReport associated with this datafile:
126	// (this is a single table entry; ok to just issue an explicit
127	// DELETE query for it - as there's no complex cascade to resolve)
128	resetIngestStats(uningest, ctxt);	×
129
130	//probably unnecessary - why would you add tags to a file and then say "oops this shouldn't have been ingested"?
131	DataFileTag tag;
132	for (DataFileTag tagLoop: uningest.getTags()){	×
133	tag = ctxt.em().find(DataFileTag.class, tagLoop.getId());	×
134	ctxt.em().remove(tag);	×
135	}	×
136	uningest.setTags(null);	×
137	// Do the DB merge:
138	ctxt.em().merge(uningest);	×
139
140	// Modify the file name - which is stored in FileMetadata, and there
141	// could be more than one:
142
143	// String originalExtension = FileUtil.generateOriginalExtension(originalFileFormat);
144	for (FileMetadata fm : uningest.getFileMetadatas()) {	×
145
146	fm.setLabel(originalFileName);	×
147	ctxt.em().merge(fm);	×
148
149	/*
150	getOriginalFileName method replaces this code
151	String filename = fm.getLabel();
152	String extensionToRemove = StringUtil.substringIncludingLast(filename, ".");
153	if (StringUtil.nonEmpty(extensionToRemove)) {
154	String newFileName = filename.replace(extensionToRemove, originalExtension);
155	fm.setLabel(newFileName);
156	ctxt.em().merge(fm);
157	}
158	*/
159
160	DatasetVersion dv = fm.getDatasetVersion();	×
161
162	// And, while we are here, recalculate the UNF for this DatasetVersion:
163	dv.setUNF(null);	×
164	ctxt.em().merge(dv);	×
165	ctxt.datasetVersion().fixMissingUnf(dv.getId().toString(), true);	×
166	}	×
167
168	try{
169	dataAccess.deleteAllAuxObjects();	×
170	} catch (IOException e){	×
171	logger.warning("Io Exception deleting all aux objects : " + uningest.getId());	×
172	}	×
173
174	// Finally, adjust the recorded storage use for the ancestral
175	// DvObjectContainers (the parent dataset + all the parent collections
176	// up to the root):
177	if (archivalFileSize > 0) {	×
178	ctxt.storageUse().incrementStorageSizeRecursively(uningest.getOwner().getId(), (0L - archivalFileSize));	×
179	}
180
181	}	×
182
183	@Override
184	public boolean onSuccess(CommandContext ctxt, Object r) {
185
186	return true;	×
187	}
188
189	private void resetIngestStats(DataFile uningest, CommandContext ctxt){
190
191	Long fileid = uningest.getId();	×
192	Query query = ctxt.em().createQuery("DELETE from IngestReport as o where o.dataFile.id =:fileid");	×
193	query.setParameter("fileid", fileid);	×
194	query.executeUpdate();	×
195	uningest.setIngestStatus("A".charAt(0));	×
196
197	}	×
198
199	}

IQSS / dataverse / #22002

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous