1359

Committed 02 Apr 2024 09:47AM UTC coverage: 25.104%. First build

Build # 1359

Build Type

push

jenkins

Committed by

web-flow

Commit Message

Closes #2440: Improved shapefile handling (#2443)

* Closes #2440: Improved shapefile handler, error handling and simpler api, use commons-compress for extraction supporting unicode extra fields in zip

* review comments

Run Details

96 of 123 new or added lines in 6 files covered. (78.05%)

17423 of 69404 relevant lines covered (25.1%)

0.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.44

/dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/datafile/IngestServiceShapefileHelper.java

package edu.harvard.iq.dataverse.datafile;

import com.google.common.base.Preconditions;
import edu.harvard.iq.dataverse.util.ShapefileHandler;
import io.vavr.control.Try;
import org.apache.commons.io.FileUtils;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;



/**
 * Used by the IngestServiceBean to redistribute a zipped Shapefile*
 * <p>
 * - Accomplish the actions described in the ShapefileHandler function "rezipShapefileSets"
 * - Return a list of DataFile objects
 *
 * @author raprasad
 */
public class IngestServiceShapefileHelper implements Closeable {

    private static final Logger logger = Logger.getLogger(IngestServiceShapefileHelper.class.getCanonicalName());

    private final File zippedShapefile;
    private final File reZipFolder;
    private final File unZipFolder;

    // -------------------- CONSTRUCTOR --------------------

    /**
     * Constructor that accepts a file object
     */
    public IngestServiceShapefileHelper(File zippedShapefile, File workingFolderBase) {
        Preconditions.checkArgument(isValidFile(zippedShapefile));
        Preconditions.checkArgument(isValidFolder(workingFolderBase));

        this.zippedShapefile = zippedShapefile;
        String id = new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss-SSS").format(new Date());
        this.reZipFolder = getShapefileUnzipTempDirectory(workingFolderBase, "shp_" + id + "_rezip");
        this.unZipFolder = getShapefileUnzipTempDirectory(workingFolderBase, "shp_" + id + "_unzip");

    }

    // -------------------- LOGIC --------------------

    public List<File> processFile() {
        try {
            // (1) Use the ShapefileHandler to the .zip for a shapefile
            //
            ShapefileHandler shpHandler = new ShapefileHandler(zippedShapefile);
            if (!shpHandler.containsShapefile()) {
                logger.severe("Shapefile was incorrectly detected upon Ingest (FileUtil) and passed here");
                throw new IllegalStateException("Shapefile was incorrectly detected upon Ingest (FileUtil) and passed here");
            }

            //  (2) Rezip the shapefile pieces
            return shpHandler.reZipShapefileSets(unZipFolder, reZipFolder);
        } catch (Exception ex) {
            throw new IllegalStateException("Shapefile was not correctly unpacked/repacked", ex);
        }
    }

    @Override
    public void close() throws IOException {
        deleteDirectory(unZipFolder);
        deleteDirectory(reZipFolder);
    }

    // -------------------- PRIVATE --------------------

    private void deleteDirectory(File directory) {
        Try.run(() -> FileUtils.deleteDirectory(directory))
                .onFailure(ex -> logger.log(Level.SEVERE, "Error cleaning shapefile working directory:" + directory, ex));
    }

    private static File getShapefileUnzipTempDirectory(File tempDirectoryBase, String directoryName) {
        File datestampedFolder = new File(tempDirectoryBase, directoryName);
        if (!datestampedFolder.isDirectory()) {
            /* Note that "createDirectories()" must be used - not
             * "createDirectory()", to make sure all the parent
             * directories that may not yet exist are created as well.
             */
            try {
                Files.createDirectories(datestampedFolder.toPath());
            } catch (IOException ex) {
                throw new IllegalStateException("Failed to create temp. directory to unzip shapefile: " + datestampedFolder.toString(), ex);
            }
        }
        return datestampedFolder;
    }

    private boolean isValidFile(File fileObject) {

        if (fileObject == null) {
            logger.warning("fileObject was null");
            return false;
        }
        if (!fileObject.isFile()) {
            logger.warning("fileObject was not a file.  Failed \"isFile()\": " + fileObject.getAbsolutePath());
            return false;
        }
        return true;
    }


    private boolean isValidFolder(File fileObject) {

        if (fileObject == null) {
            logger.warning("fileObject was null");
            return false;
        }
        if (!fileObject.isDirectory()) {
            logger.warning("fileObject was not a directory.  Failed \"isFile()\": " + fileObject.getAbsolutePath());
            return false;
        }
        return true;
    }
}

1	package edu.harvard.iq.dataverse.datafile;
2
3	import com.google.common.base.Preconditions;
4	import edu.harvard.iq.dataverse.util.ShapefileHandler;
5	import io.vavr.control.Try;
6	import org.apache.commons.io.FileUtils;
7
8	import java.io.Closeable;
9	import java.io.File;
10	import java.io.IOException;
11	import java.nio.file.Files;
12	import java.text.SimpleDateFormat;
13	import java.util.Date;
14	import java.util.List;
15	import java.util.logging.Level;
16	import java.util.logging.Logger;
17
18
19
20	/**
21	* Used by the IngestServiceBean to redistribute a zipped Shapefile*
22	* <p>
23	* - Accomplish the actions described in the ShapefileHandler function "rezipShapefileSets"
24	* - Return a list of DataFile objects
25	*
26	* @author raprasad
27	*/
28	public class IngestServiceShapefileHelper implements Closeable {
29
30	private static final Logger logger = Logger.getLogger(IngestServiceShapefileHelper.class.getCanonicalName());	1✔
31
32	private final File zippedShapefile;
33	private final File reZipFolder;
34	private final File unZipFolder;
35
36	// -------------------- CONSTRUCTOR --------------------
37
38	/**
39	* Constructor that accepts a file object
40	*/
41	public IngestServiceShapefileHelper(File zippedShapefile, File workingFolderBase) {	1✔
42	Preconditions.checkArgument(isValidFile(zippedShapefile));	1✔
43	Preconditions.checkArgument(isValidFolder(workingFolderBase));	1✔
44
45	this.zippedShapefile = zippedShapefile;	1✔
46	String id = new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss-SSS").format(new Date());	1✔
47	this.reZipFolder = getShapefileUnzipTempDirectory(workingFolderBase, "shp_" + id + "_rezip");	1✔
48	this.unZipFolder = getShapefileUnzipTempDirectory(workingFolderBase, "shp_" + id + "_unzip");	1✔
49
50	}	1✔
51
52	// -------------------- LOGIC --------------------
53
54	public List<File> processFile() {
55	try {
56	// (1) Use the ShapefileHandler to the .zip for a shapefile
57	//
58	ShapefileHandler shpHandler = new ShapefileHandler(zippedShapefile);	1✔
59	if (!shpHandler.containsShapefile()) {	1✔
NEW 60	logger.severe("Shapefile was incorrectly detected upon Ingest (FileUtil) and passed here");	×
NEW 61	throw new IllegalStateException("Shapefile was incorrectly detected upon Ingest (FileUtil) and passed here");	×
62	}
63
64	// (2) Rezip the shapefile pieces
65	return shpHandler.reZipShapefileSets(unZipFolder, reZipFolder);	1✔
NEW 66	} catch (Exception ex) {	×
NEW 67	throw new IllegalStateException("Shapefile was not correctly unpacked/repacked", ex);	×
68	}
69	}
70
71	@Override
72	public void close() throws IOException {
73	deleteDirectory(unZipFolder);	1✔
74	deleteDirectory(reZipFolder);	1✔
75	}	1✔
76
77	// -------------------- PRIVATE --------------------
78
79	private void deleteDirectory(File directory) {
80	Try.run(() -> FileUtils.deleteDirectory(directory))	1✔
81	.onFailure(ex -> logger.log(Level.SEVERE, "Error cleaning shapefile working directory:" + directory, ex));	1✔
82	}	1✔
83
84	private static File getShapefileUnzipTempDirectory(File tempDirectoryBase, String directoryName) {
85	File datestampedFolder = new File(tempDirectoryBase, directoryName);	1✔
86	if (!datestampedFolder.isDirectory()) {	1✔
87	/* Note that "createDirectories()" must be used - not
88	* "createDirectory()", to make sure all the parent
89	* directories that may not yet exist are created as well.
90	*/
91	try {
92	Files.createDirectories(datestampedFolder.toPath());	1✔
93	} catch (IOException ex) {	×
94	throw new IllegalStateException("Failed to create temp. directory to unzip shapefile: " + datestampedFolder.toString(), ex);	×
95	}	1✔
96	}
97	return datestampedFolder;	1✔
98	}
99
100	private boolean isValidFile(File fileObject) {
101
102	if (fileObject == null) {	1✔
NEW 103	logger.warning("fileObject was null");	×
NEW 104	return false;	×
105	}
106	if (!fileObject.isFile()) {	1✔
NEW 107	logger.warning("fileObject was not a file. Failed \"isFile()\": " + fileObject.getAbsolutePath());	×
NEW 108	return false;	×
109	}
110	return true;	1✔
111	}
112
113
114	private boolean isValidFolder(File fileObject) {
115
116	if (fileObject == null) {	1✔
NEW 117	logger.warning("fileObject was null");	×
NEW 118	return false;	×
119	}
120	if (!fileObject.isDirectory()) {	1✔
NEW 121	logger.warning("fileObject was not a directory. Failed \"isFile()\": " + fileObject.getAbsolutePath());	×
NEW 122	return false;	×
123	}
124	return true;	1✔
125	}
126	}

CeON / dataverse / 1359

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous