IQSS / dataverse / #23580

package edu.harvard.iq.dataverse.engine.command.impl;

import edu.harvard.iq.dataverse.DataFile;

import edu.harvard.iq.dataverse.DatasetVersion;

import edu.harvard.iq.dataverse.Dataverse;

import edu.harvard.iq.dataverse.authorization.Permission;

import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;

import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker;

import edu.harvard.iq.dataverse.engine.command.AbstractCommand;

import edu.harvard.iq.dataverse.engine.command.CommandContext;

import edu.harvard.iq.dataverse.engine.command.DataverseRequest;

import edu.harvard.iq.dataverse.engine.command.exception.CommandException;

import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;

import edu.harvard.iq.dataverse.ingest.IngestServiceShapefileHelper;

import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;

import edu.harvard.iq.dataverse.util.BundleUtil;

import edu.harvard.iq.dataverse.util.FileUtil;

import edu.harvard.iq.dataverse.util.ShapefileHandler;

import edu.harvard.iq.dataverse.util.StringUtil;

import edu.harvard.iq.dataverse.util.file.BagItFileHandler;

import edu.harvard.iq.dataverse.util.file.BagItFileHandlerFactory;

import edu.harvard.iq.dataverse.util.file.CreateDataFileResult;

import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;

import jakarta.enterprise.inject.spi.CDI;

import org.apache.commons.io.FileUtils;

import org.apache.commons.lang3.StringUtils;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.io.InputStream;

import java.nio.charset.Charset;

import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.nio.file.StandardCopyOption;

import java.text.MessageFormat;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Collections;

import java.util.HashMap;

import java.util.HashSet;

import java.util.List;

import java.util.Map;

import java.util.Optional;

import java.util.Set;

import java.util.logging.Logger;

import java.util.zip.GZIPInputStream;

import java.util.zip.ZipEntry;

import java.util.zip.ZipFile;

import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;

import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT;

import static edu.harvard.iq.dataverse.util.FileUtil.createIngestFailureReport;

import static edu.harvard.iq.dataverse.util.FileUtil.determineFileType;

import static edu.harvard.iq.dataverse.util.FileUtil.determineFileTypeByNameAndExtension;

import static edu.harvard.iq.dataverse.util.FileUtil.getFilesTempDirectory;

import static edu.harvard.iq.dataverse.util.FileUtil.saveInputStreamInTempFile;

import static edu.harvard.iq.dataverse.util.FileUtil.useRecognizedType;

/**

 *

 * @author landreev

 */

// Note the commented out @RequiredPermissions. We need to use dynamic 

// permissions instead, to accommodate both adding files to an existing 

// dataset and files being uploaded in the context of creating a new dataset

// via the Add Dataset page. 

//@RequiredPermissions( Permission.EditDataset )

public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileResult> {

    private final DatasetVersion version;

    private final InputStream inputStream;

    private final String fileName;

    private final String suppliedContentType; 

    private final UploadSessionQuotaLimit quota;

    // parent Dataverse must be specified when the command is called on Create 

    // of a new dataset that does not exist in the database yet (for the purposes

    // of authorization - see getRequiredPermissions() below):

    private final Dataverse parentDataverse;

    // With Direct Upload the following values already exist and are passed to the command:

    private final String newStorageIdentifier; 

    private final String newCheckSum; 

    private DataFile.ChecksumType newCheckSumType;

    private final Long newFileSize;

    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UploadSessionQuotaLimit quota, String newCheckSum) {

    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UploadSessionQuotaLimit quota, String newCheckSum, DataFile.ChecksumType newCheckSumType) {

    // This version of the command must be used when files are created in the 

    // context of creating a brand new dataset (from the Add Dataset page):

    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UploadSessionQuotaLimit quota, String newCheckSum, DataFile.ChecksumType newCheckSumType, Long newFileSize, Dataverse dataverse) {

    @Override

    public CreateDataFileResult execute(CommandContext ctxt) throws CommandException {

        //When there is no checksum/checksumtype being sent (normal upload, needs to be calculated), set the type to the current default

        }

        // save the file, in the temporary location for now: 

            }

        }

                try {

                    // "temporary" location is the key here; this is why we are not using

                    // the DataStore framework for this - the assumption is that

                    // temp files will always be stored on the local filesystem.

                    // -- L.A. Jul. 2014

                // A file size check, before we do anything else:

                // (note that "no size limit set" = "unlimited")

                // (also note, that if this is a zip file, we'll be checking

                // the size limit for each of the individual unpacked files)

                    try {

                        // ignore - but log a warning

                }

            } else {

            }

            // Let's try our own utilities (Jhove, etc.) to determine the file type

            // of the uploaded file. (We may already have a mime type supplied for this

            // file - maybe the type that the browser recognized on upload; or, if

            // it's a harvest, maybe the remote server has already given us the type

            // for this file... with our own type utility we may or may not do better

            // than the type supplied:

            // -- L.A.

            try {

                    }

                }

            }

            // A few special cases:

            // if this is a gzipped FITS file, we'll uncompress it, and ingest it as

            // a regular FITS file:

                // if the file name had the ".gz" extension, remove it,

                // since we are going to uncompress it:

                }

                try {

                    // it looks like we simply skip the file silently, if its uncompressed size

                    // exceeds the limit. we should probably report this in detail instead.

                } finally {

                        try {

                    }

                }

                // If we were able to produce an uncompressed file, we'll use it

                // to create and return a final DataFile; if not, we're not going

                // to do anything - and then a new DataFile will be created further

                // down, from the original, uncompressed file.

                    // remove the compressed temp file:

                    try {

                        // (this is very non-fatal)

                    // Update quota if present

                    }

                }

                // If it's a ZIP file, we are going to unpack it and create multiple

                // DataFile objects from its contents:

                try {

                    /*

                        TODO: (?)

                        We may want to investigate somehow letting the user specify

                        the charset for the filenames in the zip file...

                    - otherwise, ZipInputStream bails out if it encounteres a file

                        name that's not valid in the current charest (i.e., UTF-8, in

                    our case). It would be a bit trickier than what we're doing for

                    SPSS tabular ingests - with the lang. encoding pulldown menu -

                        because this encoding needs to be specified *before* we upload and

                    attempt to unzip the file.

                                -- L.A. 4.0 beta12

                        logger.info("default charset is "+Charset.defaultCharset().name());

                        if (Charset.isSupported("US-ASCII")) {

                            logger.info("charset US-ASCII is supported.");

                            charset = Charset.forName("US-ASCII");

                            if (charset != null) {

                                   logger.info("was able to obtain charset for US-ASCII");

                            }

                         }

                     */

                    /**

                     * Perform a quick check for how many individual files are

                     * inside this zip archive. If it's above the limit, we can

                     * give up right away, without doing any unpacking.

                     * This should be a fairly inexpensive operation, we just need

                     * to read the directory at the end of the file.

                     */

                    /**

                     * The ZipFile constructors in openZipFile will throw ZipException -

                     * a type of IOException - if there's something wrong 

                     * with this file as a zip. There's no need to intercept it

                     * here, it will be caught further below, with other IOExceptions,

                     * at which point we'll give up on trying to unpack it and

                     * then attempt to save it as is.

                     */

                    /**

                     * Note that we can't just use zipFile.size(),

                     * unfortunately, since that's the total number of entries,

                     * some of which can be directories. So we need to go

                     * through all the individual zipEntries and count the ones

                     * that are files.

                     */

                                                 + "); please upload a zip archive with fewer files, if you want them to be ingested "

                                                 + "as individual DataFiles.";

                            }

                            // In addition to counting the files, we can

                            // also check the file size while we're here,

                            // provided the size limit is defined; if a single

                            // file is above the individual size limit, unzipped,

                            // we give up on unpacking this zip archive as well:

                            }

                            // Similarly, we want to check if saving all these unpacked

                            // files is going to push the disk usage over the

                            // quota:

                                    //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(combinedUnzippedFileSize), bytesToHumanReadable(storageQuotaLimit)));

                                    // change of plans: if the unzipped content inside exceeds the remaining quota,

                                    // we reject the upload outright, rather than accepting the zip

                                    // file as is.

                                }

                            }

                        // OK we're still here - that means we can proceed unzipping.

                        // reset:

                                        + "); please upload a zip archive with fewer files, if you want them to be ingested "

                                        + "as individual DataFiles.";

                            }

                            // No need to check the size of this unpacked file against the size limit,

                            // since we've already checked for that in the first pass.

                                MIME_TYPE_UNDETERMINED_DEFAULT,

                                // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes),

                                // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all

                                // the leading, trailing and duplicate slashes; then replace all the characters that

                                // don't pass our validation rules.

                                // if (!"".equals(directoryName)) {

                                }

                            }

                                // We have created this datafile with the mime type "unknown";

                                // Now that we have it saved in a temporary location,

                                // let's try and determine its real type:

                                try {

                                    // null the File explicitly, to release any open FDs:

                                    }

                            }

                    }

                    // just clear the datafiles list and let

                    // ingest default to creating a single DataFile out

                    // of the unzipped file.

                    }

                    //logger.warning("One of the unzipped files exceeds the storage quota limit; resorting to saving the file as is. " + fesqx.getMessage());

                    //warningMessage =  BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.quota", Arrays.asList(FileSizeChecker.bytesToHumanReadable(storageQuotaLimit)));

                    //datafiles.clear();

                    throw new CommandExecutionException(fesqx.getMessage(), fesqx, this);

                }*/

                    // remove the uploaded zip file:

                    try {

                        // do nothing - it's just a temp file.

                    // update the quota object: 

                    }

                    // and return:

                }

                // Shape files may have to be split into multiple files,

                // one zip archive per each complete set of shape files:

                // File rezipFolder = new File(this.getFilesTempDirectory());

                IngestServiceShapefileHelper shpIngestHelper;

                }

                try {

                        }

                                // This file lives in a subfolder - we want to 

                                // preserve it in the FileMetadata:

                                }

                            }

                        }

                            // todo: can this new_datafile be null?

                        } else {

                        }

                        try {

                            // this one can be ignored

                    // (or should we throw an exception, instead of skipping it quietly?

                    // todo? - maybe try to provide a more detailed explanation, of which repackaged component, etc.?

                // Delete the temp directory used for unzipping

                // The try-catch is due to error encountered in using NFS for stocking file,

                // cf. https://github.com/IQSS/dataverse/issues/5909

                try {

                    // do nothing - it's a temp folder.

                    // remove the uploaded zip file:

                    try {

                        // ignore - it's just a temp file - but let's log a warning

                        // same

                    // update the quota object: 

                    }

                } else {

                }

                try { 

                    }

            }

            // These are the final File and its size that will be used to 

            // add create a single Datafile: 

            // Direct upload.

            // Since this is a direct upload, and therefore no temp file associated 

            // with it, we may, OR MAY NOT know the size of the file. If this is 

            // a direct upload via the UI, the page must have already looked up 

            // the size, after the client confirmed that the upload had completed. 

            // (so that we can reject the upload here, i.e. before the user clicks

            // save, if it's over the size limit or storage quota). However, if 

            // this is a direct upload via the API, we will wait until the 

            // upload is finalized in the saveAndAddFiles method to enforce the 

            // limits. 

                // if the size is specified, and it's above the individual size 

                // limit for this store, we can reject it now:

                }

            }

            // Default to suppliedContentType if set or the overall undetermined default if a contenttype isn't supplied

                //Use rules for deciding when to trust browser supplied type

                }

            }

        }

        // Finally, if none of the special cases above were applicable (or 

        // if we were unable to unpack an uploaded file, etc.), we'll just 

        // create and return a single DataFile:

        // We have already checked that this file does not exceed the individual size limit; 

        // but if we are processing it as is, as a single file, we need to check if 

        // its size does not go beyond the allocated storage quota (if specified):

                // Remove the temp. file, if this is a non-direct upload. 

                // If this is a direct upload, it will be a responsibility of the 

                // component calling the command to remove the file that may have

                // already been saved in the S3 volume. 

                try {

                    // ignore - but log a warning

            }

        } 

            }

            }

            // Update the quota definition for the *current upload session*

            // This is relevant for the uploads going through the UI page 

            // (where there may be an appreciable amount of time between the user

            // uploading the files and clicking "save". The file size should be 

            // available here for both direct and local uploads via the UI. 

            // It is not yet available if this is direct-via-API - but 

            // for API uploads the quota check will be enforced during the final 

            // save. 

            }

        }

    }   // end createDataFiles

    private static List<? extends ZipEntry> filteredZipEntries(ZipFile zipFile) {

    }

    private static ZipFile openZipFile(Path tempFile, Charset charset) throws IOException {

        }

        else {

        }

    }

    private static boolean isFileToSkip(String fileName) {

        // check if it's a "fake" file - a zip archive entry

        // created for a MacOS X filesystem element: (these

        // start with "._")

    }

    private static String getShortName(String fileName) {

    }

    @Override

    public Map<String, Set<Permission>> getRequiredPermissions() {

            // The command is called in the context of uploading files on 

            // create of a new dataset

        } else {

            // An existing dataset

        }

    }

}