#22693

Committed 03 Jul 2024 01:09PM CUT coverage: 20.626% (-0.09%) from 20.716%

Build # #22693

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #10664 from IQSS/develop

merge develop into master for 6.3

Run Details

195 of 1852 new or added lines in 82 files covered. (10.53%)

72 existing lines in 33 files now uncovered.

17335 of 84043 relevant lines covered (20.63%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.03

/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java

package edu.harvard.iq.dataverse.api;

import edu.harvard.iq.dataverse.ControlledVocabAlternate;
import edu.harvard.iq.dataverse.ControlledVocabularyValue;
import edu.harvard.iq.dataverse.ControlledVocabularyValueServiceBean;
import edu.harvard.iq.dataverse.DatasetField;
import edu.harvard.iq.dataverse.DatasetFieldConstant;
import edu.harvard.iq.dataverse.DatasetFieldType;
import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
import edu.harvard.iq.dataverse.DatasetFieldType.FieldType;
import edu.harvard.iq.dataverse.DataverseServiceBean;
import edu.harvard.iq.dataverse.MetadataBlock;
import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jakarta.ejb.EJB;
import jakarta.ejb.EJBException;
import jakarta.json.Json;
import jakarta.json.JsonArrayBuilder;
import jakarta.validation.ConstraintViolationException;
import jakarta.ws.rs.Consumes;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.POST;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.core.Response;

import edu.harvard.iq.dataverse.util.BundleUtil;
import edu.harvard.iq.dataverse.util.ConstraintViolationUtil;
import org.apache.commons.lang3.StringUtils;
import static edu.harvard.iq.dataverse.util.json.JsonPrinter.asJsonArray;
import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;

import java.util.logging.Level;
import java.util.logging.Logger;
import jakarta.persistence.NoResultException;
import jakarta.persistence.TypedQuery;
import jakarta.ws.rs.core.Response.Status;

import java.io.BufferedInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

@Path("admin/datasetfield")
public class DatasetFieldServiceApi extends AbstractApiBean {

    @EJB
    DatasetFieldServiceBean datasetFieldService;

    @EJB
    DataverseServiceBean dataverseService;

    @EJB
    MetadataBlockServiceBean metadataBlockService;

    @EJB
    ControlledVocabularyValueServiceBean controlledVocabularyValueService;

    private static final Logger logger = Logger.getLogger(DatasetFieldServiceApi.class.getName());
    
    @GET
    public Response getAll() {
        try {
            List<String> listOfIsHasParentsTrue = new ArrayList<>();
            List<String> listOfIsHasParentsFalse = new ArrayList<>();
            List<String> listOfIsAllowsMultiplesTrue = new ArrayList<>();
            List<String> listOfIsAllowsMultiplesFalse = new ArrayList<>();
            for (DatasetFieldType dsf : datasetFieldService.findAllOrderedById()) {
                if (dsf.isHasParent()) {
                    listOfIsHasParentsTrue.add(dsf.getName());
                    listOfIsAllowsMultiplesTrue.add(dsf.getName());
                } else {
                    listOfIsHasParentsFalse.add(dsf.getName());
                    listOfIsAllowsMultiplesFalse.add(dsf.getName());
                }
            }
            final List<DatasetFieldType> requiredFields = datasetFieldService.findAllRequiredFields();
            final List<String> requiredFieldNames = new ArrayList<>(requiredFields.size());
            for ( DatasetFieldType dt : requiredFields ) {
                requiredFieldNames.add( dt.getName() );
            }
            return ok( Json.createObjectBuilder().add("haveParents", asJsonArray(listOfIsHasParentsTrue))
                    .add("noParents", asJsonArray(listOfIsHasParentsFalse))
                    .add("allowsMultiples", asJsonArray(listOfIsAllowsMultiplesTrue))
                    .add("allowsMultiples", asJsonArray(listOfIsAllowsMultiplesTrue))
                    .add("doesNotAllowMultiples", asJsonArray(listOfIsAllowsMultiplesFalse))
                    .add("required", asJsonArray(requiredFieldNames))
            );
            
        } catch (EJBException ex) {
            Throwable cause = ex;
            StringBuilder sb = new StringBuilder();
            sb.append(ex).append(" ");
            while (cause.getCause() != null) {
                cause = cause.getCause();
                sb.append(cause.getClass().getCanonicalName()).append(" ");
                sb.append(cause.getMessage()).append(" ");
                if (cause instanceof ConstraintViolationException) {
                    sb.append(ConstraintViolationUtil.getErrorStringForConstraintViolations(cause));
                }
            }
            return error(Status.INTERNAL_SERVER_ERROR, sb.toString());
        }
    }

    @GET
    @Path("{name}")
    public Response getByName(@PathParam("name") String name) {
        try {
            DatasetFieldType dsf = datasetFieldService.findByName(name);
            Long id = dsf.getId();
            String title = dsf.getTitle();
            FieldType fieldType = dsf.getFieldType();
            String solrFieldSearchable = dsf.getSolrField().getNameSearchable();
            String solrFieldFacetable = dsf.getSolrField().getNameFacetable();
            String metadataBlock = dsf.getMetadataBlock().getName();
            String uri=dsf.getUri();
            boolean hasParent = dsf.isHasParent();
            boolean allowsMultiples = dsf.isAllowMultiples();
            boolean isRequired = dsf.isRequired();
            String parentAllowsMultiplesDisplay = "N/A (no parent)";
            boolean parentAllowsMultiplesBoolean;
            if (hasParent) {
                DatasetFieldType parent = dsf.getParentDatasetFieldType();
                parentAllowsMultiplesBoolean = parent.isAllowMultiples();
                parentAllowsMultiplesDisplay = Boolean.toString(parentAllowsMultiplesBoolean);
            }
            JsonArrayBuilder controlledVocabularyValues = Json.createArrayBuilder();
            for (ControlledVocabularyValue controlledVocabularyValue : dsf.getControlledVocabularyValues()) {
                controlledVocabularyValues.add(NullSafeJsonBuilder.jsonObjectBuilder()
                        .add("id", controlledVocabularyValue.getId())
                        .add("strValue", controlledVocabularyValue.getStrValue())
                        .add("displayOrder", controlledVocabularyValue.getDisplayOrder())
                        .add("identifier", controlledVocabularyValue.getIdentifier())
                );
            }
            return ok(NullSafeJsonBuilder.jsonObjectBuilder()
                    .add("name", dsf.getName())
                    .add("id", id )
                    .add("title", title)
                    .add( "metadataBlock", metadataBlock)
                    .add("fieldType", fieldType.name())
                    .add("allowsMultiples", allowsMultiples)
                    .add("hasParent", hasParent)
                    .add("controlledVocabularyValues", controlledVocabularyValues)
                    .add("parentAllowsMultiples", parentAllowsMultiplesDisplay)
                    .add("solrFieldSearchable", solrFieldSearchable)
                    .add("solrFieldFacetable", solrFieldFacetable)
                    .add("isRequired", isRequired)
                    .add("uri", uri));
        
        } catch ( NoResultException nre ) {
            return notFound(name);
            
        } catch (EJBException | NullPointerException ex) {
            Throwable cause = ex;
            StringBuilder sb = new StringBuilder();
            sb.append(ex).append(" ");
            while (cause.getCause() != null) {
                cause = cause.getCause();
                sb.append(cause.getClass().getCanonicalName()).append(" ");
                sb.append(cause.getMessage()).append(" ");
                if (cause instanceof ConstraintViolationException) {
                    sb.append(ConstraintViolationUtil.getErrorStringForConstraintViolations(cause));
                }
            }
            return error( Status.INTERNAL_SERVER_ERROR, sb.toString() );
        }

    }

    /**
     *
     * See also http://irclog.greptilian.com/rest/2015-02-07#i_95635
     *
     * @todo is our convention camelCase? Or lisp-case? Or snake_case?
     */
    @GET
    @Path("controlledVocabulary/subject")
    public Response showControlledVocabularyForSubject() {
        DatasetFieldType subjectDatasetField = datasetFieldService.findByName(DatasetFieldConstant.subject);
        JsonArrayBuilder possibleSubjects = Json.createArrayBuilder();
        for (ControlledVocabularyValue subjectValue : controlledVocabularyValueService.findByDatasetFieldTypeId(subjectDatasetField.getId())) {
            String subject = subjectValue.getStrValue();
            if (subject != null) {
                possibleSubjects.add(subject);
            }
        }
        return ok(possibleSubjects);
    }
    
    
    // TODO consider replacing with a @Startup method on the datasetFieldServiceBean
    @GET
    @Path("loadNAControlledVocabularyValue")
    public Response loadNAControlledVocabularyValue() {
        // the find will throw a NoResultException if no values are in db
//            datasetFieldService.findNAControlledVocabularyValue();
        TypedQuery<ControlledVocabularyValue> naValueFinder = em.createQuery("SELECT OBJECT(o) FROM ControlledVocabularyValue AS o WHERE o.datasetFieldType is null AND o.strValue = :strvalue", ControlledVocabularyValue.class);
        naValueFinder.setParameter("strvalue", DatasetField.NA_VALUE);
        
        if ( naValueFinder.getResultList().isEmpty() ) {
            ControlledVocabularyValue naValue = new ControlledVocabularyValue();
            naValue.setStrValue(DatasetField.NA_VALUE);
            datasetFieldService.save(naValue);
            return ok("NA value created.");

        } else {
            return ok("NA value exists.");
        }
    }

    public enum HeaderType {
        METADATABLOCK, DATASETFIELD, CONTROLLEDVOCABULARY
    }

    @POST
    @Consumes("text/tab-separated-values")
    @Path("load")
    public Response loadDatasetFields(File file) {
        ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "loadDatasetFields");
        alr.setInfo( file.getName() );
        BufferedReader br = null;
        String line;
        String splitBy = "\t";
        int lineNumber = 0;
        HeaderType header = null;
        JsonArrayBuilder responseArr = Json.createArrayBuilder();
        String[] values = null;
        try {
            br = new BufferedReader(new FileReader("/" + file));
            while ((line = br.readLine()) != null) {
                lineNumber++;
                values = line.split(splitBy);
                if (values[0].startsWith("#")) { // Header row
                    switch (values[0]) {
                        case "#metadataBlock":
                            header = HeaderType.METADATABLOCK;
                            break;
                        case "#datasetField":
                            header = HeaderType.DATASETFIELD;
                            break;
                        case "#controlledVocabulary":
                            header = HeaderType.CONTROLLEDVOCABULARY;
                            break;
                        default:
                            throw new IOException("Encountered unknown #header type at line lineNumber " + lineNumber);
                    }
                } else {
                    switch (header) {
                        case METADATABLOCK:
                            responseArr.add( Json.createObjectBuilder()
                                                    .add("name", parseMetadataBlock(values))
                                                    .add("type", "MetadataBlock"));
                            break;
                            
                        case DATASETFIELD:
                            responseArr.add( Json.createObjectBuilder()
                                                    .add("name", parseDatasetField(values))
                                                    .add("type", "DatasetField") );
                            break;
                            
                        case CONTROLLEDVOCABULARY:
                            responseArr.add( Json.createObjectBuilder()
                                                    .add("name", parseControlledVocabulary(values))
                                                    .add("type", "Controlled Vocabulary") );
                            break;
                            
                        default:
                            throw new IOException("No #header defined in file.");

                    }
                }
            }
        } catch (FileNotFoundException e) {
            alr.setActionResult(ActionLogRecord.Result.BadRequest);
            alr.setInfo( alr.getInfo() + "// file not found");
            return error(Status.EXPECTATION_FAILED, "File not found");

        } catch (ArrayIndexOutOfBoundsException e) {
            String message = getArrayIndexOutOfBoundMessage(header, lineNumber, values.length);
            logger.log(Level.WARNING, message, e);
            alr.setActionResult(ActionLogRecord.Result.InternalError);
            alr.setInfo(alr.getInfo() + "// " + message);
            return error(Status.INTERNAL_SERVER_ERROR, message);

        } catch (Exception e) {
            String message = getGeneralErrorMessage(header, lineNumber, e.getMessage());
            logger.log(Level.WARNING, message, e);
            alr.setActionResult(ActionLogRecord.Result.InternalError);
            alr.setInfo( alr.getInfo() + "// " + message);
            return error(Status.INTERNAL_SERVER_ERROR, message);
            
        } finally {
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e) {
                    logger.log(Level.WARNING, "Error closing the reader while importing Dataset Fields.");
                }
            }
            actionLogSvc.log(alr);
        }

        return ok( Json.createObjectBuilder().add("added", responseArr) );
    }

    /**
     * Provide a general error message including the part and line number
     * @param header
     * @param lineNumber
     * @param message
     * @return
     */
    public String getGeneralErrorMessage(HeaderType header, int lineNumber, String message) {
        List<String> arguments = new ArrayList<>();
        arguments.add(header.name());
        arguments.add(String.valueOf(lineNumber));
        arguments.add(message);
        return BundleUtil.getStringFromBundle("api.admin.datasetfield.load.GeneralErrorMessage", arguments);
    }

    /**
     * Turn ArrayIndexOutOfBoundsException into an informative error message
     * @param lineNumber
     * @param header
     * @param e
     * @return
     */
    public String getArrayIndexOutOfBoundMessage(HeaderType header,
                                                 int lineNumber,
                                                 int wrongIndex) {

        List<String> columns = getColumnsByHeader(header);
        
        String column = columns.get(wrongIndex - 1);
        List<String> arguments = new ArrayList<>();
        arguments.add(header.name());
        arguments.add(String.valueOf(lineNumber));
        arguments.add(column);
        arguments.add(String.valueOf(wrongIndex + 1));
        return BundleUtil.getStringFromBundle(
            "api.admin.datasetfield.load.ArrayIndexOutOfBoundMessage",
            arguments
        );
    }

    /**
     * Get the list of columns by the type of header
     * @param header
     * @return
     */
    private List<String> getColumnsByHeader(HeaderType header) {
        List<String> columns = null;
        if (header.equals(HeaderType.METADATABLOCK)) {
            columns = Arrays.asList("name", "dataverseAlias", "displayName");
        } else if (header.equals(HeaderType.DATASETFIELD)) {
            columns = Arrays.asList("name", "title", "description", "watermark",
              "fieldType", "displayOrder", "displayFormat", "advancedSearchField",
              "allowControlledVocabulary", "allowmultiples", "facetable",
              "displayoncreate", "required", "parent", "metadatablock_id");
        } else if (header.equals(HeaderType.CONTROLLEDVOCABULARY)) {
            columns = Arrays.asList("DatasetField", "Value", "identifier", "displayOrder");
        }

        return columns;
    }

    private String parseMetadataBlock(String[] values) {
        //Test to see if it exists by name
        MetadataBlock mdb = metadataBlockService.findByName(values[1]);
        if (mdb == null){
            mdb = new MetadataBlock();
        }
        mdb.setName(values[1]);
        if (!values[2].isEmpty()){
            mdb.setOwner(dataverseService.findByAlias(values[2]));
        }
        mdb.setDisplayName(values[3]);
        if (values.length>4 && !StringUtils.isEmpty(values[4])) {
            mdb.setNamespaceUri(values[4]);
        }

        metadataBlockService.save(mdb);
        return mdb.getName();
    }

    private String parseDatasetField(String[] values) {
        
        //First see if it exists
        DatasetFieldType dsf = datasetFieldService.findByName(values[1]);
        if (dsf == null) {
            //if not create new
            dsf = new DatasetFieldType();
        }
        //add(update) values
        dsf.setName(values[1]);
        dsf.setTitle(values[2]);
        dsf.setDescription(values[3]);
        dsf.setWatermark(values[4]);
        dsf.setFieldType(FieldType.valueOf(values[5].toUpperCase()));
        dsf.setDisplayOrder(Integer.parseInt(values[6]));
        dsf.setDisplayFormat(values[7]);
        dsf.setAdvancedSearchFieldType(Boolean.parseBoolean(values[8]));
        dsf.setAllowControlledVocabulary(Boolean.parseBoolean(values[9]));
        dsf.setAllowMultiples(Boolean.parseBoolean(values[10]));
        dsf.setFacetable(Boolean.parseBoolean(values[11]));
        dsf.setDisplayOnCreate(Boolean.parseBoolean(values[12]));
        dsf.setRequired(Boolean.parseBoolean(values[13]));
        if (!StringUtils.isEmpty(values[14])) {
            dsf.setParentDatasetFieldType(datasetFieldService.findByName(values[14]));
        } else {
            dsf.setParentDatasetFieldType(null);
        }
        dsf.setMetadataBlock(dataverseService.findMDBByName(values[15]));
        if(values.length>16 && !StringUtils.isEmpty(values[16])) {
          dsf.setUri(values[16]);
        }
        datasetFieldService.save(dsf);
        return dsf.getName();
    }

    private String parseControlledVocabulary(String[] values) {
        
        DatasetFieldType dsv = datasetFieldService.findByName(values[1]);
        //See if it already exists
        /*
         Matching relies on assumption that only one cv value will exist for a given identifier or display value
        If the lookup queries return multiple matches then retval is null 
        */
        //First see if cvv exists based on display name
        ControlledVocabularyValue cvv = datasetFieldService.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsv, values[2], true);
        
        //then see if there's a match on identifier
        ControlledVocabularyValue cvvi = null;
        if (values[3] != null && !values[3].trim().isEmpty()){
            cvvi = datasetFieldService.findControlledVocabularyValueByDatasetFieldTypeAndIdentifier(dsv, values[3]);
        }
        
        //if there's a match on identifier use it
        if (cvvi != null){
            cvv = cvvi;
        }

        //if there's no match create a new one
        if (cvv == null) {
            cvv = new ControlledVocabularyValue();
            cvv.setDatasetFieldType(dsv);
        }
        
        // Alternate variants for this controlled vocab. value: 
        
        // Note that these are overwritten every time:
        cvv.getControlledVocabAlternates().clear();
        // - meaning, if an alternate has been removed from the tsv file, 
        // it will be removed from the database! -- L.A. 5.4
        
        for (int i = 5; i < values.length; i++) {
            ControlledVocabAlternate alt = new ControlledVocabAlternate();
            alt.setDatasetFieldType(dsv);
            alt.setControlledVocabularyValue(cvv);
            alt.setStrValue(values[i]);
            cvv.getControlledVocabAlternates().add(alt);
        }
        
        cvv.setStrValue(values[2]);
        cvv.setIdentifier(values[3]);
        cvv.setDisplayOrder(Integer.parseInt(values[4]));
        datasetFieldService.save(cvv);
        return cvv.getStrValue();
    }


    @POST
    @Consumes("application/zip")
    @Path("loadpropertyfiles")
    public Response loadLanguagePropertyFile(File inputFile) {
        try (ZipFile file = new ZipFile(inputFile)) {
            //Get file entries
            Enumeration<? extends ZipEntry> entries = file.entries();

            //We will unzip files in this folder
            String dataverseLangDirectory = getDataverseLangDirectory();

            //Iterate over entries
            while (entries.hasMoreElements())
            {
                ZipEntry entry = entries.nextElement();
                String dataverseLangFileName = dataverseLangDirectory + "/" + entry.getName();
                File entryFile = new File(dataverseLangFileName);
                String canonicalPath = entryFile.getCanonicalPath();
                if (canonicalPath.startsWith(dataverseLangDirectory + "/")) {
                    try (FileOutputStream fileOutput = new FileOutputStream(dataverseLangFileName)) {

                        InputStream is = file.getInputStream(entry);
                        BufferedInputStream bis = new BufferedInputStream(is);

                        while (bis.available() > 0) {
                            fileOutput.write(bis.read());
                        }
                    }
                } else {
                    logger.log(Level.SEVERE, "Zip Slip prevented: uploaded zip file tried to write to {}", canonicalPath);
                    return Response.status(400).entity("The zip file includes an illegal file path").build();
                }
            }
        }
        catch(IOException e) {
            logger.log(Level.SEVERE, "Reading the language property zip file failed", e);
            return Response.status(500).entity("Internal server error. More details available at the server logs.").build();
        }

        return Response.status(200).entity("Uploaded the file successfully ").build();
    }

    public static String getDataverseLangDirectory() {
        String dataverseLangDirectory = System.getProperty("dataverse.lang.directory");
        if (dataverseLangDirectory == null || dataverseLangDirectory.equals("")) {
            dataverseLangDirectory = "/tmp/files";
        }

        if (!Files.exists(Paths.get(dataverseLangDirectory))) {
            try {
                Files.createDirectories(Paths.get(dataverseLangDirectory));
            } catch (IOException ex) {
                logger.severe("Failed to create dataverseLangDirectory: " + dataverseLangDirectory );
                return null;
            }
        }

        return dataverseLangDirectory;
    }

}

1	package edu.harvard.iq.dataverse.api;
2
3	import edu.harvard.iq.dataverse.ControlledVocabAlternate;
4	import edu.harvard.iq.dataverse.ControlledVocabularyValue;
5	import edu.harvard.iq.dataverse.ControlledVocabularyValueServiceBean;
6	import edu.harvard.iq.dataverse.DatasetField;
7	import edu.harvard.iq.dataverse.DatasetFieldConstant;
8	import edu.harvard.iq.dataverse.DatasetFieldType;
9	import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
10	import edu.harvard.iq.dataverse.DatasetFieldType.FieldType;
11	import edu.harvard.iq.dataverse.DataverseServiceBean;
12	import edu.harvard.iq.dataverse.MetadataBlock;
13	import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
14	import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord;
15	import java.io.BufferedReader;
16	import java.io.File;
17	import java.io.FileNotFoundException;
18	import java.io.FileReader;
19	import java.io.IOException;
20	import java.util.ArrayList;
21	import java.util.Arrays;
22	import java.util.List;
23	import jakarta.ejb.EJB;
24	import jakarta.ejb.EJBException;
25	import jakarta.json.Json;
26	import jakarta.json.JsonArrayBuilder;
27	import jakarta.validation.ConstraintViolationException;
28	import jakarta.ws.rs.Consumes;
29	import jakarta.ws.rs.GET;
30	import jakarta.ws.rs.POST;
31	import jakarta.ws.rs.Path;
32	import jakarta.ws.rs.PathParam;
33	import jakarta.ws.rs.core.Response;
34
35	import edu.harvard.iq.dataverse.util.BundleUtil;
36	import edu.harvard.iq.dataverse.util.ConstraintViolationUtil;
37	import org.apache.commons.lang3.StringUtils;
38	import static edu.harvard.iq.dataverse.util.json.JsonPrinter.asJsonArray;
39	import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
40
41	import java.util.logging.Level;
42	import java.util.logging.Logger;
43	import jakarta.persistence.NoResultException;
44	import jakarta.persistence.TypedQuery;
45	import jakarta.ws.rs.core.Response.Status;
46
47	import java.io.BufferedInputStream;
48	import java.io.FileOutputStream;
49	import java.io.InputStream;
50	import java.nio.file.Files;
51	import java.nio.file.Paths;
52	import java.util.Enumeration;
53	import java.util.zip.ZipEntry;
54	import java.util.zip.ZipFile;
55
56	@Path("admin/datasetfield")
57	public class DatasetFieldServiceApi extends AbstractApiBean {	1✔
58
59	@EJB
60	DatasetFieldServiceBean datasetFieldService;
61
62	@EJB
63	DataverseServiceBean dataverseService;
64
65	@EJB
66	MetadataBlockServiceBean metadataBlockService;
67
68	@EJB
69	ControlledVocabularyValueServiceBean controlledVocabularyValueService;
70
71	private static final Logger logger = Logger.getLogger(DatasetFieldServiceApi.class.getName());	1✔
72
73	@GET
74	public Response getAll() {
75	try {
76	List<String> listOfIsHasParentsTrue = new ArrayList<>();	×
77	List<String> listOfIsHasParentsFalse = new ArrayList<>();	×
78	List<String> listOfIsAllowsMultiplesTrue = new ArrayList<>();	×
79	List<String> listOfIsAllowsMultiplesFalse = new ArrayList<>();	×
80	for (DatasetFieldType dsf : datasetFieldService.findAllOrderedById()) {	×
81	if (dsf.isHasParent()) {	×
82	listOfIsHasParentsTrue.add(dsf.getName());	×
83	listOfIsAllowsMultiplesTrue.add(dsf.getName());	×
84	} else {
85	listOfIsHasParentsFalse.add(dsf.getName());	×
86	listOfIsAllowsMultiplesFalse.add(dsf.getName());	×
87	}
88	}	×
89	final List<DatasetFieldType> requiredFields = datasetFieldService.findAllRequiredFields();	×
90	final List<String> requiredFieldNames = new ArrayList<>(requiredFields.size());	×
91	for ( DatasetFieldType dt : requiredFields ) {	×
92	requiredFieldNames.add( dt.getName() );	×
93	}	×
94	return ok( Json.createObjectBuilder().add("haveParents", asJsonArray(listOfIsHasParentsTrue))	×
95	.add("noParents", asJsonArray(listOfIsHasParentsFalse))	×
96	.add("allowsMultiples", asJsonArray(listOfIsAllowsMultiplesTrue))	×
97	.add("allowsMultiples", asJsonArray(listOfIsAllowsMultiplesTrue))	×
98	.add("doesNotAllowMultiples", asJsonArray(listOfIsAllowsMultiplesFalse))	×
99	.add("required", asJsonArray(requiredFieldNames))	×
100	);
101
102	} catch (EJBException ex) {	×
103	Throwable cause = ex;	×
104	StringBuilder sb = new StringBuilder();	×
105	sb.append(ex).append(" ");	×
106	while (cause.getCause() != null) {	×
107	cause = cause.getCause();	×
108	sb.append(cause.getClass().getCanonicalName()).append(" ");	×
109	sb.append(cause.getMessage()).append(" ");	×
110	if (cause instanceof ConstraintViolationException) {	×
111	sb.append(ConstraintViolationUtil.getErrorStringForConstraintViolations(cause));	×
112	}
113	}
114	return error(Status.INTERNAL_SERVER_ERROR, sb.toString());	×
115	}
116	}
117
118	@GET
119	@Path("{name}")
120	public Response getByName(@PathParam("name") String name) {
121	try {
122	DatasetFieldType dsf = datasetFieldService.findByName(name);	×
123	Long id = dsf.getId();	×
124	String title = dsf.getTitle();	×
125	FieldType fieldType = dsf.getFieldType();	×
126	String solrFieldSearchable = dsf.getSolrField().getNameSearchable();	×
127	String solrFieldFacetable = dsf.getSolrField().getNameFacetable();	×
128	String metadataBlock = dsf.getMetadataBlock().getName();	×
129	String uri=dsf.getUri();	×
130	boolean hasParent = dsf.isHasParent();	×
131	boolean allowsMultiples = dsf.isAllowMultiples();	×
132	boolean isRequired = dsf.isRequired();	×
133	String parentAllowsMultiplesDisplay = "N/A (no parent)";	×
134	boolean parentAllowsMultiplesBoolean;
135	if (hasParent) {	×
136	DatasetFieldType parent = dsf.getParentDatasetFieldType();	×
137	parentAllowsMultiplesBoolean = parent.isAllowMultiples();	×
138	parentAllowsMultiplesDisplay = Boolean.toString(parentAllowsMultiplesBoolean);	×
139	}
140	JsonArrayBuilder controlledVocabularyValues = Json.createArrayBuilder();	×
141	for (ControlledVocabularyValue controlledVocabularyValue : dsf.getControlledVocabularyValues()) {	×
142	controlledVocabularyValues.add(NullSafeJsonBuilder.jsonObjectBuilder()	×
143	.add("id", controlledVocabularyValue.getId())	×
144	.add("strValue", controlledVocabularyValue.getStrValue())	×
145	.add("displayOrder", controlledVocabularyValue.getDisplayOrder())	×
146	.add("identifier", controlledVocabularyValue.getIdentifier())	×
147	);
148	}	×
149	return ok(NullSafeJsonBuilder.jsonObjectBuilder()	×
150	.add("name", dsf.getName())	×
151	.add("id", id )	×
152	.add("title", title)	×
153	.add( "metadataBlock", metadataBlock)	×
154	.add("fieldType", fieldType.name())	×
155	.add("allowsMultiples", allowsMultiples)	×
156	.add("hasParent", hasParent)	×
157	.add("controlledVocabularyValues", controlledVocabularyValues)	×
158	.add("parentAllowsMultiples", parentAllowsMultiplesDisplay)	×
159	.add("solrFieldSearchable", solrFieldSearchable)	×
160	.add("solrFieldFacetable", solrFieldFacetable)	×
161	.add("isRequired", isRequired)	×
162	.add("uri", uri));	×
163
164	} catch ( NoResultException nre ) {	×
165	return notFound(name);	×
166
167	} catch (EJBException \| NullPointerException ex) {	×
168	Throwable cause = ex;	×
169	StringBuilder sb = new StringBuilder();	×
170	sb.append(ex).append(" ");	×
171	while (cause.getCause() != null) {	×
172	cause = cause.getCause();	×
173	sb.append(cause.getClass().getCanonicalName()).append(" ");	×
174	sb.append(cause.getMessage()).append(" ");	×
175	if (cause instanceof ConstraintViolationException) {	×
176	sb.append(ConstraintViolationUtil.getErrorStringForConstraintViolations(cause));	×
177	}
178	}
179	return error( Status.INTERNAL_SERVER_ERROR, sb.toString() );	×
180	}
181
182	}
183
184	/**
185	*
186	* See also http://irclog.greptilian.com/rest/2015-02-07#i_95635
187	*
188	* @todo is our convention camelCase? Or lisp-case? Or snake_case?
189	*/
190	@GET
191	@Path("controlledVocabulary/subject")
192	public Response showControlledVocabularyForSubject() {
193	DatasetFieldType subjectDatasetField = datasetFieldService.findByName(DatasetFieldConstant.subject);	×
194	JsonArrayBuilder possibleSubjects = Json.createArrayBuilder();	×
195	for (ControlledVocabularyValue subjectValue : controlledVocabularyValueService.findByDatasetFieldTypeId(subjectDatasetField.getId())) {	×
196	String subject = subjectValue.getStrValue();	×
197	if (subject != null) {	×
198	possibleSubjects.add(subject);	×
199	}
200	}	×
201	return ok(possibleSubjects);	×
202	}
203
204
205	// TODO consider replacing with a @Startup method on the datasetFieldServiceBean
206	@GET
207	@Path("loadNAControlledVocabularyValue")
208	public Response loadNAControlledVocabularyValue() {
209	// the find will throw a NoResultException if no values are in db
210	// datasetFieldService.findNAControlledVocabularyValue();
211	TypedQuery<ControlledVocabularyValue> naValueFinder = em.createQuery("SELECT OBJECT(o) FROM ControlledVocabularyValue AS o WHERE o.datasetFieldType is null AND o.strValue = :strvalue", ControlledVocabularyValue.class);	×
212	naValueFinder.setParameter("strvalue", DatasetField.NA_VALUE);	×
213
214	if ( naValueFinder.getResultList().isEmpty() ) {	×
215	ControlledVocabularyValue naValue = new ControlledVocabularyValue();	×
216	naValue.setStrValue(DatasetField.NA_VALUE);	×
217	datasetFieldService.save(naValue);	×
218	return ok("NA value created.");	×
219
220	} else {
221	return ok("NA value exists.");	×
222	}
223	}
224
225	public enum HeaderType {	1✔
226	METADATABLOCK, DATASETFIELD, CONTROLLEDVOCABULARY	1✔
227	}
228
229	@POST
230	@Consumes("text/tab-separated-values")
231	@Path("load")
232	public Response loadDatasetFields(File file) {
233	ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "loadDatasetFields");	×
234	alr.setInfo( file.getName() );	×
235	BufferedReader br = null;	×
236	String line;
237	String splitBy = "\t";	×
238	int lineNumber = 0;	×
239	HeaderType header = null;	×
240	JsonArrayBuilder responseArr = Json.createArrayBuilder();	×
241	String[] values = null;	×
242	try {
243	br = new BufferedReader(new FileReader("/" + file));	×
244	while ((line = br.readLine()) != null) {	×
245	lineNumber++;	×
246	values = line.split(splitBy);	×
247	if (values[0].startsWith("#")) { // Header row	×
248	switch (values[0]) {	×
249	case "#metadataBlock":
250	header = HeaderType.METADATABLOCK;	×
251	break;	×
252	case "#datasetField":
253	header = HeaderType.DATASETFIELD;	×
254	break;	×
255	case "#controlledVocabulary":
256	header = HeaderType.CONTROLLEDVOCABULARY;	×
257	break;	×
258	default:
259	throw new IOException("Encountered unknown #header type at line lineNumber " + lineNumber);	×
260	}	×
261	} else {
262	switch (header) {	×
263	case METADATABLOCK:
264	responseArr.add( Json.createObjectBuilder()	×
265	.add("name", parseMetadataBlock(values))	×
266	.add("type", "MetadataBlock"));	×
267	break;	×
268
269	case DATASETFIELD:
270	responseArr.add( Json.createObjectBuilder()	×
271	.add("name", parseDatasetField(values))	×
272	.add("type", "DatasetField") );	×
273	break;	×
274
275	case CONTROLLEDVOCABULARY:
276	responseArr.add( Json.createObjectBuilder()	×
277	.add("name", parseControlledVocabulary(values))	×
278	.add("type", "Controlled Vocabulary") );	×
279	break;	×
280
281	default:
282	throw new IOException("No #header defined in file.");	×
283
284	}
285	}
286	}
287	} catch (FileNotFoundException e) {	×
288	alr.setActionResult(ActionLogRecord.Result.BadRequest);	×
289	alr.setInfo( alr.getInfo() + "// file not found");	×
290	return error(Status.EXPECTATION_FAILED, "File not found");	×
291
292	} catch (ArrayIndexOutOfBoundsException e) {	×
293	String message = getArrayIndexOutOfBoundMessage(header, lineNumber, values.length);	×
294	logger.log(Level.WARNING, message, e);	×
295	alr.setActionResult(ActionLogRecord.Result.InternalError);	×
296	alr.setInfo(alr.getInfo() + "// " + message);	×
297	return error(Status.INTERNAL_SERVER_ERROR, message);	×
298
299	} catch (Exception e) {	×
300	String message = getGeneralErrorMessage(header, lineNumber, e.getMessage());	×
301	logger.log(Level.WARNING, message, e);	×
302	alr.setActionResult(ActionLogRecord.Result.InternalError);	×
303	alr.setInfo( alr.getInfo() + "// " + message);	×
304	return error(Status.INTERNAL_SERVER_ERROR, message);	×
305
306	} finally {
307	if (br != null) {	×
308	try {
309	br.close();	×
310	} catch (IOException e) {	×
311	logger.log(Level.WARNING, "Error closing the reader while importing Dataset Fields.");	×
312	}	×
313	}
314	actionLogSvc.log(alr);	×
315	}
316
317	return ok( Json.createObjectBuilder().add("added", responseArr) );	×
318	}
319
320	/**
321	* Provide a general error message including the part and line number
322	* @param header
323	* @param lineNumber
324	* @param message
325	* @return
326	*/
327	public String getGeneralErrorMessage(HeaderType header, int lineNumber, String message) {
328	List<String> arguments = new ArrayList<>();	1✔
329	arguments.add(header.name());	1✔
330	arguments.add(String.valueOf(lineNumber));	1✔
331	arguments.add(message);	1✔
332	return BundleUtil.getStringFromBundle("api.admin.datasetfield.load.GeneralErrorMessage", arguments);	1✔
333	}
334
335	/**
336	* Turn ArrayIndexOutOfBoundsException into an informative error message
337	* @param lineNumber
338	* @param header
339	* @param e
340	* @return
341	*/
342	public String getArrayIndexOutOfBoundMessage(HeaderType header,
343	int lineNumber,
344	int wrongIndex) {
345
346	List<String> columns = getColumnsByHeader(header);	1✔
347
348	String column = columns.get(wrongIndex - 1);	1✔
349	List<String> arguments = new ArrayList<>();	1✔
350	arguments.add(header.name());	1✔
351	arguments.add(String.valueOf(lineNumber));	1✔
352	arguments.add(column);	1✔
353	arguments.add(String.valueOf(wrongIndex + 1));	1✔
354	return BundleUtil.getStringFromBundle(	1✔
355	"api.admin.datasetfield.load.ArrayIndexOutOfBoundMessage",
356	arguments
357	);
358	}
359
360	/**
361	* Get the list of columns by the type of header
362	* @param header
363	* @return
364	*/
365	private List<String> getColumnsByHeader(HeaderType header) {
366	List<String> columns = null;	1✔
367	if (header.equals(HeaderType.METADATABLOCK)) {	1✔
368	columns = Arrays.asList("name", "dataverseAlias", "displayName");	×
369	} else if (header.equals(HeaderType.DATASETFIELD)) {	1✔
370	columns = Arrays.asList("name", "title", "description", "watermark",	1✔
371	"fieldType", "displayOrder", "displayFormat", "advancedSearchField",
372	"allowControlledVocabulary", "allowmultiples", "facetable",
373	"displayoncreate", "required", "parent", "metadatablock_id");
374	} else if (header.equals(HeaderType.CONTROLLEDVOCABULARY)) {	×
375	columns = Arrays.asList("DatasetField", "Value", "identifier", "displayOrder");	×
376	}
377
378	return columns;	1✔
379	}
380
381	private String parseMetadataBlock(String[] values) {
382	//Test to see if it exists by name
383	MetadataBlock mdb = metadataBlockService.findByName(values[1]);	×
384	if (mdb == null){	×
385	mdb = new MetadataBlock();	×
386	}
387	mdb.setName(values[1]);	×
388	if (!values[2].isEmpty()){	×
389	mdb.setOwner(dataverseService.findByAlias(values[2]));	×
390	}
391	mdb.setDisplayName(values[3]);	×
392	if (values.length>4 && !StringUtils.isEmpty(values[4])) {	×
393	mdb.setNamespaceUri(values[4]);	×
394	}
395
396	metadataBlockService.save(mdb);	×
397	return mdb.getName();	×
398	}
399
400	private String parseDatasetField(String[] values) {
401
402	//First see if it exists
403	DatasetFieldType dsf = datasetFieldService.findByName(values[1]);	×
404	if (dsf == null) {	×
405	//if not create new
406	dsf = new DatasetFieldType();	×
407	}
408	//add(update) values
409	dsf.setName(values[1]);	×
410	dsf.setTitle(values[2]);	×
411	dsf.setDescription(values[3]);	×
412	dsf.setWatermark(values[4]);	×
413	dsf.setFieldType(FieldType.valueOf(values[5].toUpperCase()));	×
414	dsf.setDisplayOrder(Integer.parseInt(values[6]));	×
415	dsf.setDisplayFormat(values[7]);	×
416	dsf.setAdvancedSearchFieldType(Boolean.parseBoolean(values[8]));	×
417	dsf.setAllowControlledVocabulary(Boolean.parseBoolean(values[9]));	×
418	dsf.setAllowMultiples(Boolean.parseBoolean(values[10]));	×
419	dsf.setFacetable(Boolean.parseBoolean(values[11]));	×
420	dsf.setDisplayOnCreate(Boolean.parseBoolean(values[12]));	×
421	dsf.setRequired(Boolean.parseBoolean(values[13]));	×
422	if (!StringUtils.isEmpty(values[14])) {	×
423	dsf.setParentDatasetFieldType(datasetFieldService.findByName(values[14]));	×
424	} else {
425	dsf.setParentDatasetFieldType(null);	×
426	}
427	dsf.setMetadataBlock(dataverseService.findMDBByName(values[15]));	×
428	if(values.length>16 && !StringUtils.isEmpty(values[16])) {	×
429	dsf.setUri(values[16]);	×
430	}
431	datasetFieldService.save(dsf);	×
432	return dsf.getName();	×
433	}
434
435	private String parseControlledVocabulary(String[] values) {
436
437	DatasetFieldType dsv = datasetFieldService.findByName(values[1]);	×
438	//See if it already exists
439	/*
440	Matching relies on assumption that only one cv value will exist for a given identifier or display value
441	If the lookup queries return multiple matches then retval is null
442	*/
443	//First see if cvv exists based on display name
444	ControlledVocabularyValue cvv = datasetFieldService.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsv, values[2], true);	×
445
446	//then see if there's a match on identifier
447	ControlledVocabularyValue cvvi = null;	×
448	if (values[3] != null && !values[3].trim().isEmpty()){	×
449	cvvi = datasetFieldService.findControlledVocabularyValueByDatasetFieldTypeAndIdentifier(dsv, values[3]);	×
450	}
451
452	//if there's a match on identifier use it
453	if (cvvi != null){	×
454	cvv = cvvi;	×
455	}
456
457	//if there's no match create a new one
458	if (cvv == null) {	×
459	cvv = new ControlledVocabularyValue();	×
460	cvv.setDatasetFieldType(dsv);	×
461	}
462
463	// Alternate variants for this controlled vocab. value:
464
465	// Note that these are overwritten every time:
466	cvv.getControlledVocabAlternates().clear();	×
467	// - meaning, if an alternate has been removed from the tsv file,
468	// it will be removed from the database! -- L.A. 5.4
469
470	for (int i = 5; i < values.length; i++) {	×
471	ControlledVocabAlternate alt = new ControlledVocabAlternate();	×
472	alt.setDatasetFieldType(dsv);	×
473	alt.setControlledVocabularyValue(cvv);	×
474	alt.setStrValue(values[i]);	×
475	cvv.getControlledVocabAlternates().add(alt);	×
476	}
477
478	cvv.setStrValue(values[2]);	×
479	cvv.setIdentifier(values[3]);	×
480	cvv.setDisplayOrder(Integer.parseInt(values[4]));	×
481	datasetFieldService.save(cvv);	×
482	return cvv.getStrValue();	×
483	}
484
485
486	@POST
487	@Consumes("application/zip")
488	@Path("loadpropertyfiles")
489	public Response loadLanguagePropertyFile(File inputFile) {
NEW 490	try (ZipFile file = new ZipFile(inputFile)) {	×
491	//Get file entries
492	Enumeration<? extends ZipEntry> entries = file.entries();	×
493
494	//We will unzip files in this folder
495	String dataverseLangDirectory = getDataverseLangDirectory();	×
496
497	//Iterate over entries
498	while (entries.hasMoreElements())	×
499	{
500	ZipEntry entry = entries.nextElement();	×
501	String dataverseLangFileName = dataverseLangDirectory + "/" + entry.getName();	×
NEW 502	File entryFile = new File(dataverseLangFileName);	×
NEW 503	String canonicalPath = entryFile.getCanonicalPath();	×
NEW 504	if (canonicalPath.startsWith(dataverseLangDirectory + "/")) {	×
NEW 505	try (FileOutputStream fileOutput = new FileOutputStream(dataverseLangFileName)) {	×
506
NEW 507	InputStream is = file.getInputStream(entry);	×
NEW 508	BufferedInputStream bis = new BufferedInputStream(is);	×
509
NEW 510	while (bis.available() > 0) {	×
NEW 511	fileOutput.write(bis.read());	×
512	}
513	}
514	} else {
NEW 515	logger.log(Level.SEVERE, "Zip Slip prevented: uploaded zip file tried to write to {}", canonicalPath);	×
NEW 516	return Response.status(400).entity("The zip file includes an illegal file path").build();	×
517	}
518	}	×
519	}	×
NEW 520	catch(IOException e) {	×
NEW 521	logger.log(Level.SEVERE, "Reading the language property zip file failed", e);	×
522	return Response.status(500).entity("Internal server error. More details available at the server logs.").build();	×
523	}	×
524
525	return Response.status(200).entity("Uploaded the file successfully ").build();	×
526	}
527
528	public static String getDataverseLangDirectory() {
529	String dataverseLangDirectory = System.getProperty("dataverse.lang.directory");	×
530	if (dataverseLangDirectory == null \|\| dataverseLangDirectory.equals("")) {	×
531	dataverseLangDirectory = "/tmp/files";	×
532	}
533
534	if (!Files.exists(Paths.get(dataverseLangDirectory))) {	×
535	try {
536	Files.createDirectories(Paths.get(dataverseLangDirectory));	×
537	} catch (IOException ex) {	×
538	logger.severe("Failed to create dataverseLangDirectory: " + dataverseLangDirectory );	×
539	return null;	×
540	}	×
541	}
542
543	return dataverseLangDirectory;	×
544	}
545
546	}

IQSS / dataverse / #22693

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous