#22540

Committed 17 Jun 2024 02:25PM CUT coverage: 20.63% (+0.06%) from 20.574%

Build # #22540

Build Type

Pull #10543

github

Committed by

stevenwinship

Commit Message

remove hard coded datasetschema.json and load from file

Pull Request Pull Request #10543: Improved JSON Schema validation for datasets

Run Details

66 of 106 new or added lines in 2 files covered. (62.26%)

1 existing line in 1 file now uncovered.

17307 of 83891 relevant lines covered (20.63%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.53

/src/main/java/edu/harvard/iq/dataverse/validation/JSONDataValidation.java

package edu.harvard.iq.dataverse.validation;

import com.mashape.unirest.http.JsonNode;
import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
import edu.harvard.iq.dataverse.DatasetFieldType;
import edu.harvard.iq.dataverse.util.BundleUtil;
import jakarta.enterprise.inject.spi.CDI;
import org.everit.json.schema.Schema;
import org.everit.json.schema.ValidationException;
import org.json.JSONArray;

import java.util.*;
import java.util.logging.Logger;
import java.util.stream.Collectors;

public class JSONDataValidation {
    private static final Logger logger = Logger.getLogger(JSONDataValidation.class.getCanonicalName());
    private static DatasetFieldServiceBean datasetFieldService = null;

    /**
     *
     * @param schema Schema file defining the JSON objects to be validated
     * @param jsonInput JSON string to validate against the schema
     * @throws ValidationException
     */
    public static void validate(Schema schema, Map<String, Map<String, List<String>>> schemaChildMap, String jsonInput) throws ValidationException {
        if (datasetFieldService == null) {
            datasetFieldService = CDI.current().select(DatasetFieldServiceBean.class).get();
        }
        JsonNode node = new JsonNode(jsonInput);
        if (node.isArray()) {
            JSONArray arrayNode = node.getArray();
            validateObject(schema, schemaChildMap, "root", arrayNode.toList());
        } else {
            node.getObject().toMap().forEach((k,v) -> {
                validateObject(schema, schemaChildMap, k, (v instanceof JSONArray) ? ((JSONArray) v).toList() : v);
            });
        }
    }

    /*
     * Validate objects recursively
     */
    private static void validateObject(Schema schema, Map<String, Map<String,List<String>>> schemaChildMap, String key, Object value) {
        if (value instanceof Map<?,?>) {
            validateSchemaObject(schema, schemaChildMap, key, (Map) value);

            ((Map<?, ?>) value).entrySet().forEach(e -> {
                validateObject(schema, schemaChildMap, (String) e.getKey(), e.getValue());
            });
        } else if (value instanceof List) {
            ((List<?>) value).listIterator().forEachRemaining(v -> {
                validateObject(schema, schemaChildMap, key, v);
            });
        }
    }

    /*
     * Validate objects specific to a type. Currently only validating Datasets
     */
    private static void validateSchemaObject(Schema schema, Map<String, Map<String,List<String>>> schemaChildMap, String key, Map valueMap) {
        if (schema.definesProperty("datasetVersion")) {
            validateDatasetObject(schema, schemaChildMap, key, valueMap);
        }
    }

    /*
     * Specific validation for Dataset objects
     */
    private static void validateDatasetObject(Schema schema, Map<String, Map<String,List<String>>> schemaChildMap, String key, Map valueMap) {
        if (valueMap != null && valueMap.containsKey("typeClass")) {
            validateTypeClass(schema, schemaChildMap, key, valueMap, valueMap.get("value"), "dataset");
        }
    }

    /*
     * key: The name of the parent object
     * valueMap: Map of all the metadata of the object
     * value: The value field of the object
     * messageType: Refers to the parent: if this is an object from a dataset the messageType would be 'dataset'
     *              This needs to match the Bundle.properties for mapping the error messages when an exception occurs
     *
     *  Rules for typeClass:
     *      The contents of value depend on the field attributes
     *      if single/primitive, value is a String
     *      if multiple, value is a JsonArray
     *         multiple/primitive: each JsonArray element will contain String
     *         multiple/compound: each JsonArray element will contain Set of FieldDTOs
     */
    private static void validateTypeClass(Schema schema, Map<String, Map<String,List<String>>> schemaChildMap, String key, Map valueMap, Object value, String messageType) {

        String typeClass = valueMap.containsKey("typeClass") ? valueMap.get("typeClass").toString() : "";
        String typeName = valueMap.containsKey("typeName") ? valueMap.get("typeName").toString() : "";
        boolean multiple = Boolean.valueOf(String.valueOf(valueMap.getOrDefault("multiple", "false")));

        // make sure there is a value since 'value' is required
        if (value == null) {
            throwValidationException("value.missing", List.of(key, typeName));
        }

        if (multiple && !(value instanceof List<?>)) {
            throwValidationException("notlist.multiple", List.of(key, typeName, typeClass));
        }
        if (!multiple && value instanceof List<?>) {
            throwValidationException("list.notmultiple", List.of(key, typeName));
        }
        if ("primitive".equals(typeClass) && !multiple && !(value instanceof String)) {
            throwValidationException("type", List.of(key, typeName, typeClass));
        }
        if ("primitive".equals(typeClass) && multiple) {
            ((List<?>) value).listIterator().forEachRemaining(primitive -> {
                if (!(primitive instanceof String)) {
                    throwValidationException("type", List.of(key, typeName, typeClass));
                }
            });
        }
        if ("compound".equals(typeClass)) {
            if (multiple && value instanceof List<?>) {
                ((List<?>) value).listIterator().forEachRemaining(item -> {
                    if (!(item instanceof Map<?, ?>)) {
                        throwValidationException("compound", List.of(key, typeName, typeClass));
                    } else {
                        ((Map) item).forEach((k,val) -> {
                            if (!(val instanceof Map<?, ?>)) {
                                throwValidationException("compound", List.of(key, typeName, typeClass));
                            }
                            // validate mismatch between compound object key and typeName in value
                            String valTypeName = ((Map<?, ?>) val).containsKey("typeName") ? (String) ((Map<?, ?>) val).get("typeName") : "";
                            if (!k.equals(valTypeName)) {
                                throwValidationException("compound.mismatch", List.of((String) k, valTypeName));
                            }
                        });
                        validateChildren(schema, schemaChildMap, key, ((Map) item).values(), typeName, messageType);
                    }
                });
            }
        }

        if ("controlledVocabulary".equals(typeClass)) {
            DatasetFieldType dsft = datasetFieldService.findByName(typeName);
            if (value instanceof List<?>) {
                ((List<?>) value).listIterator().forEachRemaining(cvv -> {
                    if (datasetFieldService.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, (String) cvv, true) == null) {
                        throwValidationException("dataset", "cvv.missing", List.of(key, typeName, (String) cvv));
                    }
                });
            } else {
                if (datasetFieldService.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, (String) value, true) == null) {
                    throwValidationException("dataset", "cvv.missing", List.of(key, typeName, (String) value));
                }
            }
        }
    }

    // If value is another object or list of objects that need to be further validated then childType refers to the parent
    // Example: If this is a dsDescriptionValue from a dataset the messageType would be dataset.dsDescriptionValue
    // This needs to match the Bundle.properties for mapping the error messages when an exception occurs
    private static void validateChildren(Schema schema, Map<String, Map<String,List<String>>> schemaChildMap, String key, Collection<Object> children, String typeName, String messageType) {
        if (children == null || children.isEmpty()) {
            return;
        }
        List<String> requiredFields = new ArrayList<>();
        requiredFields.addAll((List)schemaChildMap.getOrDefault(typeName, Collections.EMPTY_MAP).getOrDefault("required", Collections.EMPTY_LIST));
        List<String> allowedFields = (List)schemaChildMap.getOrDefault(typeName, Collections.EMPTY_MAP).getOrDefault("allowed", Collections.EMPTY_LIST);
        children.forEach(child -> {
            if (child instanceof Map<?, ?>) {
                String childTypeName = ((Map<?, ?>) child).containsKey("typeName") ? (String)((Map<?, ?>) child).get("typeName") : "";
                if (!allowedFields.isEmpty() && !allowedFields.contains(childTypeName)) {
                    throwValidationException(messageType, "invalidType", List.of(typeName, childTypeName, allowedFields.stream().collect(Collectors.joining(", "))));
                }
                if (!requiredFields.isEmpty() && requiredFields.contains(childTypeName)) {
                    requiredFields.remove(childTypeName);
                }
            }
        });
        if (!requiredFields.isEmpty()) {
            throwValidationException(messageType, "required.missing", List.of(typeName, requiredFields.stream().collect(Collectors.joining(", ")), typeName));
        }
    }
    private static void throwValidationException(String key, List<String> argList) {
        throw new ValidationException(BundleUtil.getStringFromBundle("schema.validation.exception." + key, argList));
    }
    private static void throwValidationException(String type, String message, List<String> argList) {
        if (type != null) {
            throwValidationException(type + "." + message, argList);
        } else {
            throwValidationException(message, argList);
        }
    }
}

1	package edu.harvard.iq.dataverse.validation;
2
3	import com.mashape.unirest.http.JsonNode;
4	import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
5	import edu.harvard.iq.dataverse.DatasetFieldType;
6	import edu.harvard.iq.dataverse.util.BundleUtil;
7	import jakarta.enterprise.inject.spi.CDI;
8	import org.everit.json.schema.Schema;
9	import org.everit.json.schema.ValidationException;
10	import org.json.JSONArray;
11
12	import java.util.*;
13	import java.util.logging.Logger;
14	import java.util.stream.Collectors;
15
NEW 16	public class JSONDataValidation {	×
17	private static final Logger logger = Logger.getLogger(JSONDataValidation.class.getCanonicalName());	1✔
18	private static DatasetFieldServiceBean datasetFieldService = null;	1✔
19
20	/**
21	*
22	* @param schema Schema file defining the JSON objects to be validated
23	* @param jsonInput JSON string to validate against the schema
24	* @throws ValidationException
25	*/
26	public static void validate(Schema schema, Map<String, Map<String, List<String>>> schemaChildMap, String jsonInput) throws ValidationException {
27	if (datasetFieldService == null) {	1✔
NEW 28	datasetFieldService = CDI.current().select(DatasetFieldServiceBean.class).get();	×
29	}
30	JsonNode node = new JsonNode(jsonInput);	1✔
31	if (node.isArray()) {	1✔
NEW 32	JSONArray arrayNode = node.getArray();	×
NEW 33	validateObject(schema, schemaChildMap, "root", arrayNode.toList());	×
NEW 34	} else {	×
35	node.getObject().toMap().forEach((k,v) -> {	1✔
36	validateObject(schema, schemaChildMap, k, (v instanceof JSONArray) ? ((JSONArray) v).toList() : v);	1✔
37	});	1✔
38	}
39	}	1✔
40
41	/*
42	* Validate objects recursively
43	*/
44	private static void validateObject(Schema schema, Map<String, Map<String,List<String>>> schemaChildMap, String key, Object value) {
45	if (value instanceof Map<?,?>) {	1✔
46	validateSchemaObject(schema, schemaChildMap, key, (Map) value);	1✔
47
48	((Map<?, ?>) value).entrySet().forEach(e -> {	1✔
49	validateObject(schema, schemaChildMap, (String) e.getKey(), e.getValue());	1✔
50	});	1✔
51	} else if (value instanceof List) {	1✔
52	((List<?>) value).listIterator().forEachRemaining(v -> {	1✔
53	validateObject(schema, schemaChildMap, key, v);	1✔
54	});	1✔
55	}
56	}	1✔
57
58	/*
59	* Validate objects specific to a type. Currently only validating Datasets
60	*/
61	private static void validateSchemaObject(Schema schema, Map<String, Map<String,List<String>>> schemaChildMap, String key, Map valueMap) {
62	if (schema.definesProperty("datasetVersion")) {	1✔
63	validateDatasetObject(schema, schemaChildMap, key, valueMap);	1✔
64	}
65	}	1✔
66
67	/*
68	* Specific validation for Dataset objects
69	*/
70	private static void validateDatasetObject(Schema schema, Map<String, Map<String,List<String>>> schemaChildMap, String key, Map valueMap) {
71	if (valueMap != null && valueMap.containsKey("typeClass")) {	1✔
72	validateTypeClass(schema, schemaChildMap, key, valueMap, valueMap.get("value"), "dataset");	1✔
73	}
74	}	1✔
75
76	/*
77	* key: The name of the parent object
78	* valueMap: Map of all the metadata of the object
79	* value: The value field of the object
80	* messageType: Refers to the parent: if this is an object from a dataset the messageType would be 'dataset'
81	* This needs to match the Bundle.properties for mapping the error messages when an exception occurs
82	*
83	* Rules for typeClass:
84	* The contents of value depend on the field attributes
85	* if single/primitive, value is a String
86	* if multiple, value is a JsonArray
87	* multiple/primitive: each JsonArray element will contain String
88	* multiple/compound: each JsonArray element will contain Set of FieldDTOs
89	*/
90	private static void validateTypeClass(Schema schema, Map<String, Map<String,List<String>>> schemaChildMap, String key, Map valueMap, Object value, String messageType) {
91
92	String typeClass = valueMap.containsKey("typeClass") ? valueMap.get("typeClass").toString() : "";	1✔
93	String typeName = valueMap.containsKey("typeName") ? valueMap.get("typeName").toString() : "";	1✔
94	boolean multiple = Boolean.valueOf(String.valueOf(valueMap.getOrDefault("multiple", "false")));	1✔
95
96	// make sure there is a value since 'value' is required
97	if (value == null) {	1✔
NEW 98	throwValidationException("value.missing", List.of(key, typeName));	×
99	}
100
101	if (multiple && !(value instanceof List<?>)) {	1✔
NEW 102	throwValidationException("notlist.multiple", List.of(key, typeName, typeClass));	×
103	}
104	if (!multiple && value instanceof List<?>) {	1✔
NEW 105	throwValidationException("list.notmultiple", List.of(key, typeName));	×
106	}
107	if ("primitive".equals(typeClass) && !multiple && !(value instanceof String)) {	1✔
NEW 108	throwValidationException("type", List.of(key, typeName, typeClass));	×
109	}
110	if ("primitive".equals(typeClass) && multiple) {	1✔
NEW 111	((List<?>) value).listIterator().forEachRemaining(primitive -> {	×
NEW 112	if (!(primitive instanceof String)) {	×
NEW 113	throwValidationException("type", List.of(key, typeName, typeClass));	×
114	}
NEW 115	});	×
116	}
117	if ("compound".equals(typeClass)) {	1✔
118	if (multiple && value instanceof List<?>) {	1✔
119	((List<?>) value).listIterator().forEachRemaining(item -> {	1✔
120	if (!(item instanceof Map<?, ?>)) {	1✔
NEW 121	throwValidationException("compound", List.of(key, typeName, typeClass));	×
122	} else {
123	((Map) item).forEach((k,val) -> {	1✔
124	if (!(val instanceof Map<?, ?>)) {	1✔
NEW 125	throwValidationException("compound", List.of(key, typeName, typeClass));	×
126	}
127	// validate mismatch between compound object key and typeName in value
128	String valTypeName = ((Map<?, ?>) val).containsKey("typeName") ? (String) ((Map<?, ?>) val).get("typeName") : "";	1✔
129	if (!k.equals(valTypeName)) {	1✔
NEW 130	throwValidationException("compound.mismatch", List.of((String) k, valTypeName));	×
131	}
132	});	1✔
133	validateChildren(schema, schemaChildMap, key, ((Map) item).values(), typeName, messageType);	1✔
134	}
135	});	1✔
136	}
137	}
138
139	if ("controlledVocabulary".equals(typeClass)) {	1✔
140	DatasetFieldType dsft = datasetFieldService.findByName(typeName);	1✔
141	if (value instanceof List<?>) {	1✔
142	((List<?>) value).listIterator().forEachRemaining(cvv -> {	1✔
143	if (datasetFieldService.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, (String) cvv, true) == null) {	1✔
NEW 144	throwValidationException("dataset", "cvv.missing", List.of(key, typeName, (String) cvv));	×
145	}
146	});	1✔
147	} else {
NEW 148	if (datasetFieldService.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, (String) value, true) == null) {	×
NEW 149	throwValidationException("dataset", "cvv.missing", List.of(key, typeName, (String) value));	×
150	}
151	}
152	}
153	}	1✔
154
155	// If value is another object or list of objects that need to be further validated then childType refers to the parent
156	// Example: If this is a dsDescriptionValue from a dataset the messageType would be dataset.dsDescriptionValue
157	// This needs to match the Bundle.properties for mapping the error messages when an exception occurs
158	private static void validateChildren(Schema schema, Map<String, Map<String,List<String>>> schemaChildMap, String key, Collection<Object> children, String typeName, String messageType) {
159	if (children == null \|\| children.isEmpty()) {	1✔
NEW 160	return;	×
161	}
162	List<String> requiredFields = new ArrayList<>();	1✔
163	requiredFields.addAll((List)schemaChildMap.getOrDefault(typeName, Collections.EMPTY_MAP).getOrDefault("required", Collections.EMPTY_LIST));	1✔
164	List<String> allowedFields = (List)schemaChildMap.getOrDefault(typeName, Collections.EMPTY_MAP).getOrDefault("allowed", Collections.EMPTY_LIST);	1✔
165	children.forEach(child -> {	1✔
166	if (child instanceof Map<?, ?>) {	1✔
167	String childTypeName = ((Map<?, ?>) child).containsKey("typeName") ? (String)((Map<?, ?>) child).get("typeName") : "";	1✔
168	if (!allowedFields.isEmpty() && !allowedFields.contains(childTypeName)) {	1✔
NEW 169	throwValidationException(messageType, "invalidType", List.of(typeName, childTypeName, allowedFields.stream().collect(Collectors.joining(", "))));	×
170	}
171	if (!requiredFields.isEmpty() && requiredFields.contains(childTypeName)) {	1✔
172	requiredFields.remove(childTypeName);	1✔
173	}
174	}
175	});	1✔
176	if (!requiredFields.isEmpty()) {	1✔
NEW 177	throwValidationException(messageType, "required.missing", List.of(typeName, requiredFields.stream().collect(Collectors.joining(", ")), typeName));	×
178	}
179	}	1✔
180	private static void throwValidationException(String key, List<String> argList) {
181	throw new ValidationException(BundleUtil.getStringFromBundle("schema.validation.exception." + key, argList));	1✔
182	}
183	private static void throwValidationException(String type, String message, List<String> argList) {
184	if (type != null) {	1✔
NEW 185	throwValidationException(type + "." + message, argList);	×
186	} else {
NEW 187	throwValidationException(message, argList);	×
188	}
NEW 189	}	×
190	}

IQSS / dataverse / #22540

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous