#356

Committed 29 Aug 2024 01:38PM UTC coverage: 72.027% (-0.6%) from 72.606%

Build # #356

Build Type

Pull #218

github

Committed by

web-flow

Commit Message

Merge 160bfe071 into 86457be39

Pull Request Pull Request #218: Validation speedup experiments

Run Details

65 of 93 new or added lines in 4 files covered. (69.89%)

4 existing lines in 3 files now uncovered.

860 of 1194 relevant lines covered (72.03%)

0.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

56.92

/src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java

package edu.kit.datamanager.pit.pitservice.impl;

import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import edu.kit.datamanager.pit.common.ExternalServiceException;
import edu.kit.datamanager.pit.common.RecordValidationException;
import edu.kit.datamanager.pit.configuration.ApplicationProperties;
import edu.kit.datamanager.pit.domain.PIDRecord;
import edu.kit.datamanager.pit.domain.TypeDefinition;
import edu.kit.datamanager.pit.pitservice.IValidationStrategy;
import edu.kit.datamanager.pit.util.TypeValidationUtils;

import java.util.Arrays;
import java.util.List;
import java.util.concurrent.*;
import java.util.stream.Collectors;

import org.apache.commons.lang3.stream.Streams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

/**
 * Validates a PID record using embedded profile(s).
 * 
 * - checks if all mandatory attributes are present
 * - validates all available attributes
 * - fails if an attribute is not defined within the profile
 */
public class EmbeddedStrictValidatorStrategy implements IValidationStrategy {

    private static final Logger LOG = LoggerFactory.getLogger(EmbeddedStrictValidatorStrategy.class);

    @Autowired
    public AsyncLoadingCache<String, TypeDefinition> typeLoader;

    @Autowired
    ApplicationProperties applicationProps;

    @Override
    public void validate(PIDRecord pidRecord) throws RecordValidationException, ExternalServiceException {
        String profileKey = applicationProps.getProfileKey();
        if (!pidRecord.hasProperty(profileKey)) {
            throw new RecordValidationException(
                    pidRecord,
                    "Profile attribute not found. Expected key: " + profileKey);
        }

        String[] profilePIDs = pidRecord.getPropertyValues(profileKey);
        boolean hasProfile = profilePIDs.length > 0;
        if (!hasProfile) {
            throw new RecordValidationException(
                    pidRecord,
                    "Profile attribute " + profileKey + " has no values.");
        }

        List<CompletableFuture<?>> futures = Streams.stream(Arrays.stream(profilePIDs))
                .map(profilePID -> {
                    try {
                        return this.typeLoader.get(profilePID)
                                .thenAcceptAsync(profileDefinition -> {
                                    if (profileDefinition == null) {
                                        LOG.error("No type definition found for identifier {}.", profilePID);
                                        throw new RecordValidationException(
                                                pidRecord,
                                                String.format("No type found for identifier %s.", profilePID));
                                    }
                                    this.strictProfileValidation(pidRecord, profileDefinition);
                                });
                    } catch (RuntimeException e) {
                        LOG.error("Could not resolve identifier {}.", profilePID);
                        throw new ExternalServiceException(
                                applicationProps.getTypeRegistryUri().toString());
                    }
                })
                .collect(Collectors.toList());
        try {
            CompletableFuture.allOf(futures.toArray(new CompletableFuture<?>[0])).join();
        } catch (CompletionException e) {
            throwRecordValidationExceptionCause(e);
            throw new ExternalServiceException(
                    applicationProps.getTypeRegistryUri().toString());
        } catch (CancellationException e) {
            throwRecordValidationExceptionCause(e);
            throw new RecordValidationException(
                    pidRecord,
                    String.format("Validation task was cancelled for %s. Please report.", pidRecord.getPid()));
        }
    }

    private static void throwRecordValidationExceptionCause(Throwable e) {
        if (e.getCause() instanceof RecordValidationException rve) {
            throw rve;
        }
    }

    /**
     * Exceptions indicate failure. No Exceptions mean success.
     * 
     * @param pidRecord the PID record to validate.
     * @param profile   the profile to validate against.
     * @throws RecordValidationException with error message on validation errors.
     */
    private void strictProfileValidation(PIDRecord pidRecord, TypeDefinition profile) throws RecordValidationException {
        // if (profile.hasSchema()) {
        // TODO issue https://github.com/kit-data-manager/pit-service/issues/104
        // validate using schema and you are done (strict validation)
        // String jsonRecord = ""; // TODO format depends on schema source
        // return profile.validate(jsonRecord);
        // }

        LOG.trace("Validating PID record against profile {}.", profile.getIdentifier());

        TypeValidationUtils.checkMandatoryAttributes(pidRecord, profile);

        for (String attributeKey : pidRecord.getPropertyIdentifiers()) {
            LOG.trace("Checking PID record key {}.", attributeKey);

            TypeDefinition type = profile.getSubTypes().get(attributeKey);
            if (type == null) {
                LOG.error("No sub-type found for key {}.", attributeKey);
                // TODO try to resolve it (for later when we support "allow additional
                // attributes")
                // if profile.allowsAdditionalAttributes() {...} else
                throw new RecordValidationException(
                        pidRecord,
                        String.format("Attribute %s is not allowed in profile %s",
                                attributeKey,
                                profile.getIdentifier()));
            }

            validateValuesForKey(pidRecord, attributeKey, type);
        }
        LOG.debug("successfully validated {}", profile.getIdentifier());
    }

    /**
     * Validates all values of an attribute against a given type definition.
     * 
     * @param pidRecord the record containing the attribute and value.
     * @param attributeKey the attribute to check the values for.
     * @param type the type definition to check against.
     * @throws RecordValidationException on error.
     */
    private void validateValuesForKey(PIDRecord pidRecord, String attributeKey, TypeDefinition type)
            throws RecordValidationException {
        String[] values = pidRecord.getPropertyValues(attributeKey);
        for (String value : values) {
            if (value == null) {
                LOG.error("'null' record value found for key {}.", attributeKey);
                throw new RecordValidationException(
                        pidRecord,
                        String.format("Validation of value %s against type %s failed.",
                                value,
                                type.getIdentifier()));
            }

            if (!type.validate(value)) {
                LOG.error("Validation of value {} against type {} failed.", value, type.getIdentifier());
                throw new RecordValidationException(
                        pidRecord,
                        String.format("Validation of value %s against type %s failed.",
                                value,
                                type.getIdentifier()));
            }
        }
    }
}

1	package edu.kit.datamanager.pit.pitservice.impl;
2
3	import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
4	import edu.kit.datamanager.pit.common.ExternalServiceException;
5	import edu.kit.datamanager.pit.common.RecordValidationException;
6	import edu.kit.datamanager.pit.configuration.ApplicationProperties;
7	import edu.kit.datamanager.pit.domain.PIDRecord;
8	import edu.kit.datamanager.pit.domain.TypeDefinition;
9	import edu.kit.datamanager.pit.pitservice.IValidationStrategy;
10	import edu.kit.datamanager.pit.util.TypeValidationUtils;
11
12	import java.util.Arrays;
13	import java.util.List;
14	import java.util.concurrent.*;
15	import java.util.stream.Collectors;
16
17	import org.apache.commons.lang3.stream.Streams;
18	import org.slf4j.Logger;
19	import org.slf4j.LoggerFactory;
20	import org.springframework.beans.factory.annotation.Autowired;
21
22	/**
23	* Validates a PID record using embedded profile(s).
24	*
25	* - checks if all mandatory attributes are present
26	* - validates all available attributes
27	* - fails if an attribute is not defined within the profile
28	*/
29	public class EmbeddedStrictValidatorStrategy implements IValidationStrategy {	1✔
30
31	private static final Logger LOG = LoggerFactory.getLogger(EmbeddedStrictValidatorStrategy.class);	1✔
32
33	@Autowired
34	public AsyncLoadingCache<String, TypeDefinition> typeLoader;
35
36	@Autowired
37	ApplicationProperties applicationProps;
38
39	@Override
40	public void validate(PIDRecord pidRecord) throws RecordValidationException, ExternalServiceException {
41	String profileKey = applicationProps.getProfileKey();	1✔
42	if (!pidRecord.hasProperty(profileKey)) {	1✔
43	throw new RecordValidationException(	1✔
44	pidRecord,
45	"Profile attribute not found. Expected key: " + profileKey);
46	}
47
48	String[] profilePIDs = pidRecord.getPropertyValues(profileKey);	1✔
49	boolean hasProfile = profilePIDs.length > 0;	1✔
50	if (!hasProfile) {	1✔
51	throw new RecordValidationException(	×
52	pidRecord,
53	"Profile attribute " + profileKey + " has no values.");
54	}
55
56	List<CompletableFuture<?>> futures = Streams.stream(Arrays.stream(profilePIDs))	1✔
57	.map(profilePID -> {	1✔
58	try {
59	return this.typeLoader.get(profilePID)	1✔
60	.thenAcceptAsync(profileDefinition -> {	1✔
61	if (profileDefinition == null) {	1✔
NEW 62	LOG.error("No type definition found for identifier {}.", profilePID);	×
NEW 63	throw new RecordValidationException(	×
64	pidRecord,
NEW 65	String.format("No type found for identifier %s.", profilePID));	×
66	}
67	this.strictProfileValidation(pidRecord, profileDefinition);	1✔
68	});	1✔
NEW 69	} catch (RuntimeException e) {	×
NEW 70	LOG.error("Could not resolve identifier {}.", profilePID);	×
NEW 71	throw new ExternalServiceException(	×
NEW 72	applicationProps.getTypeRegistryUri().toString());	×
73	}
74	})
75	.collect(Collectors.toList());	1✔
76	try {
77	CompletableFuture.allOf(futures.toArray(new CompletableFuture<?>[0])).join();	1✔
78	} catch (CompletionException e) {	1✔
NEW 79	throwRecordValidationExceptionCause(e);	×
NEW 80	throw new ExternalServiceException(	×
NEW 81	applicationProps.getTypeRegistryUri().toString());	×
NEW 82	} catch (CancellationException e) {	×
NEW 83	throwRecordValidationExceptionCause(e);	×
NEW 84	throw new RecordValidationException(	×
85	pidRecord,
NEW 86	String.format("Validation task was cancelled for %s. Please report.", pidRecord.getPid()));	×
87	}	1✔
88	}	1✔
89
90	private static void throwRecordValidationExceptionCause(Throwable e) {
91	if (e.getCause() instanceof RecordValidationException rve) {	1✔
92	throw rve;	1✔
93	}
UNCOV 94	}	×
95
96	/**
97	* Exceptions indicate failure. No Exceptions mean success.
98	*
99	* @param pidRecord the PID record to validate.
100	* @param profile the profile to validate against.
101	* @throws RecordValidationException with error message on validation errors.
102	*/
103	private void strictProfileValidation(PIDRecord pidRecord, TypeDefinition profile) throws RecordValidationException {
104	// if (profile.hasSchema()) {
105	// TODO issue https://github.com/kit-data-manager/pit-service/issues/104
106	// validate using schema and you are done (strict validation)
107	// String jsonRecord = ""; // TODO format depends on schema source
108	// return profile.validate(jsonRecord);
109	// }
110
111	LOG.trace("Validating PID record against profile {}.", profile.getIdentifier());	1✔
112
113	TypeValidationUtils.checkMandatoryAttributes(pidRecord, profile);	1✔
114
115	for (String attributeKey : pidRecord.getPropertyIdentifiers()) {	1✔
116	LOG.trace("Checking PID record key {}.", attributeKey);	1✔
117
118	TypeDefinition type = profile.getSubTypes().get(attributeKey);	1✔
119	if (type == null) {	1✔
120	LOG.error("No sub-type found for key {}.", attributeKey);	×
121	// TODO try to resolve it (for later when we support "allow additional
122	// attributes")
123	// if profile.allowsAdditionalAttributes() {...} else
124	throw new RecordValidationException(	×
125	pidRecord,
126	String.format("Attribute %s is not allowed in profile %s",	×
127	attributeKey,
128	profile.getIdentifier()));	×
129	}
130
131	validateValuesForKey(pidRecord, attributeKey, type);	1✔
132	}	1✔
133	LOG.debug("successfully validated {}", profile.getIdentifier());	1✔
134	}	1✔
135
136	/**
137	* Validates all values of an attribute against a given type definition.
138	*
139	* @param pidRecord the record containing the attribute and value.
140	* @param attributeKey the attribute to check the values for.
141	* @param type the type definition to check against.
142	* @throws RecordValidationException on error.
143	*/
144	private void validateValuesForKey(PIDRecord pidRecord, String attributeKey, TypeDefinition type)
145	throws RecordValidationException {
146	String[] values = pidRecord.getPropertyValues(attributeKey);	1✔
147	for (String value : values) {	1✔
148	if (value == null) {	1✔
149	LOG.error("'null' record value found for key {}.", attributeKey);	×
150	throw new RecordValidationException(	×
151	pidRecord,
152	String.format("Validation of value %s against type %s failed.",	×
153	value,
154	type.getIdentifier()));	×
155	}
156
157	if (!type.validate(value)) {	1✔
158	LOG.error("Validation of value {} against type {} failed.", value, type.getIdentifier());	×
159	throw new RecordValidationException(	×
160	pidRecord,
161	String.format("Validation of value %s against type %s failed.",	×
162	value,
163	type.getIdentifier()));	×
164	}
165	}
166	}	1✔
167	}

kit-data-manager / pit-service / #356

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous