#358

Committed 29 Aug 2024 02:28PM UTC coverage: 72.074% (-0.5%) from 72.606%

Build # #358

Build Type

Pull #218

github

Committed by

web-flow

Commit Message

Merge a38cdeee2 into 86457be39

Pull Request Pull Request #218: Validation speedup experiments

Run Details

67 of 95 new or added lines in 4 files covered. (70.53%)

4 existing lines in 3 files now uncovered.

862 of 1196 relevant lines covered (72.07%)

0.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

57.58

/src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java

package edu.kit.datamanager.pit.pitservice.impl;

import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import edu.kit.datamanager.pit.common.ExternalServiceException;
import edu.kit.datamanager.pit.common.RecordValidationException;
import edu.kit.datamanager.pit.configuration.ApplicationProperties;
import edu.kit.datamanager.pit.domain.PIDRecord;
import edu.kit.datamanager.pit.domain.TypeDefinition;
import edu.kit.datamanager.pit.pitservice.IValidationStrategy;
import edu.kit.datamanager.pit.util.TypeValidationUtils;

import java.util.Arrays;
import java.util.List;
import java.util.concurrent.*;
import java.util.stream.Collectors;

import org.apache.commons.lang3.stream.Streams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

/**
 * Validates a PID record using embedded profile(s).
 * 
 * - checks if all mandatory attributes are present
 * - validates all available attributes
 * - fails if an attribute is not defined within the profile
 */
public class EmbeddedStrictValidatorStrategy implements IValidationStrategy {

    private static final Logger LOG = LoggerFactory.getLogger(EmbeddedStrictValidatorStrategy.class);
    protected static final Executor EXECUTOR = Executors.newWorkStealingPool();

    @Autowired
    public AsyncLoadingCache<String, TypeDefinition> typeLoader;

    @Autowired
    ApplicationProperties applicationProps;

    @Override
    public void validate(PIDRecord pidRecord) throws RecordValidationException, ExternalServiceException {
        String profileKey = applicationProps.getProfileKey();
        if (!pidRecord.hasProperty(profileKey)) {
            throw new RecordValidationException(
                    pidRecord,
                    "Profile attribute not found. Expected key: " + profileKey);
        }

        String[] profilePIDs = pidRecord.getPropertyValues(profileKey);
        boolean hasProfile = profilePIDs.length > 0;
        if (!hasProfile) {
            throw new RecordValidationException(
                    pidRecord,
                    "Profile attribute " + profileKey + " has no values.");
        }

        List<CompletableFuture<?>> futures = Streams.stream(Arrays.stream(profilePIDs))
                .map(profilePID -> {
                    try {
                        return this.typeLoader.get(profilePID)
                                .thenAcceptAsync(profileDefinition -> {
                                    if (profileDefinition == null) {
                                        LOG.error("No type definition found for identifier {}.", profilePID);
                                        throw new RecordValidationException(
                                                pidRecord,
                                                String.format("No type found for identifier %s.", profilePID));
                                    }
                                    this.strictProfileValidation(pidRecord, profileDefinition);
                                }, EXECUTOR);
                    } catch (RuntimeException e) {
                        LOG.error("Could not resolve identifier {}.", profilePID);
                        throw new ExternalServiceException(
                                applicationProps.getTypeRegistryUri().toString());
                    }
                })
                .collect(Collectors.toList());
        try {
            CompletableFuture.allOf(futures.toArray(new CompletableFuture<?>[0])).join();
        } catch (CompletionException e) {
            throwRecordValidationExceptionCause(e);
            throw new ExternalServiceException(
                    applicationProps.getTypeRegistryUri().toString());
        } catch (CancellationException e) {
            throwRecordValidationExceptionCause(e);
            throw new RecordValidationException(
                    pidRecord,
                    String.format("Validation task was cancelled for %s. Please report.", pidRecord.getPid()));
        }
    }

    private static void throwRecordValidationExceptionCause(Throwable e) {
        if (e.getCause() instanceof RecordValidationException rve) {
            throw rve;
        }
    }

    /**
     * Exceptions indicate failure. No Exceptions mean success.
     * 
     * @param pidRecord the PID record to validate.
     * @param profile   the profile to validate against.
     * @throws RecordValidationException with error message on validation errors.
     */
    private void strictProfileValidation(PIDRecord pidRecord, TypeDefinition profile) throws RecordValidationException {
        // if (profile.hasSchema()) {
        // TODO issue https://github.com/kit-data-manager/pit-service/issues/104
        // validate using schema and you are done (strict validation)
        // String jsonRecord = ""; // TODO format depends on schema source
        // return profile.validate(jsonRecord);
        // }

        LOG.trace("Validating PID record against profile {}.", profile.getIdentifier());

        TypeValidationUtils.checkMandatoryAttributes(pidRecord, profile);

        for (String attributeKey : pidRecord.getPropertyIdentifiers()) {
            LOG.trace("Checking PID record key {}.", attributeKey);

            TypeDefinition type = profile.getSubTypes().get(attributeKey);
            if (type == null) {
                LOG.error("No sub-type found for key {}.", attributeKey);
                // TODO try to resolve it (for later when we support "allow additional
                // attributes")
                // if profile.allowsAdditionalAttributes() {...} else
                throw new RecordValidationException(
                        pidRecord,
                        String.format("Attribute %s is not allowed in profile %s",
                                attributeKey,
                                profile.getIdentifier()));
            }

            validateValuesForKey(pidRecord, attributeKey, type);
        }
        LOG.debug("successfully validated {}", profile.getIdentifier());
    }

    /**
     * Validates all values of an attribute against a given type definition.
     * 
     * @param pidRecord the record containing the attribute and value.
     * @param attributeKey the attribute to check the values for.
     * @param type the type definition to check against.
     * @throws RecordValidationException on error.
     */
    private void validateValuesForKey(PIDRecord pidRecord, String attributeKey, TypeDefinition type)
            throws RecordValidationException {
        String[] values = pidRecord.getPropertyValues(attributeKey);
        for (String value : values) {
            if (value == null) {
                LOG.error("'null' record value found for key {}.", attributeKey);
                throw new RecordValidationException(
                        pidRecord,
                        String.format("Validation of value %s against type %s failed.",
                                value,
                                type.getIdentifier()));
            }

            if (!type.validate(value)) {
                LOG.error("Validation of value {} against type {} failed.", value, type.getIdentifier());
                throw new RecordValidationException(
                        pidRecord,
                        String.format("Validation of value %s against type %s failed.",
                                value,
                                type.getIdentifier()));
            }
        }
    }
}

1	package edu.kit.datamanager.pit.pitservice.impl;
2
3	import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
4	import edu.kit.datamanager.pit.common.ExternalServiceException;
5	import edu.kit.datamanager.pit.common.RecordValidationException;
6	import edu.kit.datamanager.pit.configuration.ApplicationProperties;
7	import edu.kit.datamanager.pit.domain.PIDRecord;
8	import edu.kit.datamanager.pit.domain.TypeDefinition;
9	import edu.kit.datamanager.pit.pitservice.IValidationStrategy;
10	import edu.kit.datamanager.pit.util.TypeValidationUtils;
11
12	import java.util.Arrays;
13	import java.util.List;
14	import java.util.concurrent.*;
15	import java.util.stream.Collectors;
16
17	import org.apache.commons.lang3.stream.Streams;
18	import org.slf4j.Logger;
19	import org.slf4j.LoggerFactory;
20	import org.springframework.beans.factory.annotation.Autowired;
21
22	/**
23	* Validates a PID record using embedded profile(s).
24	*
25	* - checks if all mandatory attributes are present
26	* - validates all available attributes
27	* - fails if an attribute is not defined within the profile
28	*/
29	public class EmbeddedStrictValidatorStrategy implements IValidationStrategy {	1✔
30
31	private static final Logger LOG = LoggerFactory.getLogger(EmbeddedStrictValidatorStrategy.class);	1✔
32	protected static final Executor EXECUTOR = Executors.newWorkStealingPool();	1✔
33
34	@Autowired
35	public AsyncLoadingCache<String, TypeDefinition> typeLoader;
36
37	@Autowired
38	ApplicationProperties applicationProps;
39
40	@Override
41	public void validate(PIDRecord pidRecord) throws RecordValidationException, ExternalServiceException {
42	String profileKey = applicationProps.getProfileKey();	1✔
43	if (!pidRecord.hasProperty(profileKey)) {	1✔
44	throw new RecordValidationException(	1✔
45	pidRecord,
46	"Profile attribute not found. Expected key: " + profileKey);
47	}
48
49	String[] profilePIDs = pidRecord.getPropertyValues(profileKey);	1✔
50	boolean hasProfile = profilePIDs.length > 0;	1✔
51	if (!hasProfile) {	1✔
52	throw new RecordValidationException(	×
53	pidRecord,
54	"Profile attribute " + profileKey + " has no values.");
55	}
56
57	List<CompletableFuture<?>> futures = Streams.stream(Arrays.stream(profilePIDs))	1✔
58	.map(profilePID -> {	1✔
59	try {
60	return this.typeLoader.get(profilePID)	1✔
61	.thenAcceptAsync(profileDefinition -> {	1✔
62	if (profileDefinition == null) {	1✔
NEW 63	LOG.error("No type definition found for identifier {}.", profilePID);	×
NEW 64	throw new RecordValidationException(	×
65	pidRecord,
NEW 66	String.format("No type found for identifier %s.", profilePID));	×
67	}
68	this.strictProfileValidation(pidRecord, profileDefinition);	1✔
69	}, EXECUTOR);	1✔
NEW 70	} catch (RuntimeException e) {	×
NEW 71	LOG.error("Could not resolve identifier {}.", profilePID);	×
NEW 72	throw new ExternalServiceException(	×
NEW 73	applicationProps.getTypeRegistryUri().toString());	×
74	}
75	})
76	.collect(Collectors.toList());	1✔
77	try {
78	CompletableFuture.allOf(futures.toArray(new CompletableFuture<?>[0])).join();	1✔
79	} catch (CompletionException e) {	1✔
NEW 80	throwRecordValidationExceptionCause(e);	×
NEW 81	throw new ExternalServiceException(	×
NEW 82	applicationProps.getTypeRegistryUri().toString());	×
NEW 83	} catch (CancellationException e) {	×
NEW 84	throwRecordValidationExceptionCause(e);	×
NEW 85	throw new RecordValidationException(	×
86	pidRecord,
NEW 87	String.format("Validation task was cancelled for %s. Please report.", pidRecord.getPid()));	×
88	}	1✔
89	}	1✔
90
91	private static void throwRecordValidationExceptionCause(Throwable e) {
92	if (e.getCause() instanceof RecordValidationException rve) {	1✔
93	throw rve;	1✔
94	}
UNCOV 95	}	×
96
97	/**
98	* Exceptions indicate failure. No Exceptions mean success.
99	*
100	* @param pidRecord the PID record to validate.
101	* @param profile the profile to validate against.
102	* @throws RecordValidationException with error message on validation errors.
103	*/
104	private void strictProfileValidation(PIDRecord pidRecord, TypeDefinition profile) throws RecordValidationException {
105	// if (profile.hasSchema()) {
106	// TODO issue https://github.com/kit-data-manager/pit-service/issues/104
107	// validate using schema and you are done (strict validation)
108	// String jsonRecord = ""; // TODO format depends on schema source
109	// return profile.validate(jsonRecord);
110	// }
111
112	LOG.trace("Validating PID record against profile {}.", profile.getIdentifier());	1✔
113
114	TypeValidationUtils.checkMandatoryAttributes(pidRecord, profile);	1✔
115
116	for (String attributeKey : pidRecord.getPropertyIdentifiers()) {	1✔
117	LOG.trace("Checking PID record key {}.", attributeKey);	1✔
118
119	TypeDefinition type = profile.getSubTypes().get(attributeKey);	1✔
120	if (type == null) {	1✔
121	LOG.error("No sub-type found for key {}.", attributeKey);	×
122	// TODO try to resolve it (for later when we support "allow additional
123	// attributes")
124	// if profile.allowsAdditionalAttributes() {...} else
125	throw new RecordValidationException(	×
126	pidRecord,
127	String.format("Attribute %s is not allowed in profile %s",	×
128	attributeKey,
129	profile.getIdentifier()));	×
130	}
131
132	validateValuesForKey(pidRecord, attributeKey, type);	1✔
133	}	1✔
134	LOG.debug("successfully validated {}", profile.getIdentifier());	1✔
135	}	1✔
136
137	/**
138	* Validates all values of an attribute against a given type definition.
139	*
140	* @param pidRecord the record containing the attribute and value.
141	* @param attributeKey the attribute to check the values for.
142	* @param type the type definition to check against.
143	* @throws RecordValidationException on error.
144	*/
145	private void validateValuesForKey(PIDRecord pidRecord, String attributeKey, TypeDefinition type)
146	throws RecordValidationException {
147	String[] values = pidRecord.getPropertyValues(attributeKey);	1✔
148	for (String value : values) {	1✔
149	if (value == null) {	1✔
150	LOG.error("'null' record value found for key {}.", attributeKey);	×
151	throw new RecordValidationException(	×
152	pidRecord,
153	String.format("Validation of value %s against type %s failed.",	×
154	value,
155	type.getIdentifier()));	×
156	}
157
158	if (!type.validate(value)) {	1✔
159	LOG.error("Validation of value {} against type {} failed.", value, type.getIdentifier());	×
160	throw new RecordValidationException(	×
161	pidRecord,
162	String.format("Validation of value %s against type %s failed.",	×
163	value,
164	type.getIdentifier()));	×
165	}
166	}
167	}	1✔
168	}

kit-data-manager / pit-service / #358

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous