#359

Committed 29 Aug 2024 03:02PM UTC coverage: 72.074% (-0.5%) from 72.606%

Build # #359

Build Type

Pull #218

github

Committed by

web-flow

Commit Message

Merge b5fba649f into 86457be39

Pull Request Pull Request #218: Validation speedup experiments

Run Details

67 of 95 new or added lines in 4 files covered. (70.53%)

4 existing lines in 3 files now uncovered.

862 of 1196 relevant lines covered (72.07%)

0.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.59

/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeRegistry.java

package edu.kit.datamanager.pit.typeregistry.impl;

import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.util.Map;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import edu.kit.datamanager.pit.configuration.ApplicationProperties;
import edu.kit.datamanager.pit.domain.ProvenanceInformation;
import edu.kit.datamanager.pit.domain.TypeDefinition;
import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
import java.net.URISyntaxException;
import java.time.Instant;
import java.time.format.DateTimeParseException;
import java.util.concurrent.*;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import org.apache.commons.lang3.stream.Streams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;

/**
 * Accessor for a specific instance of a TypeRegistry. The TypeRegistry is
 * uniquely identified by a baseUrl and an identifierPrefix which all types of
 * this particular registry are using. The prefix also allows to determine,
 * whether a given PID might be a type or property registered at this
 * TypeRegistry.
 */
public class TypeRegistry implements ITypeRegistry {

    private static final Logger LOG = LoggerFactory.getLogger(TypeRegistry.class);
    protected static final Executor EXECUTOR = Executors.newWorkStealingPool(10);

    @Autowired
    public AsyncLoadingCache<String, TypeDefinition> typeCache;
    @Autowired
    private ApplicationProperties applicationProperties;

    protected RestTemplate restTemplate = new RestTemplate();

    @Override
    public TypeDefinition queryTypeDefinition(String typeIdentifier) throws IOException, URISyntaxException {
        LOG.trace("Performing queryTypeDefinition({}).", typeIdentifier);
        String[] segments = typeIdentifier.split("/");
        UriComponentsBuilder uriBuilder = UriComponentsBuilder
                .fromUri(
                        applicationProperties
                                .getHandleBaseUri()
                                .toURI())
                .pathSegment(segments);
        LOG.trace("Querying for type definition at URI {}.", uriBuilder);
        ResponseEntity<String> response = restTemplate.exchange(uriBuilder.build().toUri(), HttpMethod.GET,
                HttpEntity.EMPTY, String.class);
        ObjectMapper mapper = new ObjectMapper();
        JsonNode rootNode = mapper.readTree(response.getBody());
        LOG.trace("Constructing type definition from response.");
        return constructTypeDefinition(rootNode);
    }

    /**
     * Helper method to construct a type definition from a JSON response
     * received from the TypeRegistry.
     *
     * @param registryRepresentation The type definition.
     * @return The TypeDefinition as object.
     */
    private TypeDefinition constructTypeDefinition(JsonNode registryRepresentation)
            throws JsonProcessingException, IOException, URISyntaxException {
        // TODO We are doing things too complicated here. Deserialization should be
        // easy.
        // But before we change the domain model to do so, we need a lot of tests to
        // make sure things work as before after the changes.
        LOG.trace("Performing constructTypeDefinition(<rootNode>).");
        final String identifier = registryRepresentation.path("identifier").asText(null);
        if (identifier == null) {
            LOG.error("No 'identifier' property found in entry: {}", registryRepresentation);
            throw new IOException("No 'identifier' attribute found in type definition.");
        }

        LOG.trace("Checking for 'properties' attribute.");
        Map<String, TypeDefinition> properties = new ConcurrentHashMap<>();
        List<CompletableFuture<?>> propertiesHandling = Streams.stream(StreamSupport.stream(
                        registryRepresentation.path("properties").spliterator(), false))
                .filter(property -> property.hasNonNull("name"))
                .filter(property -> property.hasNonNull("identifier"))
                .map(property -> {
                    final String name = property.path("name").asText();
                    final String pid = property.path("identifier").asText();
                    return typeCache.get(pid).thenAcceptAsync(
                            typeDefinition -> {
                                final JsonNode semantics = property.path("representationsAndSemantics").path(0);
                                final String expression = semantics.path("expression").asText(null);
                                typeDefinition.setExpression(expression);
                                final String value = semantics.path("value").asText(null);
                                typeDefinition.setValue(value);
                                final String obligation = semantics.path("obligation").asText("Mandatory");
                                typeDefinition.setOptional("Optional".equalsIgnoreCase(obligation));
                                final String repeatable = semantics.path("repeatable").asText("No");
                                typeDefinition.setRepeatable(!"No".equalsIgnoreCase(repeatable));
                                properties.put(name, typeDefinition);
                            }, EXECUTOR);
                })
                .collect(Collectors.toList());

        TypeDefinition result = new TypeDefinition();
        result.setIdentifier(identifier);
        final String description = registryRepresentation.path("description").asText(null);
        result.setDescription(description);
        final String name = registryRepresentation.path("name").asText(null);
        result.setName(name);
        final String validationSchema = registryRepresentation.path("validationSchema").asText(null);
        result.setSchema(validationSchema);

        if (registryRepresentation.has("provenance")) {
            ProvenanceInformation prov = new ProvenanceInformation();
            JsonNode provNode = registryRepresentation.get("provenance");
            if (provNode.has("creationDate")) {
                String creationDate = provNode.get("creationDate").asText();
                try {
                    prov.setCreationDate(Date.from(Instant.parse(creationDate)));
                } catch (DateTimeParseException ex) {
                    LOG.error("Failed to parse creationDate from value " + creationDate + ".", ex);
                }
            }
            if (provNode.has("lastModificationDate")) {
                String lastModificationDate = provNode.get("lastModificationDate").asText();
                try {
                    prov.setLastModificationDate(Date.from(Instant.parse(lastModificationDate)));
                } catch (DateTimeParseException ex) {
                    LOG.error("Failed to parse lastModificationDate from value " + lastModificationDate + ".", ex);
                }
            }
            for (JsonNode entryKV : provNode.get("contributors")) {
                String identified = null;
                String contributorName = null;
                String details = null;

                if (registryRepresentation.has("identifiedBy")) {
                    identified = entryKV.get("identifiedBy").asText();
                }
                if (registryRepresentation.has("name")) {
                    contributorName = entryKV.get("name").asText();
                }
                if (registryRepresentation.has("details")) {
                    details = entryKV.get("details").asText();
                }
                prov.addContributor(identified, contributorName, details);
            }
            result.setProvenance(prov);
        }

        LOG.trace("Finalizing and returning type definition.");
        CompletableFuture.allOf(propertiesHandling.toArray(new CompletableFuture<?>[0])).join();
        properties.keySet().forEach(pd -> result.addSubType(properties.get(pd)));
        this.typeCache.put(identifier, CompletableFuture.completedFuture(result));
        return result;
    }
}

1	package edu.kit.datamanager.pit.typeregistry.impl;
2
3	import java.io.IOException;
4	import java.util.Date;
5	import java.util.List;
6	import java.util.Map;
7
8	import com.fasterxml.jackson.core.JsonProcessingException;
9	import com.fasterxml.jackson.databind.JsonNode;
10	import com.fasterxml.jackson.databind.ObjectMapper;
11	import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
12	import edu.kit.datamanager.pit.configuration.ApplicationProperties;
13	import edu.kit.datamanager.pit.domain.ProvenanceInformation;
14	import edu.kit.datamanager.pit.domain.TypeDefinition;
15	import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
16	import java.net.URISyntaxException;
17	import java.time.Instant;
18	import java.time.format.DateTimeParseException;
19	import java.util.concurrent.*;
20	import java.util.stream.Collectors;
21	import java.util.stream.StreamSupport;
22
23	import org.apache.commons.lang3.stream.Streams;
24	import org.slf4j.Logger;
25	import org.slf4j.LoggerFactory;
26	import org.springframework.beans.factory.annotation.Autowired;
27	import org.springframework.http.HttpEntity;
28	import org.springframework.http.HttpMethod;
29	import org.springframework.http.ResponseEntity;
30	import org.springframework.web.client.RestTemplate;
31	import org.springframework.web.util.UriComponentsBuilder;
32
33	/**
34	* Accessor for a specific instance of a TypeRegistry. The TypeRegistry is
35	* uniquely identified by a baseUrl and an identifierPrefix which all types of
36	* this particular registry are using. The prefix also allows to determine,
37	* whether a given PID might be a type or property registered at this
38	* TypeRegistry.
39	*/
40	public class TypeRegistry implements ITypeRegistry {	1✔
41
42	private static final Logger LOG = LoggerFactory.getLogger(TypeRegistry.class);	1✔
43	protected static final Executor EXECUTOR = Executors.newWorkStealingPool(10);	1✔
44
45	@Autowired
46	public AsyncLoadingCache<String, TypeDefinition> typeCache;
47	@Autowired
48	private ApplicationProperties applicationProperties;
49
50	protected RestTemplate restTemplate = new RestTemplate();	1✔
51
52	@Override
53	public TypeDefinition queryTypeDefinition(String typeIdentifier) throws IOException, URISyntaxException {
54	LOG.trace("Performing queryTypeDefinition({}).", typeIdentifier);	1✔
55	String[] segments = typeIdentifier.split("/");	1✔
56	UriComponentsBuilder uriBuilder = UriComponentsBuilder	1✔
57	.fromUri(	1✔
58	applicationProperties
59	.getHandleBaseUri()	1✔
60	.toURI())	1✔
61	.pathSegment(segments);	1✔
62	LOG.trace("Querying for type definition at URI {}.", uriBuilder);	1✔
63	ResponseEntity<String> response = restTemplate.exchange(uriBuilder.build().toUri(), HttpMethod.GET,	1✔
64	HttpEntity.EMPTY, String.class);
65	ObjectMapper mapper = new ObjectMapper();	1✔
66	JsonNode rootNode = mapper.readTree(response.getBody());	1✔
67	LOG.trace("Constructing type definition from response.");	1✔
68	return constructTypeDefinition(rootNode);	1✔
69	}
70
71	/**
72	* Helper method to construct a type definition from a JSON response
73	* received from the TypeRegistry.
74	*
75	* @param registryRepresentation The type definition.
76	* @return The TypeDefinition as object.
77	*/
78	private TypeDefinition constructTypeDefinition(JsonNode registryRepresentation)
79	throws JsonProcessingException, IOException, URISyntaxException {
80	// TODO We are doing things too complicated here. Deserialization should be
81	// easy.
82	// But before we change the domain model to do so, we need a lot of tests to
83	// make sure things work as before after the changes.
84	LOG.trace("Performing constructTypeDefinition(<rootNode>).");	1✔
85	final String identifier = registryRepresentation.path("identifier").asText(null);	1✔
86	if (identifier == null) {	1✔
NEW 87	LOG.error("No 'identifier' property found in entry: {}", registryRepresentation);	×
UNCOV 88	throw new IOException("No 'identifier' attribute found in type definition.");	×
89	}
90
91	LOG.trace("Checking for 'properties' attribute.");	1✔
92	Map<String, TypeDefinition> properties = new ConcurrentHashMap<>();	1✔
93	List<CompletableFuture<?>> propertiesHandling = Streams.stream(StreamSupport.stream(	1✔
94	registryRepresentation.path("properties").spliterator(), false))	1✔
95	.filter(property -> property.hasNonNull("name"))	1✔
96	.filter(property -> property.hasNonNull("identifier"))	1✔
97	.map(property -> {	1✔
98	final String name = property.path("name").asText();	1✔
99	final String pid = property.path("identifier").asText();	1✔
100	return typeCache.get(pid).thenAcceptAsync(	1✔
101	typeDefinition -> {
102	final JsonNode semantics = property.path("representationsAndSemantics").path(0);	1✔
103	final String expression = semantics.path("expression").asText(null);	1✔
104	typeDefinition.setExpression(expression);	1✔
105	final String value = semantics.path("value").asText(null);	1✔
106	typeDefinition.setValue(value);	1✔
107	final String obligation = semantics.path("obligation").asText("Mandatory");	1✔
108	typeDefinition.setOptional("Optional".equalsIgnoreCase(obligation));	1✔
109	final String repeatable = semantics.path("repeatable").asText("No");	1✔
110	typeDefinition.setRepeatable(!"No".equalsIgnoreCase(repeatable));	1✔
111	properties.put(name, typeDefinition);	1✔
112	}, EXECUTOR);	1✔
113	})
114	.collect(Collectors.toList());	1✔
115
116	TypeDefinition result = new TypeDefinition();	1✔
117	result.setIdentifier(identifier);	1✔
118	final String description = registryRepresentation.path("description").asText(null);	1✔
119	result.setDescription(description);	1✔
120	final String name = registryRepresentation.path("name").asText(null);	1✔
121	result.setName(name);	1✔
122	final String validationSchema = registryRepresentation.path("validationSchema").asText(null);	1✔
123	result.setSchema(validationSchema);	1✔
124
125	if (registryRepresentation.has("provenance")) {	1✔
126	ProvenanceInformation prov = new ProvenanceInformation();	1✔
127	JsonNode provNode = registryRepresentation.get("provenance");	1✔
128	if (provNode.has("creationDate")) {	1✔
129	String creationDate = provNode.get("creationDate").asText();	1✔
130	try {
131	prov.setCreationDate(Date.from(Instant.parse(creationDate)));	1✔
132	} catch (DateTimeParseException ex) {	×
133	LOG.error("Failed to parse creationDate from value " + creationDate + ".", ex);	×
134	}	1✔
135	}
136	if (provNode.has("lastModificationDate")) {	1✔
137	String lastModificationDate = provNode.get("lastModificationDate").asText();	1✔
138	try {
139	prov.setLastModificationDate(Date.from(Instant.parse(lastModificationDate)));	1✔
140	} catch (DateTimeParseException ex) {	×
141	LOG.error("Failed to parse lastModificationDate from value " + lastModificationDate + ".", ex);	×
142	}	1✔
143	}
144	for (JsonNode entryKV : provNode.get("contributors")) {	1✔
145	String identified = null;	1✔
146	String contributorName = null;	1✔
147	String details = null;	1✔
148
149	if (registryRepresentation.has("identifiedBy")) {	1✔
150	identified = entryKV.get("identifiedBy").asText();	×
151	}
152	if (registryRepresentation.has("name")) {	1✔
153	contributorName = entryKV.get("name").asText();	1✔
154	}
155	if (registryRepresentation.has("details")) {	1✔
156	details = entryKV.get("details").asText();	×
157	}
158	prov.addContributor(identified, contributorName, details);	1✔
159	}	1✔
160	result.setProvenance(prov);	1✔
161	}
162
163	LOG.trace("Finalizing and returning type definition.");	1✔
164	CompletableFuture.allOf(propertiesHandling.toArray(new CompletableFuture<?>[0])).join();	1✔
165	properties.keySet().forEach(pd -> result.addSubType(properties.get(pd)));	1✔
166	this.typeCache.put(identifier, CompletableFuture.completedFuture(result));	1✔
167	return result;	1✔
168	}
169	}

kit-data-manager / pit-service / #359

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous