#447

Committed 22 Jan 2025 04:06PM UTC coverage: 75.468% (+3.1%) from 72.4%

Build # #447

Build Type

Pull #218

github

Committed by

web-flow

Commit Message

Merge e4aa487af into 459f0c036

Pull Request Pull Request #218: Type-Api support and validation speedup

Run Details

257 of 322 new or added lines in 18 files covered. (79.81%)

3 existing lines in 2 files now uncovered.

886 of 1174 relevant lines covered (75.47%)

0.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.63

/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java

package edu.kit.datamanager.pit.typeregistry.impl;

import com.fasterxml.jackson.databind.JsonNode;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.Caffeine;
import edu.kit.datamanager.pit.Application;
import edu.kit.datamanager.pit.common.ExternalServiceException;
import edu.kit.datamanager.pit.common.InvalidConfigException;
import edu.kit.datamanager.pit.common.TypeNotFoundException;
import edu.kit.datamanager.pit.configuration.ApplicationProperties;
import edu.kit.datamanager.pit.domain.ImmutableList;
import edu.kit.datamanager.pit.typeregistry.AttributeInfo;
import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
import edu.kit.datamanager.pit.typeregistry.RegisteredProfile;
import edu.kit.datamanager.pit.typeregistry.RegisteredProfileAttribute;
import edu.kit.datamanager.pit.typeregistry.schema.SchemaInfo;
import edu.kit.datamanager.pit.typeregistry.schema.SchemaSetGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.client.RestClient;

import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.StreamSupport;

public class TypeApi implements ITypeRegistry {

    private static final Logger LOG = LoggerFactory.getLogger(TypeApi.class);

    protected final URL baseUrl;
    protected final RestClient http;
    protected final AsyncLoadingCache<String, RegisteredProfile> profileCache;
    protected final AsyncLoadingCache<String, AttributeInfo> attributeCache;

    protected final SchemaSetGenerator schemaSetGenerator;

    public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGenerator) {
        this.schemaSetGenerator = schemaSetGenerator;
        this.baseUrl = properties.getTypeRegistryUri();
        String baseUri;
        try {
            baseUri = baseUrl.toURI().resolve("v1/types/").toString();
        } catch (URISyntaxException e) {
            throw new InvalidConfigException("Type-Api base url not valid: " + baseUrl);
        }
        this.http = RestClient.builder().baseUrl(baseUri).build();

        int maximumSize = properties.getCacheMaxEntries();
        long expireAfterWrite = properties.getCacheExpireAfterWriteLifetime();

        this.profileCache = Caffeine.newBuilder()
                .maximumSize(maximumSize)
                .executor(Application.newExecutor())
                .refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))
                .expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)
                .removalListener((key, value, cause) ->
                        LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)
                )
                .buildAsync(maybeProfilePid -> {
                    LOG.trace("Loading profile {} to cache.", maybeProfilePid);
                    return this.queryProfile(maybeProfilePid);
                });

        this.attributeCache = Caffeine.newBuilder()
                .maximumSize(maximumSize)
                .executor(Application.newExecutor())
                .refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))
                .expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)
                .removalListener((key, value, cause) ->
                    LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)
                )
                .buildAsync(attributePid -> {
                    LOG.trace("Loading attribute {} to cache.", attributePid);
                    return this.queryAttribute(attributePid);
                });
    }

    protected AttributeInfo queryAttribute(String attributePid) {
        return http.get()
                .uri(uriBuilder -> uriBuilder
                        .path(attributePid)
                        .build())
                .exchange((clientRequest, clientResponse) -> {
                    if (clientResponse.getStatusCode().is2xxSuccessful()) {
                        try (InputStream inputStream = clientResponse.getBody()) {
                            String body = new String(inputStream.readAllBytes());
                            return extractAttributeInformation(attributePid, Application.jsonObjectMapper().readTree(body));
                        } catch (IOException e) {
                            throw new TypeNotFoundException(attributePid);
                        }
                    } else {
                        throw new TypeNotFoundException(attributePid);
                    }
                });
    }

    protected AttributeInfo extractAttributeInformation(String attributePid, JsonNode jsonNode) {
        String typeName = jsonNode.path("type").asText();
        String name = jsonNode.path("name").asText();
        Set<SchemaInfo> schemas = this.querySchemas(attributePid);
        return new AttributeInfo(attributePid, name, typeName, schemas);
    }

    protected Set<SchemaInfo> querySchemas(String maybeSchemaPid) throws TypeNotFoundException, ExternalServiceException {
        return schemaSetGenerator.generateFor(maybeSchemaPid).join();
    }

    protected RegisteredProfile queryProfile(String maybeProfilePid) throws TypeNotFoundException, ExternalServiceException {
        return http.get()
                .uri(uriBuilder -> uriBuilder
                        .path(maybeProfilePid)
                        .build())
                .exchange((clientRequest, clientResponse) -> {
                    if (clientResponse.getStatusCode().is2xxSuccessful()) {
                        InputStream inputStream = clientResponse.getBody();
                        String body = new String(inputStream.readAllBytes());
                        inputStream.close();
                        return extractProfileInformation(maybeProfilePid, Application.jsonObjectMapper().readTree(body));
                    } else {
                        throw new TypeNotFoundException(maybeProfilePid);
                    }
                });
    }

    protected RegisteredProfile extractProfileInformation(String profilePid, JsonNode typeApiResponse)
            throws TypeNotFoundException, ExternalServiceException {

        List<RegisteredProfileAttribute> attributes = new ArrayList<>();
        typeApiResponse.path("content").path("properties").forEach(item -> {

            String attributePid = Optional.ofNullable(item.path("pid").asText(null))
                    .or(() -> Optional.ofNullable(item.path("identifier").asText(null)))
                    .or(() -> Optional.ofNullable(item.path("id").asText()))
                    .orElse("");

            JsonNode representations = item.path("representationsAndSemantics").path(0);

            JsonNode obligationNode = representations.path("obligation");
            boolean attributeMandatory = obligationNode.isBoolean() ? obligationNode.asBoolean()
                    : List.of("mandatory", "yes", "true").contains(obligationNode.asText().trim().toLowerCase());

            JsonNode repeatableNode = representations.path("repeatable");
            boolean attributeRepeatable = repeatableNode.isBoolean() ? repeatableNode.asBoolean()
                    : List.of("yes", "true", "repeatable").contains(repeatableNode.asText().trim().toLowerCase());

            RegisteredProfileAttribute attribute = new RegisteredProfileAttribute(
                    attributePid,
                    attributeMandatory,
                    attributeRepeatable);

            if (obligationNode.isNull() || repeatableNode.isNull() || attributePid.trim().isEmpty()) {
                throw new ExternalServiceException(baseUrl.toString(), "Malformed attribute in profile (%s): " + attribute);
            }
            attributes.add(attribute);

        });

        boolean additionalAttributesDtrTestStyle = StreamSupport.stream(typeApiResponse
                .path("content")
                .path("representationsAndSemantics")
                .spliterator(),
                true)
                .filter(JsonNode::isObject)
                .filter(node -> node.path("expression").asText("").equals("Format"))
                .map(node -> node.path("subSchemaRelation").asText("").equals("allowAdditionalProperties"))
                .findFirst()
                .orElse(true);
        boolean additionalAttributesEoscStyle = typeApiResponse
                .path("content")
                .path("addProps")
                .asBoolean(true);
        // As the default is true, we assume that additional attributes are disallowed if one indicator is false:
        boolean profileDefinitionAllowsAdditionalAttributes = additionalAttributesDtrTestStyle && additionalAttributesEoscStyle;

        return new RegisteredProfile(profilePid, profileDefinitionAllowsAdditionalAttributes, new ImmutableList<>(attributes));
    }

    @Override
    public CompletableFuture<AttributeInfo> queryAttributeInfo(String attributePid) {
        return this.attributeCache.get(attributePid);
    }

    @Override
    public CompletableFuture<RegisteredProfile> queryAsProfile(String profilePid) {
        return this.profileCache.get(profilePid);
    }

    @Override
    public String getRegistryIdentifier() {
        return baseUrl.toString();
    }
}

1	package edu.kit.datamanager.pit.typeregistry.impl;
2
3	import com.fasterxml.jackson.databind.JsonNode;
4	import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
5	import com.github.benmanes.caffeine.cache.Caffeine;
6	import edu.kit.datamanager.pit.Application;
7	import edu.kit.datamanager.pit.common.ExternalServiceException;
8	import edu.kit.datamanager.pit.common.InvalidConfigException;
9	import edu.kit.datamanager.pit.common.TypeNotFoundException;
10	import edu.kit.datamanager.pit.configuration.ApplicationProperties;
11	import edu.kit.datamanager.pit.domain.ImmutableList;
12	import edu.kit.datamanager.pit.typeregistry.AttributeInfo;
13	import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
14	import edu.kit.datamanager.pit.typeregistry.RegisteredProfile;
15	import edu.kit.datamanager.pit.typeregistry.RegisteredProfileAttribute;
16	import edu.kit.datamanager.pit.typeregistry.schema.SchemaInfo;
17	import edu.kit.datamanager.pit.typeregistry.schema.SchemaSetGenerator;
18	import org.slf4j.Logger;
19	import org.slf4j.LoggerFactory;
20	import org.springframework.web.client.RestClient;
21
22	import java.io.IOException;
23	import java.io.InputStream;
24	import java.net.URISyntaxException;
25	import java.net.URL;
26	import java.time.Duration;
27	import java.util.ArrayList;
28	import java.util.List;
29	import java.util.Optional;
30	import java.util.Set;
31	import java.util.concurrent.CompletableFuture;
32	import java.util.concurrent.TimeUnit;
33	import java.util.stream.StreamSupport;
34
35	public class TypeApi implements ITypeRegistry {
36
37	private static final Logger LOG = LoggerFactory.getLogger(TypeApi.class);	1✔
38
39	protected final URL baseUrl;
40	protected final RestClient http;
41	protected final AsyncLoadingCache<String, RegisteredProfile> profileCache;
42	protected final AsyncLoadingCache<String, AttributeInfo> attributeCache;
43
44	protected final SchemaSetGenerator schemaSetGenerator;
45
46	public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGenerator) {	1✔
47	this.schemaSetGenerator = schemaSetGenerator;	1✔
48	this.baseUrl = properties.getTypeRegistryUri();	1✔
49	String baseUri;
50	try {
51	baseUri = baseUrl.toURI().resolve("v1/types/").toString();	1✔
NEW 52	} catch (URISyntaxException e) {	×
NEW 53	throw new InvalidConfigException("Type-Api base url not valid: " + baseUrl);	×
54	}	1✔
55	this.http = RestClient.builder().baseUrl(baseUri).build();	1✔
56
57	int maximumSize = properties.getCacheMaxEntries();	1✔
58	long expireAfterWrite = properties.getCacheExpireAfterWriteLifetime();	1✔
59
60	this.profileCache = Caffeine.newBuilder()	1✔
61	.maximumSize(maximumSize)	1✔
62	.executor(Application.newExecutor())	1✔
63	.refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))	1✔
64	.expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)	1✔
65	.removalListener((key, value, cause) ->	1✔
NEW 66	LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)	×
67	)
68	.buildAsync(maybeProfilePid -> {	1✔
69	LOG.trace("Loading profile {} to cache.", maybeProfilePid);	1✔
70	return this.queryProfile(maybeProfilePid);	1✔
71	});
72
73	this.attributeCache = Caffeine.newBuilder()	1✔
74	.maximumSize(maximumSize)	1✔
75	.executor(Application.newExecutor())	1✔
76	.refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))	1✔
77	.expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)	1✔
78	.removalListener((key, value, cause) ->	1✔
NEW 79	LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)	×
80	)
81	.buildAsync(attributePid -> {	1✔
82	LOG.trace("Loading attribute {} to cache.", attributePid);	1✔
83	return this.queryAttribute(attributePid);	1✔
84	});
85	}	1✔
86
87	protected AttributeInfo queryAttribute(String attributePid) {
88	return http.get()	1✔
89	.uri(uriBuilder -> uriBuilder	1✔
90	.path(attributePid)	1✔
91	.build())	1✔
92	.exchange((clientRequest, clientResponse) -> {	1✔
93	if (clientResponse.getStatusCode().is2xxSuccessful()) {	1✔
94	try (InputStream inputStream = clientResponse.getBody()) {	1✔
95	String body = new String(inputStream.readAllBytes());	1✔
96	return extractAttributeInformation(attributePid, Application.jsonObjectMapper().readTree(body));	1✔
NEW 97	} catch (IOException e) {	×
NEW 98	throw new TypeNotFoundException(attributePid);	×
99	}
100	} else {
101	throw new TypeNotFoundException(attributePid);	1✔
102	}
103	});
104	}
105
106	protected AttributeInfo extractAttributeInformation(String attributePid, JsonNode jsonNode) {
107	String typeName = jsonNode.path("type").asText();	1✔
108	String name = jsonNode.path("name").asText();	1✔
109	Set<SchemaInfo> schemas = this.querySchemas(attributePid);	1✔
110	return new AttributeInfo(attributePid, name, typeName, schemas);	1✔
111	}
112
113	protected Set<SchemaInfo> querySchemas(String maybeSchemaPid) throws TypeNotFoundException, ExternalServiceException {
114	return schemaSetGenerator.generateFor(maybeSchemaPid).join();	1✔
115	}
116
117	protected RegisteredProfile queryProfile(String maybeProfilePid) throws TypeNotFoundException, ExternalServiceException {
118	return http.get()	1✔
119	.uri(uriBuilder -> uriBuilder	1✔
120	.path(maybeProfilePid)	1✔
121	.build())	1✔
122	.exchange((clientRequest, clientResponse) -> {	1✔
123	if (clientResponse.getStatusCode().is2xxSuccessful()) {	1✔
124	InputStream inputStream = clientResponse.getBody();	1✔
125	String body = new String(inputStream.readAllBytes());	1✔
126	inputStream.close();	1✔
127	return extractProfileInformation(maybeProfilePid, Application.jsonObjectMapper().readTree(body));	1✔
128	} else {
NEW 129	throw new TypeNotFoundException(maybeProfilePid);	×
130	}
131	});
132	}
133
134	protected RegisteredProfile extractProfileInformation(String profilePid, JsonNode typeApiResponse)
135	throws TypeNotFoundException, ExternalServiceException {
136
137	List<RegisteredProfileAttribute> attributes = new ArrayList<>();	1✔
138	typeApiResponse.path("content").path("properties").forEach(item -> {	1✔
139
140	String attributePid = Optional.ofNullable(item.path("pid").asText(null))	1✔
141	.or(() -> Optional.ofNullable(item.path("identifier").asText(null)))	1✔
142	.or(() -> Optional.ofNullable(item.path("id").asText()))	1✔
143	.orElse("");	1✔
144
145	JsonNode representations = item.path("representationsAndSemantics").path(0);	1✔
146
147	JsonNode obligationNode = representations.path("obligation");	1✔
148	boolean attributeMandatory = obligationNode.isBoolean() ? obligationNode.asBoolean()	1✔
149	: List.of("mandatory", "yes", "true").contains(obligationNode.asText().trim().toLowerCase());	1✔
150
151	JsonNode repeatableNode = representations.path("repeatable");	1✔
152	boolean attributeRepeatable = repeatableNode.isBoolean() ? repeatableNode.asBoolean()	1✔
153	: List.of("yes", "true", "repeatable").contains(repeatableNode.asText().trim().toLowerCase());	1✔
154
155	RegisteredProfileAttribute attribute = new RegisteredProfileAttribute(	1✔
156	attributePid,
157	attributeMandatory,
158	attributeRepeatable);
159
160	if (obligationNode.isNull() \|\| repeatableNode.isNull() \|\| attributePid.trim().isEmpty()) {	1✔
NEW 161	throw new ExternalServiceException(baseUrl.toString(), "Malformed attribute in profile (%s): " + attribute);	×
162	}
163	attributes.add(attribute);	1✔
164
165	});	1✔
166
167	boolean additionalAttributesDtrTestStyle = StreamSupport.stream(typeApiResponse	1✔
168	.path("content")	1✔
169	.path("representationsAndSemantics")	1✔
170	.spliterator(),	1✔
171	true)
172	.filter(JsonNode::isObject)	1✔
173	.filter(node -> node.path("expression").asText("").equals("Format"))	1✔
174	.map(node -> node.path("subSchemaRelation").asText("").equals("allowAdditionalProperties"))	1✔
175	.findFirst()	1✔
176	.orElse(true);	1✔
177	boolean additionalAttributesEoscStyle = typeApiResponse	1✔
178	.path("content")	1✔
179	.path("addProps")	1✔
180	.asBoolean(true);	1✔
181	// As the default is true, we assume that additional attributes are disallowed if one indicator is false:
182	boolean profileDefinitionAllowsAdditionalAttributes = additionalAttributesDtrTestStyle && additionalAttributesEoscStyle;	1✔
183
184	return new RegisteredProfile(profilePid, profileDefinitionAllowsAdditionalAttributes, new ImmutableList<>(attributes));	1✔
185	}
186
187	@Override
188	public CompletableFuture<AttributeInfo> queryAttributeInfo(String attributePid) {
189	return this.attributeCache.get(attributePid);	1✔
190	}
191
192	@Override
193	public CompletableFuture<RegisteredProfile> queryAsProfile(String profilePid) {
194	return this.profileCache.get(profilePid);	1✔
195	}
196
197	@Override
198	public String getRegistryIdentifier() {
NEW 199	return baseUrl.toString();	×
200	}
201	}

kit-data-manager / pit-service / #447

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous