#451

Committed 23 Jan 2025 02:48PM UTC coverage: 75.997% (+3.6%) from 72.4%

Build # #451

Build Type

Pull #218

github

Committed by

web-flow

Commit Message

Merge a92bd4d02 into 459f0c036

Pull Request Pull Request #218: Type-Api support and validation speedup

Run Details

287 of 352 new or added lines in 18 files covered. (81.53%)

3 existing lines in 2 files now uncovered.

915 of 1204 relevant lines covered (76.0%)

0.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.51

/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java

package edu.kit.datamanager.pit.typeregistry.impl;

import com.fasterxml.jackson.databind.JsonNode;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.Caffeine;
import edu.kit.datamanager.pit.Application;
import edu.kit.datamanager.pit.common.ExternalServiceException;
import edu.kit.datamanager.pit.common.InvalidConfigException;
import edu.kit.datamanager.pit.common.TypeNotFoundException;
import edu.kit.datamanager.pit.configuration.ApplicationProperties;
import edu.kit.datamanager.pit.domain.ImmutableList;
import edu.kit.datamanager.pit.typeregistry.AttributeInfo;
import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
import edu.kit.datamanager.pit.typeregistry.RegisteredProfile;
import edu.kit.datamanager.pit.typeregistry.RegisteredProfileAttribute;
import edu.kit.datamanager.pit.typeregistry.schema.SchemaInfo;
import edu.kit.datamanager.pit.typeregistry.schema.SchemaSetGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.web.client.RestClient;

import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.StreamSupport;

public class TypeApi implements ITypeRegistry {

    private static final Logger LOG = LoggerFactory.getLogger(TypeApi.class);

    protected final URL baseUrl;
    protected final RestClient http;
    protected final AsyncLoadingCache<String, RegisteredProfile> profileCache;
    protected final AsyncLoadingCache<String, AttributeInfo> attributeCache;

    protected final SchemaSetGenerator schemaSetGenerator;

    public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGenerator) {
        this.schemaSetGenerator = schemaSetGenerator;
        this.baseUrl = properties.getTypeRegistryUri();
        String baseUri;
        try {
            baseUri = baseUrl.toURI().resolve("v1/types/").toString();
        } catch (URISyntaxException e) {
            throw new InvalidConfigException("Type-Api base url not valid: " + baseUrl);
        }
        this.http = RestClient.builder()
                .baseUrl(baseUri)
                .requestInterceptor((request, body, execution) -> {
                    long start = System.currentTimeMillis();
                    ClientHttpResponse response = execution.execute(request,  body);
                    long timeSpan = System.currentTimeMillis() - start;
                    boolean isLongRequest = timeSpan > Application.LONG_HTTP_REQUEST_THRESHOLD;
                    if (isLongRequest) {
                        LOG.warn("Long http request to {} ({}ms)", request.getURI(), timeSpan);
                    }
                    return response;
                })
                .build();

        int maximumSize = properties.getCacheMaxEntries();
        long expireAfterWrite = properties.getCacheExpireAfterWriteLifetime();

        this.profileCache = Caffeine.newBuilder()
                .maximumSize(maximumSize)
                .executor(Application.newExecutor())
                .refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))
                .expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)
                .removalListener((key, value, cause) ->
                        LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)
                )
                .buildAsync(maybeProfilePid -> {
                    LOG.trace("Loading profile {} to cache.", maybeProfilePid);
                    return this.queryProfile(maybeProfilePid);
                });

        this.attributeCache = Caffeine.newBuilder()
                .maximumSize(maximumSize)
                .executor(Application.newExecutor())
                .refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))
                .expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)
                .removalListener((key, value, cause) ->
                    LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)
                )
                .buildAsync(attributePid -> {
                    LOG.trace("Loading attribute {} to cache.", attributePid);
                    return this.queryAttribute(attributePid);
                });
    }

    protected AttributeInfo queryAttribute(String attributePid) {
        return http.get()
                .uri(uriBuilder -> uriBuilder
                        .path(attributePid)
                        .build())
                .exchange((clientRequest, clientResponse) -> {
                    if (clientResponse.getStatusCode().is2xxSuccessful()) {
                        try (InputStream inputStream = clientResponse.getBody()) {
                            String body = new String(inputStream.readAllBytes());
                            return extractAttributeInformation(attributePid, Application.jsonObjectMapper().readTree(body));
                        } catch (IOException e) {
                            throw new TypeNotFoundException(attributePid);
                        }
                    } else {
                        throw new TypeNotFoundException(attributePid);
                    }
                });
    }

    protected AttributeInfo extractAttributeInformation(String attributePid, JsonNode jsonNode) {
        String typeName = jsonNode.path("type").asText();
        String name = jsonNode.path("name").asText();
        Set<SchemaInfo> schemas = this.querySchemas(attributePid);
        return new AttributeInfo(attributePid, name, typeName, schemas);
    }

    protected Set<SchemaInfo> querySchemas(String maybeSchemaPid) throws TypeNotFoundException, ExternalServiceException {
        return schemaSetGenerator.generateFor(maybeSchemaPid).join();
    }

    protected RegisteredProfile queryProfile(String maybeProfilePid) throws TypeNotFoundException, ExternalServiceException {
        return http.get()
                .uri(uriBuilder -> uriBuilder
                        .path(maybeProfilePid)
                        .build())
                .exchange((clientRequest, clientResponse) -> {
                    if (clientResponse.getStatusCode().is2xxSuccessful()) {
                        InputStream inputStream = clientResponse.getBody();
                        String body = new String(inputStream.readAllBytes());
                        inputStream.close();
                        return extractProfileInformation(maybeProfilePid, Application.jsonObjectMapper().readTree(body));
                    } else {
                        throw new TypeNotFoundException(maybeProfilePid);
                    }
                });
    }

    protected RegisteredProfile extractProfileInformation(String profilePid, JsonNode typeApiResponse)
            throws TypeNotFoundException, ExternalServiceException {

        List<RegisteredProfileAttribute> attributes = new ArrayList<>();
        typeApiResponse.path("content").path("properties").forEach(item -> {

            String attributePid = Optional.ofNullable(item.path("pid").asText(null))
                    .or(() -> Optional.ofNullable(item.path("identifier").asText(null)))
                    .or(() -> Optional.ofNullable(item.path("id").asText()))
                    .orElse("");

            JsonNode representations = item.path("representationsAndSemantics").path(0);

            JsonNode obligationNode = representations.path("obligation");
            boolean attributeMandatory = obligationNode.isBoolean() ? obligationNode.asBoolean()
                    : List.of("mandatory", "yes", "true").contains(obligationNode.asText().trim().toLowerCase());

            JsonNode repeatableNode = representations.path("repeatable");
            boolean attributeRepeatable = repeatableNode.isBoolean() ? repeatableNode.asBoolean()
                    : List.of("yes", "true", "repeatable").contains(repeatableNode.asText().trim().toLowerCase());

            RegisteredProfileAttribute attribute = new RegisteredProfileAttribute(
                    attributePid,
                    attributeMandatory,
                    attributeRepeatable);

            if (obligationNode.isNull() || repeatableNode.isNull() || attributePid.trim().isEmpty()) {
                throw new ExternalServiceException(baseUrl.toString(), "Malformed attribute in profile (%s): " + attribute);
            }
            attributes.add(attribute);

        });

        boolean additionalAttributesDtrTestStyle = StreamSupport.stream(typeApiResponse
                .path("content")
                .path("representationsAndSemantics")
                .spliterator(),
                true)
                .filter(JsonNode::isObject)
                .filter(node -> node.path("expression").asText("").equals("Format"))
                .map(node -> node.path("subSchemaRelation").asText("").equals("allowAdditionalProperties"))
                .findFirst()
                .orElse(true);
        boolean additionalAttributesEoscStyle = typeApiResponse
                .path("content")
                .path("addProps")
                .asBoolean(true);
        // As the default is true, we assume that additional attributes are disallowed if one indicator is false:
        boolean profileDefinitionAllowsAdditionalAttributes = additionalAttributesDtrTestStyle && additionalAttributesEoscStyle;

        return new RegisteredProfile(profilePid, profileDefinitionAllowsAdditionalAttributes, new ImmutableList<>(attributes));
    }

    @Override
    public CompletableFuture<AttributeInfo> queryAttributeInfo(String attributePid) {
        return this.attributeCache.get(attributePid);
    }

    @Override
    public CompletableFuture<RegisteredProfile> queryAsProfile(String profilePid) {
        return this.profileCache.get(profilePid);
    }

    @Override
    public String getRegistryIdentifier() {
        return baseUrl.toString();
    }
}

1	package edu.kit.datamanager.pit.typeregistry.impl;
2
3	import com.fasterxml.jackson.databind.JsonNode;
4	import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
5	import com.github.benmanes.caffeine.cache.Caffeine;
6	import edu.kit.datamanager.pit.Application;
7	import edu.kit.datamanager.pit.common.ExternalServiceException;
8	import edu.kit.datamanager.pit.common.InvalidConfigException;
9	import edu.kit.datamanager.pit.common.TypeNotFoundException;
10	import edu.kit.datamanager.pit.configuration.ApplicationProperties;
11	import edu.kit.datamanager.pit.domain.ImmutableList;
12	import edu.kit.datamanager.pit.typeregistry.AttributeInfo;
13	import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
14	import edu.kit.datamanager.pit.typeregistry.RegisteredProfile;
15	import edu.kit.datamanager.pit.typeregistry.RegisteredProfileAttribute;
16	import edu.kit.datamanager.pit.typeregistry.schema.SchemaInfo;
17	import edu.kit.datamanager.pit.typeregistry.schema.SchemaSetGenerator;
18	import org.slf4j.Logger;
19	import org.slf4j.LoggerFactory;
20	import org.springframework.http.client.ClientHttpResponse;
21	import org.springframework.web.client.RestClient;
22
23	import java.io.IOException;
24	import java.io.InputStream;
25	import java.net.URISyntaxException;
26	import java.net.URL;
27	import java.time.Duration;
28	import java.util.ArrayList;
29	import java.util.List;
30	import java.util.Optional;
31	import java.util.Set;
32	import java.util.concurrent.CompletableFuture;
33	import java.util.concurrent.TimeUnit;
34	import java.util.stream.StreamSupport;
35
36	public class TypeApi implements ITypeRegistry {
37
38	private static final Logger LOG = LoggerFactory.getLogger(TypeApi.class);	1✔
39
40	protected final URL baseUrl;
41	protected final RestClient http;
42	protected final AsyncLoadingCache<String, RegisteredProfile> profileCache;
43	protected final AsyncLoadingCache<String, AttributeInfo> attributeCache;
44
45	protected final SchemaSetGenerator schemaSetGenerator;
46
47	public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGenerator) {	1✔
48	this.schemaSetGenerator = schemaSetGenerator;	1✔
49	this.baseUrl = properties.getTypeRegistryUri();	1✔
50	String baseUri;
51	try {
52	baseUri = baseUrl.toURI().resolve("v1/types/").toString();	1✔
NEW 53	} catch (URISyntaxException e) {	×
NEW 54	throw new InvalidConfigException("Type-Api base url not valid: " + baseUrl);	×
55	}	1✔
56	this.http = RestClient.builder()	1✔
57	.baseUrl(baseUri)	1✔
58	.requestInterceptor((request, body, execution) -> {	1✔
59	long start = System.currentTimeMillis();	1✔
60	ClientHttpResponse response = execution.execute(request, body);	1✔
61	long timeSpan = System.currentTimeMillis() - start;	1✔
62	boolean isLongRequest = timeSpan > Application.LONG_HTTP_REQUEST_THRESHOLD;	1✔
63	if (isLongRequest) {	1✔
64	LOG.warn("Long http request to {} ({}ms)", request.getURI(), timeSpan);	1✔
65	}
66	return response;	1✔
67	})
68	.build();	1✔
69
70	int maximumSize = properties.getCacheMaxEntries();	1✔
71	long expireAfterWrite = properties.getCacheExpireAfterWriteLifetime();	1✔
72
73	this.profileCache = Caffeine.newBuilder()	1✔
74	.maximumSize(maximumSize)	1✔
75	.executor(Application.newExecutor())	1✔
76	.refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))	1✔
77	.expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)	1✔
78	.removalListener((key, value, cause) ->	1✔
NEW 79	LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)	×
80	)
81	.buildAsync(maybeProfilePid -> {	1✔
82	LOG.trace("Loading profile {} to cache.", maybeProfilePid);	1✔
83	return this.queryProfile(maybeProfilePid);	1✔
84	});
85
86	this.attributeCache = Caffeine.newBuilder()	1✔
87	.maximumSize(maximumSize)	1✔
88	.executor(Application.newExecutor())	1✔
89	.refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))	1✔
90	.expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)	1✔
91	.removalListener((key, value, cause) ->	1✔
NEW 92	LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)	×
93	)
94	.buildAsync(attributePid -> {	1✔
95	LOG.trace("Loading attribute {} to cache.", attributePid);	1✔
96	return this.queryAttribute(attributePid);	1✔
97	});
98	}	1✔
99
100	protected AttributeInfo queryAttribute(String attributePid) {
101	return http.get()	1✔
102	.uri(uriBuilder -> uriBuilder	1✔
103	.path(attributePid)	1✔
104	.build())	1✔
105	.exchange((clientRequest, clientResponse) -> {	1✔
106	if (clientResponse.getStatusCode().is2xxSuccessful()) {	1✔
107	try (InputStream inputStream = clientResponse.getBody()) {	1✔
108	String body = new String(inputStream.readAllBytes());	1✔
109	return extractAttributeInformation(attributePid, Application.jsonObjectMapper().readTree(body));	1✔
NEW 110	} catch (IOException e) {	×
NEW 111	throw new TypeNotFoundException(attributePid);	×
112	}
113	} else {
114	throw new TypeNotFoundException(attributePid);	1✔
115	}
116	});
117	}
118
119	protected AttributeInfo extractAttributeInformation(String attributePid, JsonNode jsonNode) {
120	String typeName = jsonNode.path("type").asText();	1✔
121	String name = jsonNode.path("name").asText();	1✔
122	Set<SchemaInfo> schemas = this.querySchemas(attributePid);	1✔
123	return new AttributeInfo(attributePid, name, typeName, schemas);	1✔
124	}
125
126	protected Set<SchemaInfo> querySchemas(String maybeSchemaPid) throws TypeNotFoundException, ExternalServiceException {
127	return schemaSetGenerator.generateFor(maybeSchemaPid).join();	1✔
128	}
129
130	protected RegisteredProfile queryProfile(String maybeProfilePid) throws TypeNotFoundException, ExternalServiceException {
131	return http.get()	1✔
132	.uri(uriBuilder -> uriBuilder	1✔
133	.path(maybeProfilePid)	1✔
134	.build())	1✔
135	.exchange((clientRequest, clientResponse) -> {	1✔
136	if (clientResponse.getStatusCode().is2xxSuccessful()) {	1✔
137	InputStream inputStream = clientResponse.getBody();	1✔
138	String body = new String(inputStream.readAllBytes());	1✔
139	inputStream.close();	1✔
140	return extractProfileInformation(maybeProfilePid, Application.jsonObjectMapper().readTree(body));	1✔
141	} else {
NEW 142	throw new TypeNotFoundException(maybeProfilePid);	×
143	}
144	});
145	}
146
147	protected RegisteredProfile extractProfileInformation(String profilePid, JsonNode typeApiResponse)
148	throws TypeNotFoundException, ExternalServiceException {
149
150	List<RegisteredProfileAttribute> attributes = new ArrayList<>();	1✔
151	typeApiResponse.path("content").path("properties").forEach(item -> {	1✔
152
153	String attributePid = Optional.ofNullable(item.path("pid").asText(null))	1✔
154	.or(() -> Optional.ofNullable(item.path("identifier").asText(null)))	1✔
155	.or(() -> Optional.ofNullable(item.path("id").asText()))	1✔
156	.orElse("");	1✔
157
158	JsonNode representations = item.path("representationsAndSemantics").path(0);	1✔
159
160	JsonNode obligationNode = representations.path("obligation");	1✔
161	boolean attributeMandatory = obligationNode.isBoolean() ? obligationNode.asBoolean()	1✔
162	: List.of("mandatory", "yes", "true").contains(obligationNode.asText().trim().toLowerCase());	1✔
163
164	JsonNode repeatableNode = representations.path("repeatable");	1✔
165	boolean attributeRepeatable = repeatableNode.isBoolean() ? repeatableNode.asBoolean()	1✔
166	: List.of("yes", "true", "repeatable").contains(repeatableNode.asText().trim().toLowerCase());	1✔
167
168	RegisteredProfileAttribute attribute = new RegisteredProfileAttribute(	1✔
169	attributePid,
170	attributeMandatory,
171	attributeRepeatable);
172
173	if (obligationNode.isNull() \|\| repeatableNode.isNull() \|\| attributePid.trim().isEmpty()) {	1✔
NEW 174	throw new ExternalServiceException(baseUrl.toString(), "Malformed attribute in profile (%s): " + attribute);	×
175	}
176	attributes.add(attribute);	1✔
177
178	});	1✔
179
180	boolean additionalAttributesDtrTestStyle = StreamSupport.stream(typeApiResponse	1✔
181	.path("content")	1✔
182	.path("representationsAndSemantics")	1✔
183	.spliterator(),	1✔
184	true)
185	.filter(JsonNode::isObject)	1✔
186	.filter(node -> node.path("expression").asText("").equals("Format"))	1✔
187	.map(node -> node.path("subSchemaRelation").asText("").equals("allowAdditionalProperties"))	1✔
188	.findFirst()	1✔
189	.orElse(true);	1✔
190	boolean additionalAttributesEoscStyle = typeApiResponse	1✔
191	.path("content")	1✔
192	.path("addProps")	1✔
193	.asBoolean(true);	1✔
194	// As the default is true, we assume that additional attributes are disallowed if one indicator is false:
195	boolean profileDefinitionAllowsAdditionalAttributes = additionalAttributesDtrTestStyle && additionalAttributesEoscStyle;	1✔
196
197	return new RegisteredProfile(profilePid, profileDefinitionAllowsAdditionalAttributes, new ImmutableList<>(attributes));	1✔
198	}
199
200	@Override
201	public CompletableFuture<AttributeInfo> queryAttributeInfo(String attributePid) {
202	return this.attributeCache.get(attributePid);	1✔
203	}
204
205	@Override
206	public CompletableFuture<RegisteredProfile> queryAsProfile(String profilePid) {
207	return this.profileCache.get(profilePid);	1✔
208	}
209
210	@Override
211	public String getRegistryIdentifier() {
NEW 212	return baseUrl.toString();	×
213	}
214	}

kit-data-manager / pit-service / #451

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous