#444

Committed 21 Jan 2025 11:55PM UTC coverage: 75.383% (+3.0%) from 72.4%

Build # #444

Build Type

Pull #218

github

Committed by

web-flow

Commit Message

Merge 0aada098b into 459f0c036

Pull Request Pull Request #218: Type-Api support and validation speedup

Run Details

257 of 322 new or added lines in 18 files covered. (79.81%)

3 existing lines in 2 files now uncovered.

885 of 1174 relevant lines covered (75.38%)

0.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.63

/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java

package edu.kit.datamanager.pit.typeregistry.impl;

import com.fasterxml.jackson.databind.JsonNode;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.Caffeine;
import edu.kit.datamanager.pit.Application;
import edu.kit.datamanager.pit.common.ExternalServiceException;
import edu.kit.datamanager.pit.common.InvalidConfigException;
import edu.kit.datamanager.pit.common.TypeNotFoundException;
import edu.kit.datamanager.pit.configuration.ApplicationProperties;
import edu.kit.datamanager.pit.domain.ImmutableList;
import edu.kit.datamanager.pit.typeregistry.AttributeInfo;
import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
import edu.kit.datamanager.pit.typeregistry.RegisteredProfile;
import edu.kit.datamanager.pit.typeregistry.RegisteredProfileAttribute;
import edu.kit.datamanager.pit.typeregistry.schema.SchemaInfo;
import edu.kit.datamanager.pit.typeregistry.schema.SchemaSetGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.client.RestClient;

import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.StreamSupport;

public class TypeApi implements ITypeRegistry {

    private static final Logger LOG = LoggerFactory.getLogger(TypeApi.class);

    protected final URL baseUrl;
    protected final RestClient http;
    protected final AsyncLoadingCache<String, RegisteredProfile> profileCache;
    protected final AsyncLoadingCache<String, AttributeInfo> attributeCache;

    protected final SchemaSetGenerator schemaSetGenerator;

    public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGenerator) {
        this.schemaSetGenerator = schemaSetGenerator;
        this.baseUrl = properties.getTypeRegistryUri();
        String baseUri;
        try {
            baseUri = baseUrl.toURI().resolve("v1/types/").toString();
        } catch (URISyntaxException e) {
            throw new InvalidConfigException("Type-Api base url not valid: " + baseUrl);
        }
        this.http = RestClient.builder().baseUrl(baseUri).build();

        // TODO better name caching properties (and consider extending them)
        int maximumSize = properties.getCacheMaxEntries();
        long expireAfterWrite = properties.getCacheExpireAfterWriteLifetime();

        this.profileCache = Caffeine.newBuilder()
                .maximumSize(maximumSize)
                .executor(Application.newExecutor())
                .refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))
                .expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)
                .removalListener((key, value, cause) ->
                        LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)
                )
                .buildAsync(maybeProfilePid -> {
                    LOG.trace("Loading profile {} to cache.", maybeProfilePid);
                    return this.queryProfile(maybeProfilePid);
                });

        this.attributeCache = Caffeine.newBuilder()
                .maximumSize(maximumSize)
                .executor(Application.newExecutor())
                .refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))
                .expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)
                .removalListener((key, value, cause) ->
                    LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)
                )
                .buildAsync(attributePid -> {
                    LOG.trace("Loading attribute {} to cache.", attributePid);
                    return this.queryAttribute(attributePid);
                });
    }

    protected AttributeInfo queryAttribute(String attributePid) {
        return http.get()
                .uri(uriBuilder -> uriBuilder
                        .path(attributePid)
                        .build())
                .exchange((clientRequest, clientResponse) -> {
                    if (clientResponse.getStatusCode().is2xxSuccessful()) {
                        try (InputStream inputStream = clientResponse.getBody()) {
                            String body = new String(inputStream.readAllBytes());
                            return extractAttributeInformation(attributePid, Application.jsonObjectMapper().readTree(body));
                        } catch (IOException e) {
                            throw new TypeNotFoundException(attributePid);
                        }
                    } else {
                        throw new TypeNotFoundException(attributePid);
                    }
                });
    }

    protected AttributeInfo extractAttributeInformation(String attributePid, JsonNode jsonNode) {
        String typeName = jsonNode.path("type").asText();
        String name = jsonNode.path("name").asText();
        Set<SchemaInfo> schemas = this.querySchemas(attributePid);
        return new AttributeInfo(attributePid, name, typeName, schemas);
    }

    protected Set<SchemaInfo> querySchemas(String maybeSchemaPid) throws TypeNotFoundException, ExternalServiceException {
        return schemaSetGenerator.generateFor(maybeSchemaPid).join();
    }

    protected RegisteredProfile queryProfile(String maybeProfilePid) throws TypeNotFoundException, ExternalServiceException {
        return http.get()
                .uri(uriBuilder -> uriBuilder
                        .path(maybeProfilePid)
                        .build())
                .exchange((clientRequest, clientResponse) -> {
                    if (clientResponse.getStatusCode().is2xxSuccessful()) {
                        InputStream inputStream = clientResponse.getBody();
                        String body = new String(inputStream.readAllBytes());
                        inputStream.close();
                        return extractProfileInformation(maybeProfilePid, Application.jsonObjectMapper().readTree(body));
                    } else {
                        throw new TypeNotFoundException(maybeProfilePid);
                    }
                });
    }

    protected RegisteredProfile extractProfileInformation(String profilePid, JsonNode typeApiResponse)
            throws TypeNotFoundException, ExternalServiceException {

        List<RegisteredProfileAttribute> attributes = new ArrayList<>();
        typeApiResponse.path("content").path("properties").forEach(item -> {

            String attributePid = Optional.ofNullable(item.path("pid").asText(null))
                    .or(() -> Optional.ofNullable(item.path("identifier").asText(null)))
                    .or(() -> Optional.ofNullable(item.path("id").asText()))
                    .orElse("");

            JsonNode representations = item.path("representationsAndSemantics").path(0);

            JsonNode obligationNode = representations.path("obligation");
            boolean attributeMandatory = obligationNode.isBoolean() ? obligationNode.asBoolean()
                    : List.of("mandatory", "yes", "true").contains(obligationNode.asText().trim().toLowerCase());

            JsonNode repeatableNode = representations.path("repeatable");
            boolean attributeRepeatable = repeatableNode.isBoolean() ? repeatableNode.asBoolean()
                    : List.of("yes", "true", "repeatable").contains(repeatableNode.asText().trim().toLowerCase());

            RegisteredProfileAttribute attribute = new RegisteredProfileAttribute(
                    attributePid,
                    attributeMandatory,
                    attributeRepeatable);

            if (obligationNode.isNull() || repeatableNode.isNull() || attributePid.trim().isEmpty()) {
                throw new ExternalServiceException(baseUrl.toString(), "Malformed attribute in profile (%s): " + attribute);
            }
            attributes.add(attribute);

        });

        boolean additionalAttributesDtrTestStyle = StreamSupport.stream(typeApiResponse
                .path("content")
                .path("representationsAndSemantics")
                .spliterator(),
                true)
                .filter(JsonNode::isObject)
                .filter(node -> node.path("expression").asText("").equals("Format"))
                .map(node -> node.path("subSchemaRelation").asText("").equals("allowAdditionalProperties"))
                .findFirst()
                .orElse(true);
        boolean additionalAttributesEoscStyle = typeApiResponse
                .path("content")
                .path("addProps")
                .asBoolean(true);
        // As the default is true, we assume that additional attributes are disallowed if one indicator is false:
        boolean profileDefinitionAllowsAdditionalAttributes = additionalAttributesDtrTestStyle && additionalAttributesEoscStyle;

        return new RegisteredProfile(profilePid, profileDefinitionAllowsAdditionalAttributes, new ImmutableList<>(attributes));
    }

    @Override
    public CompletableFuture<AttributeInfo> queryAttributeInfo(String attributePid) {
        return this.attributeCache.get(attributePid);
    }

    @Override
    public CompletableFuture<RegisteredProfile> queryAsProfile(String profilePid) {
        return this.profileCache.get(profilePid);
    }

    @Override
    public String getRegistryIdentifier() {
        return baseUrl.toString();
    }
}

1	package edu.kit.datamanager.pit.typeregistry.impl;
2
3	import com.fasterxml.jackson.databind.JsonNode;
4	import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
5	import com.github.benmanes.caffeine.cache.Caffeine;
6	import edu.kit.datamanager.pit.Application;
7	import edu.kit.datamanager.pit.common.ExternalServiceException;
8	import edu.kit.datamanager.pit.common.InvalidConfigException;
9	import edu.kit.datamanager.pit.common.TypeNotFoundException;
10	import edu.kit.datamanager.pit.configuration.ApplicationProperties;
11	import edu.kit.datamanager.pit.domain.ImmutableList;
12	import edu.kit.datamanager.pit.typeregistry.AttributeInfo;
13	import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
14	import edu.kit.datamanager.pit.typeregistry.RegisteredProfile;
15	import edu.kit.datamanager.pit.typeregistry.RegisteredProfileAttribute;
16	import edu.kit.datamanager.pit.typeregistry.schema.SchemaInfo;
17	import edu.kit.datamanager.pit.typeregistry.schema.SchemaSetGenerator;
18	import org.slf4j.Logger;
19	import org.slf4j.LoggerFactory;
20	import org.springframework.web.client.RestClient;
21
22	import java.io.IOException;
23	import java.io.InputStream;
24	import java.net.URISyntaxException;
25	import java.net.URL;
26	import java.time.Duration;
27	import java.util.ArrayList;
28	import java.util.List;
29	import java.util.Optional;
30	import java.util.Set;
31	import java.util.concurrent.CompletableFuture;
32	import java.util.concurrent.TimeUnit;
33	import java.util.stream.StreamSupport;
34
35	public class TypeApi implements ITypeRegistry {
36
37	private static final Logger LOG = LoggerFactory.getLogger(TypeApi.class);	1✔
38
39	protected final URL baseUrl;
40	protected final RestClient http;
41	protected final AsyncLoadingCache<String, RegisteredProfile> profileCache;
42	protected final AsyncLoadingCache<String, AttributeInfo> attributeCache;
43
44	protected final SchemaSetGenerator schemaSetGenerator;
45
46	public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGenerator) {	1✔
47	this.schemaSetGenerator = schemaSetGenerator;	1✔
48	this.baseUrl = properties.getTypeRegistryUri();	1✔
49	String baseUri;
50	try {
51	baseUri = baseUrl.toURI().resolve("v1/types/").toString();	1✔
NEW 52	} catch (URISyntaxException e) {	×
NEW 53	throw new InvalidConfigException("Type-Api base url not valid: " + baseUrl);	×
54	}	1✔
55	this.http = RestClient.builder().baseUrl(baseUri).build();	1✔
56
57	// TODO better name caching properties (and consider extending them)
58	int maximumSize = properties.getCacheMaxEntries();	1✔
59	long expireAfterWrite = properties.getCacheExpireAfterWriteLifetime();	1✔
60
61	this.profileCache = Caffeine.newBuilder()	1✔
62	.maximumSize(maximumSize)	1✔
63	.executor(Application.newExecutor())	1✔
64	.refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))	1✔
65	.expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)	1✔
66	.removalListener((key, value, cause) ->	1✔
NEW 67	LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)	×
68	)
69	.buildAsync(maybeProfilePid -> {	1✔
70	LOG.trace("Loading profile {} to cache.", maybeProfilePid);	1✔
71	return this.queryProfile(maybeProfilePid);	1✔
72	});
73
74	this.attributeCache = Caffeine.newBuilder()	1✔
75	.maximumSize(maximumSize)	1✔
76	.executor(Application.newExecutor())	1✔
77	.refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2))	1✔
78	.expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES)	1✔
79	.removalListener((key, value, cause) ->	1✔
NEW 80	LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause)	×
81	)
82	.buildAsync(attributePid -> {	1✔
83	LOG.trace("Loading attribute {} to cache.", attributePid);	1✔
84	return this.queryAttribute(attributePid);	1✔
85	});
86	}	1✔
87
88	protected AttributeInfo queryAttribute(String attributePid) {
89	return http.get()	1✔
90	.uri(uriBuilder -> uriBuilder	1✔
91	.path(attributePid)	1✔
92	.build())	1✔
93	.exchange((clientRequest, clientResponse) -> {	1✔
94	if (clientResponse.getStatusCode().is2xxSuccessful()) {	1✔
95	try (InputStream inputStream = clientResponse.getBody()) {	1✔
96	String body = new String(inputStream.readAllBytes());	1✔
97	return extractAttributeInformation(attributePid, Application.jsonObjectMapper().readTree(body));	1✔
NEW 98	} catch (IOException e) {	×
NEW 99	throw new TypeNotFoundException(attributePid);	×
100	}
101	} else {
102	throw new TypeNotFoundException(attributePid);	1✔
103	}
104	});
105	}
106
107	protected AttributeInfo extractAttributeInformation(String attributePid, JsonNode jsonNode) {
108	String typeName = jsonNode.path("type").asText();	1✔
109	String name = jsonNode.path("name").asText();	1✔
110	Set<SchemaInfo> schemas = this.querySchemas(attributePid);	1✔
111	return new AttributeInfo(attributePid, name, typeName, schemas);	1✔
112	}
113
114	protected Set<SchemaInfo> querySchemas(String maybeSchemaPid) throws TypeNotFoundException, ExternalServiceException {
115	return schemaSetGenerator.generateFor(maybeSchemaPid).join();	1✔
116	}
117
118	protected RegisteredProfile queryProfile(String maybeProfilePid) throws TypeNotFoundException, ExternalServiceException {
119	return http.get()	1✔
120	.uri(uriBuilder -> uriBuilder	1✔
121	.path(maybeProfilePid)	1✔
122	.build())	1✔
123	.exchange((clientRequest, clientResponse) -> {	1✔
124	if (clientResponse.getStatusCode().is2xxSuccessful()) {	1✔
125	InputStream inputStream = clientResponse.getBody();	1✔
126	String body = new String(inputStream.readAllBytes());	1✔
127	inputStream.close();	1✔
128	return extractProfileInformation(maybeProfilePid, Application.jsonObjectMapper().readTree(body));	1✔
129	} else {
NEW 130	throw new TypeNotFoundException(maybeProfilePid);	×
131	}
132	});
133	}
134
135	protected RegisteredProfile extractProfileInformation(String profilePid, JsonNode typeApiResponse)
136	throws TypeNotFoundException, ExternalServiceException {
137
138	List<RegisteredProfileAttribute> attributes = new ArrayList<>();	1✔
139	typeApiResponse.path("content").path("properties").forEach(item -> {	1✔
140
141	String attributePid = Optional.ofNullable(item.path("pid").asText(null))	1✔
142	.or(() -> Optional.ofNullable(item.path("identifier").asText(null)))	1✔
143	.or(() -> Optional.ofNullable(item.path("id").asText()))	1✔
144	.orElse("");	1✔
145
146	JsonNode representations = item.path("representationsAndSemantics").path(0);	1✔
147
148	JsonNode obligationNode = representations.path("obligation");	1✔
149	boolean attributeMandatory = obligationNode.isBoolean() ? obligationNode.asBoolean()	1✔
150	: List.of("mandatory", "yes", "true").contains(obligationNode.asText().trim().toLowerCase());	1✔
151
152	JsonNode repeatableNode = representations.path("repeatable");	1✔
153	boolean attributeRepeatable = repeatableNode.isBoolean() ? repeatableNode.asBoolean()	1✔
154	: List.of("yes", "true", "repeatable").contains(repeatableNode.asText().trim().toLowerCase());	1✔
155
156	RegisteredProfileAttribute attribute = new RegisteredProfileAttribute(	1✔
157	attributePid,
158	attributeMandatory,
159	attributeRepeatable);
160
161	if (obligationNode.isNull() \|\| repeatableNode.isNull() \|\| attributePid.trim().isEmpty()) {	1✔
NEW 162	throw new ExternalServiceException(baseUrl.toString(), "Malformed attribute in profile (%s): " + attribute);	×
163	}
164	attributes.add(attribute);	1✔
165
166	});	1✔
167
168	boolean additionalAttributesDtrTestStyle = StreamSupport.stream(typeApiResponse	1✔
169	.path("content")	1✔
170	.path("representationsAndSemantics")	1✔
171	.spliterator(),	1✔
172	true)
173	.filter(JsonNode::isObject)	1✔
174	.filter(node -> node.path("expression").asText("").equals("Format"))	1✔
175	.map(node -> node.path("subSchemaRelation").asText("").equals("allowAdditionalProperties"))	1✔
176	.findFirst()	1✔
177	.orElse(true);	1✔
178	boolean additionalAttributesEoscStyle = typeApiResponse	1✔
179	.path("content")	1✔
180	.path("addProps")	1✔
181	.asBoolean(true);	1✔
182	// As the default is true, we assume that additional attributes are disallowed if one indicator is false:
183	boolean profileDefinitionAllowsAdditionalAttributes = additionalAttributesDtrTestStyle && additionalAttributesEoscStyle;	1✔
184
185	return new RegisteredProfile(profilePid, profileDefinitionAllowsAdditionalAttributes, new ImmutableList<>(attributes));	1✔
186	}
187
188	@Override
189	public CompletableFuture<AttributeInfo> queryAttributeInfo(String attributePid) {
190	return this.attributeCache.get(attributePid);	1✔
191	}
192
193	@Override
194	public CompletableFuture<RegisteredProfile> queryAsProfile(String profilePid) {
195	return this.profileCache.get(profilePid);	1✔
196	}
197
198	@Override
199	public String getRegistryIdentifier() {
NEW 200	return baseUrl.toString();	×
201	}
202	}

kit-data-manager / pit-service / #444

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous