23138432878

Committed 16 Mar 2026 10:09AM UTC coverage: 15.99% (+0.2%) from 15.811%

Build # 23138432878

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #402 from knowledgepixels/fix/401-bounded-api-cache

Fix unbounded memory growth and resource exhaustion

Run Details

717 of 5509 branches covered (13.02%)

Branch coverage included in aggregate %.

1810 of 10295 relevant lines covered (17.58%)

2.39 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

63.41

src/main/java/com/knowledgepixels/nanodash/domain/Project.java

package com.knowledgepixels.nanodash.domain;

import com.knowledgepixels.nanodash.ApiCache;
import com.knowledgepixels.nanodash.NanodashThreadPool;
import com.knowledgepixels.nanodash.QueryApiAccess;
import com.knowledgepixels.nanodash.Utils;
import com.knowledgepixels.nanodash.template.Template;
import com.knowledgepixels.nanodash.template.TemplateData;
import com.knowledgepixels.nanodash.vocabulary.KPXL_TERMS;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.Literal;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.model.vocabulary.DCTERMS;
import org.nanopub.Nanopub;
import org.nanopub.extra.services.ApiResponse;
import org.nanopub.extra.services.ApiResponseEntry;
import org.nanopub.extra.services.QueryRef;
import org.nanopub.vocabulary.NTEMPLATE;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

/**
 * Class representing a Nanodash project.
 */
public class Project implements Serializable {

    private static final Logger logger = LoggerFactory.getLogger(Project.class);
    private static List<Project> projectList = null;
    private static ConcurrentMap<String, Project> projectsByCoreInfo = new ConcurrentHashMap<>();
    private static ConcurrentMap<String, Project> projectsById = new ConcurrentHashMap<>();

    /**
     * Refresh the list of projects from the given API response.
     *
     * @param resp The API response containing project data.
     */
    public static synchronized void refresh(ApiResponse resp) {
        projectList = new ArrayList<>();
        ConcurrentMap<String, Project> prevProjectsByCoreInfoPrev = projectsByCoreInfo;
        projectsByCoreInfo = new ConcurrentHashMap<>();
        projectsById.clear();
        for (ApiResponseEntry entry : resp.getData()) {
            Project project = new Project(entry.get("project"), entry.get("label"), entry.get("np"));
            Project prevProject = prevProjectsByCoreInfoPrev.get(project.getCoreInfoString());
            if (prevProject != null) project = prevProject;
            projectList.add(project);
            projectsByCoreInfo.put(project.getCoreInfoString(), project);
            projectsById.put(project.getId(), project);
        }
    }

    /**
     * Ensure that the project list is loaded. If not, it fetches the data from the API.
     */
    public static void ensureLoaded() {
        if (projectList == null) {
            refresh(ApiCache.retrieveResponseSync(new QueryRef(QueryApiAccess.GET_PROJECTS), true));
        }
    }

    /**
     * Get the list of all projects.
     */
    public static List<Project> getProjectList() {
        ensureLoaded();
        return projectList;
    }

    /**
     * Get a project by its ID.
     *
     * @param id The ID of the project.
     * @return The project with the given ID, or null if not found.
     */
    public static Project get(String id) {
        ensureLoaded();
        return projectsById.get(id);
    }

    /**
     * Mark all projects as needing data refresh.
     */
    public static void refresh() {
        logger.info("Refreshing projects...");
        ensureLoaded();
        for (Project project : projectList) {
            project.dataNeedsUpdate = true;
        }
    }

    private String id, label, rootNanopubId;
    private Nanopub rootNanopub = null;

    private String description = null;
    private List<IRI> owners = new ArrayList<>();
    private List<IRI> members = new ArrayList<>();
    private ConcurrentMap<String, IRI> ownerPubkeyMap = new ConcurrentHashMap<>();
    private List<Template> templates = new ArrayList<>();
    private Set<String> templateTags = new HashSet<>();
    private ConcurrentMap<String, List<Template>> templatesPerTag = new ConcurrentHashMap<>();
    private List<IRI> queryIds = new ArrayList<>();
    private IRI defaultProvenance = null;

    private boolean dataInitialized = false;
    private boolean dataNeedsUpdate = true;

    private Project(String id, String label, String rootNanopubId) {
        this.id = id;
        this.label = label;
        this.rootNanopubId = rootNanopubId;
        this.rootNanopub = Utils.getAsNanopub(rootNanopubId);

        for (Statement st : rootNanopub.getAssertion()) {
            if (st.getSubject().stringValue().equals(getId())) {
                if (st.getPredicate().equals(DCTERMS.DESCRIPTION)) {
                    description = st.getObject().stringValue();
                } else if (st.getPredicate().equals(KPXL_TERMS.HAS_OWNER) && st.getObject() instanceof IRI obj) {
                    addOwner(obj);
                } else if (st.getPredicate().equals(KPXL_TERMS.HAS_PINNED_TEMPLATE) && st.getObject() instanceof IRI obj) {
                    templates.add(TemplateData.get().getTemplate(obj.stringValue()));
                } else if (st.getPredicate().equals(KPXL_TERMS.HAS_PINNED_QUERY) && st.getObject() instanceof IRI obj) {
                    queryIds.add(obj);
                } else if (st.getPredicate().equals(NTEMPLATE.HAS_DEFAULT_PROVENANCE) && st.getObject() instanceof IRI obj) {
                    defaultProvenance = obj;
                }
            } else if (st.getPredicate().equals(NTEMPLATE.HAS_TAG) && st.getObject() instanceof Literal l) {
                templateTags.add(l.stringValue());
                List<Template> list = templatesPerTag.get(l.stringValue());
                if (list == null) {
                    list = new ArrayList<>();
                    templatesPerTag.put(l.stringValue(), list);
                }
                list.add(TemplateData.get().getTemplate(st.getSubject().stringValue()));
            }
        }

    }

    private void addOwner(IRI owner) {
        // TODO This isn't efficient for long owner lists:
        if (owners.contains(owner)) return;
        owners.add(owner);
        UserData ud = User.getUserData();
        for (String pubkeyhash : ud.getPubkeyHashes(owner, true)) {
            ownerPubkeyMap.put(pubkeyhash, owner);
        }
    }

    /**
     * Get the ID of the project.
     *
     * @return The project ID.
     */
    public String getId() {
        return id;
    }

    /**
     * Get the root nanopublication ID of the project.
     *
     * @return The root nanopublication ID.
     */
    public String getRootNanopubId() {
        return rootNanopubId;
    }

    /**
     * Get a string containing the core information of the project (ID and root nanopub ID).
     *
     * @return A string with the project ID and root nanopub ID.
     */
    public String getCoreInfoString() {
        return id + " " + rootNanopubId;
    }

    /**
     * Get the root nanopublication of the project.
     *
     * @return The root nanopublication.
     */
    public Nanopub getRootNanopub() {
        return rootNanopub;
    }

    /**
     * Get the label of the project.
     *
     * @return The project label.
     */
    public String getLabel() {
        return label;
    }

    /**
     * Get the description of the project.
     *
     * @return The project description.
     */
    public String getDescription() {
        return description;
    }

    /**
     * Check if the project data has been initialized.
     *
     * @return True if the data is initialized, false otherwise.
     */
    public boolean isDataInitialized() {
        triggerDataUpdate();
        return dataInitialized;
    }

    /**
     * Get the list of owners of the project.
     *
     * @return A list of IRIs representing the owners.
     */
    public List<IRI> getOwners() {
        triggerDataUpdate();
        return owners;
    }

    /**
     * Get the list of members of the project.
     *
     * @return A list of IRIs representing the members.
     */
    public List<IRI> getMembers() {
        triggerDataUpdate();
        return members;
    }

    /**
     * Get the list of templates associated with the project.
     *
     * @return A list of templates.
     */
    public List<Template> getTemplates() {
        return templates;
    }

    /**
     * Get the set of template tags associated with the project.
     *
     * @return A set of template tags.
     */
    public Set<String> getTemplateTags() {
        return templateTags;
    }

    /**
     * Get a map of templates categorized by their tags.
     *
     * @return A concurrent map where keys are tags and values are lists of templates.
     */
    public ConcurrentMap<String, List<Template>> getTemplatesPerTag() {
        return templatesPerTag;
    }

    /**
     * Get the list of query IDs associated with the project.
     *
     * @return A list of IRIs representing the query IDs.
     */
    public List<IRI> getQueryIds() {
        return queryIds;
    }

    /**
     * Get the default provenance IRI for the project.
     *
     * @return The default provenance IRI.
     */
    public IRI getDefaultProvenance() {
        return defaultProvenance;
    }

    private synchronized void triggerDataUpdate() {
        if (dataNeedsUpdate) {
            NanodashThreadPool.submit(() -> {
                for (ApiResponseEntry r : ApiCache.retrieveResponseSync(new QueryRef(QueryApiAccess.GET_OWNERS, "unit", id), true).getData()) {
                    String pubkeyhash = r.get("pubkeyhash");
                    if (ownerPubkeyMap.containsKey(pubkeyhash)) {
                        addOwner(Utils.vf.createIRI(r.get("owner")));
                    }
                }
                members = new ArrayList<>();
                for (ApiResponseEntry r : ApiCache.retrieveResponseSync(new QueryRef(QueryApiAccess.GET_MEMBERS, "unit", id), true).getData()) {
                    IRI memberId = Utils.vf.createIRI(r.get("member"));
                    // TODO These checks are inefficient for long member lists:
                    if (owners.contains(memberId)) continue;
                    if (members.contains(memberId)) continue;
                    members.add(memberId);
                }
                owners.sort(User.getUserData().userComparator);
                members.sort(User.getUserData().userComparator);
                dataInitialized = true;
            });
            dataNeedsUpdate = false;
        }
    }

}

1	package com.knowledgepixels.nanodash.domain;
2
3	import com.knowledgepixels.nanodash.ApiCache;
4	import com.knowledgepixels.nanodash.NanodashThreadPool;
5	import com.knowledgepixels.nanodash.QueryApiAccess;
6	import com.knowledgepixels.nanodash.Utils;
7	import com.knowledgepixels.nanodash.template.Template;
8	import com.knowledgepixels.nanodash.template.TemplateData;
9	import com.knowledgepixels.nanodash.vocabulary.KPXL_TERMS;
10	import org.eclipse.rdf4j.model.IRI;
11	import org.eclipse.rdf4j.model.Literal;
12	import org.eclipse.rdf4j.model.Statement;
13	import org.eclipse.rdf4j.model.vocabulary.DCTERMS;
14	import org.nanopub.Nanopub;
15	import org.nanopub.extra.services.ApiResponse;
16	import org.nanopub.extra.services.ApiResponseEntry;
17	import org.nanopub.extra.services.QueryRef;
18	import org.nanopub.vocabulary.NTEMPLATE;
19	import org.slf4j.Logger;
20	import org.slf4j.LoggerFactory;
21
22	import java.io.Serializable;
23	import java.util.ArrayList;
24	import java.util.HashSet;
25	import java.util.List;
26	import java.util.Set;
27	import java.util.concurrent.ConcurrentHashMap;
28	import java.util.concurrent.ConcurrentMap;
29
30	/**
31	* Class representing a Nanodash project.
32	*/
33	public class Project implements Serializable {
34
35	private static final Logger logger = LoggerFactory.getLogger(Project.class);	9✔
36	private static List<Project> projectList = null;	6✔
37	private static ConcurrentMap<String, Project> projectsByCoreInfo = new ConcurrentHashMap<>();	12✔
38	private static ConcurrentMap<String, Project> projectsById = new ConcurrentHashMap<>();	15✔
39
40	/**
41	* Refresh the list of projects from the given API response.
42	*
43	* @param resp The API response containing project data.
44	*/
45	public static synchronized void refresh(ApiResponse resp) {
46	projectList = new ArrayList<>();	12✔
47	ConcurrentMap<String, Project> prevProjectsByCoreInfoPrev = projectsByCoreInfo;	6✔
48	projectsByCoreInfo = new ConcurrentHashMap<>();	12✔
49	projectsById.clear();	6✔
50	for (ApiResponseEntry entry : resp.getData()) {	33✔
51	Project project = new Project(entry.get("project"), entry.get("label"), entry.get("np"));	39✔
52	Project prevProject = prevProjectsByCoreInfoPrev.get(project.getCoreInfoString());	18✔
53	if (prevProject != null) project = prevProject;	6!
54	projectList.add(project);	12✔
55	projectsByCoreInfo.put(project.getCoreInfoString(), project);	18✔
56	projectsById.put(project.getId(), project);	18✔
57	}	3✔
58	}	3✔
59
60	/**
61	* Ensure that the project list is loaded. If not, it fetches the data from the API.
62	*/
63	public static void ensureLoaded() {
64	if (projectList == null) {	6!
65	refresh(ApiCache.retrieveResponseSync(new QueryRef(QueryApiAccess.GET_PROJECTS), true));	21✔
66	}
67	}	3✔
68
69	/**
70	* Get the list of all projects.
71	*/
72	public static List<Project> getProjectList() {
73	ensureLoaded();	×
74	return projectList;	×
75	}
76
77	/**
78	* Get a project by its ID.
79	*
80	* @param id The ID of the project.
81	* @return The project with the given ID, or null if not found.
82	*/
83	public static Project get(String id) {
84	ensureLoaded();	×
85	return projectsById.get(id);	×
86	}
87
88	/**
89	* Mark all projects as needing data refresh.
90	*/
91	public static void refresh() {
92	logger.info("Refreshing projects...");	9✔
93	ensureLoaded();	3✔
94	for (Project project : projectList) {	30✔
95	project.dataNeedsUpdate = true;	9✔
96	}	3✔
97	}	3✔
98
99	private String id, label, rootNanopubId;
100	private Nanopub rootNanopub = null;	9✔
101
102	private String description = null;	9✔
103	private List<IRI> owners = new ArrayList<>();	15✔
104	private List<IRI> members = new ArrayList<>();	15✔
105	private ConcurrentMap<String, IRI> ownerPubkeyMap = new ConcurrentHashMap<>();	15✔
106	private List<Template> templates = new ArrayList<>();	15✔
107	private Set<String> templateTags = new HashSet<>();	15✔
108	private ConcurrentMap<String, List<Template>> templatesPerTag = new ConcurrentHashMap<>();	15✔
109	private List<IRI> queryIds = new ArrayList<>();	15✔
110	private IRI defaultProvenance = null;	9✔
111
112	private boolean dataInitialized = false;	9✔
113	private boolean dataNeedsUpdate = true;	9✔
114
115	private Project(String id, String label, String rootNanopubId) {	6✔
116	this.id = id;	9✔
117	this.label = label;	9✔
118	this.rootNanopubId = rootNanopubId;	9✔
119	this.rootNanopub = Utils.getAsNanopub(rootNanopubId);	12✔
120
121	for (Statement st : rootNanopub.getAssertion()) {	36✔
122	if (st.getSubject().stringValue().equals(getId())) {	21✔
123	if (st.getPredicate().equals(DCTERMS.DESCRIPTION)) {	15✔
124	description = st.getObject().stringValue();	18✔
125	} else if (st.getPredicate().equals(KPXL_TERMS.HAS_OWNER) && st.getObject() instanceof IRI obj) {	42!
126	addOwner(obj);	12✔
127	} else if (st.getPredicate().equals(KPXL_TERMS.HAS_PINNED_TEMPLATE) && st.getObject() instanceof IRI obj) {	42!
128	templates.add(TemplateData.get().getTemplate(obj.stringValue()));	27✔
129	} else if (st.getPredicate().equals(KPXL_TERMS.HAS_PINNED_QUERY) && st.getObject() instanceof IRI obj) {	42!
130	queryIds.add(obj);	18✔
131	} else if (st.getPredicate().equals(NTEMPLATE.HAS_DEFAULT_PROVENANCE) && st.getObject() instanceof IRI obj) {	42!
132	defaultProvenance = obj;	12✔
133	}
134	} else if (st.getPredicate().equals(NTEMPLATE.HAS_TAG) && st.getObject() instanceof Literal l) {	42!
135	templateTags.add(l.stringValue());	18✔
136	List<Template> list = templatesPerTag.get(l.stringValue());	21✔
137	if (list == null) {	6✔
138	list = new ArrayList<>();	12✔
139	templatesPerTag.put(l.stringValue(), list);	21✔
140	}
141	list.add(TemplateData.get().getTemplate(st.getSubject().stringValue()));	24✔
142	}
143	}	3✔
144
145	}	3✔
146
147	private void addOwner(IRI owner) {
148	// TODO This isn't efficient for long owner lists:
149	if (owners.contains(owner)) return;	15!
150	owners.add(owner);	15✔
151	UserData ud = User.getUserData();	6✔
152	for (String pubkeyhash : ud.getPubkeyHashes(owner, true)) {	42✔
153	ownerPubkeyMap.put(pubkeyhash, owner);	18✔
154	}	3✔
155	}	3✔
156
157	/**
158	* Get the ID of the project.
159	*
160	* @return The project ID.
161	*/
162	public String getId() {
163	return id;	9✔
164	}
165
166	/**
167	* Get the root nanopublication ID of the project.
168	*
169	* @return The root nanopublication ID.
170	*/
171	public String getRootNanopubId() {
172	return rootNanopubId;	×
173	}
174
175	/**
176	* Get a string containing the core information of the project (ID and root nanopub ID).
177	*
178	* @return A string with the project ID and root nanopub ID.
179	*/
180	public String getCoreInfoString() {
181	return id + " " + rootNanopubId;	18✔
182	}
183
184	/**
185	* Get the root nanopublication of the project.
186	*
187	* @return The root nanopublication.
188	*/
189	public Nanopub getRootNanopub() {
190	return rootNanopub;	×
191	}
192
193	/**
194	* Get the label of the project.
195	*
196	* @return The project label.
197	*/
198	public String getLabel() {
199	return label;	×
200	}
201
202	/**
203	* Get the description of the project.
204	*
205	* @return The project description.
206	*/
207	public String getDescription() {
208	return description;	×
209	}
210
211	/**
212	* Check if the project data has been initialized.
213	*
214	* @return True if the data is initialized, false otherwise.
215	*/
216	public boolean isDataInitialized() {
217	triggerDataUpdate();	×
218	return dataInitialized;	×
219	}
220
221	/**
222	* Get the list of owners of the project.
223	*
224	* @return A list of IRIs representing the owners.
225	*/
226	public List<IRI> getOwners() {
227	triggerDataUpdate();	×
228	return owners;	×
229	}
230
231	/**
232	* Get the list of members of the project.
233	*
234	* @return A list of IRIs representing the members.
235	*/
236	public List<IRI> getMembers() {
237	triggerDataUpdate();	×
238	return members;	×
239	}
240
241	/**
242	* Get the list of templates associated with the project.
243	*
244	* @return A list of templates.
245	*/
246	public List<Template> getTemplates() {
247	return templates;	×
248	}
249
250	/**
251	* Get the set of template tags associated with the project.
252	*
253	* @return A set of template tags.
254	*/
255	public Set<String> getTemplateTags() {
256	return templateTags;	×
257	}
258
259	/**
260	* Get a map of templates categorized by their tags.
261	*
262	* @return A concurrent map where keys are tags and values are lists of templates.
263	*/
264	public ConcurrentMap<String, List<Template>> getTemplatesPerTag() {
265	return templatesPerTag;	×
266	}
267
268	/**
269	* Get the list of query IDs associated with the project.
270	*
271	* @return A list of IRIs representing the query IDs.
272	*/
273	public List<IRI> getQueryIds() {
274	return queryIds;	×
275	}
276
277	/**
278	* Get the default provenance IRI for the project.
279	*
280	* @return The default provenance IRI.
281	*/
282	public IRI getDefaultProvenance() {
283	return defaultProvenance;	×
284	}
285
286	private synchronized void triggerDataUpdate() {
287	if (dataNeedsUpdate) {	×
288	NanodashThreadPool.submit(() -> {	×
289	for (ApiResponseEntry r : ApiCache.retrieveResponseSync(new QueryRef(QueryApiAccess.GET_OWNERS, "unit", id), true).getData()) {	×
290	String pubkeyhash = r.get("pubkeyhash");	×
291	if (ownerPubkeyMap.containsKey(pubkeyhash)) {	×
292	addOwner(Utils.vf.createIRI(r.get("owner")));	×
293	}
294	}	×
295	members = new ArrayList<>();	×
296	for (ApiResponseEntry r : ApiCache.retrieveResponseSync(new QueryRef(QueryApiAccess.GET_MEMBERS, "unit", id), true).getData()) {	×
297	IRI memberId = Utils.vf.createIRI(r.get("member"));	×
298	// TODO These checks are inefficient for long member lists:
299	if (owners.contains(memberId)) continue;	×
300	if (members.contains(memberId)) continue;	×
301	members.add(memberId);	×
302	}	×
303	owners.sort(User.getUserData().userComparator);	×
304	members.sort(User.getUserData().userComparator);	×
305	dataInitialized = true;	×
306	});	×
307	dataNeedsUpdate = false;	×
308	}
309	}	×
310
311	}

knowledgepixels / nanodash / 23138432878

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous