24567987783

Committed 17 Apr 2026 01:37PM UTC coverage: 62.72% (-0.3%) from 63.052%

Build # 24567987783

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #69 from knowledgepixels/fix/init-non-tracking-repos

fix: skip nanopub-count/checksum init for non-tracking repos

Coverage Stats

281 of 494 branches covered (56.88%)

Branch coverage included in aggregate %.

789 of 1212 relevant lines covered (65.1%)

9.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.11

src/main/java/com/knowledgepixels/query/TripleStore.java

package com.knowledgepixels.query;

import org.apache.commons.exec.environment.EnvironmentUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicResponseHandler;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.eclipse.rdf4j.common.transaction.IsolationLevels;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
import org.eclipse.rdf4j.repository.Repository;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.eclipse.rdf4j.repository.http.HTTPRepository;
import org.nanopub.NanopubUtils;
import org.nanopub.vocabulary.NPA;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

/**
 * Class to access the database in the form of triple stores.
 */
public class TripleStore {

    /**
     * Name of the admin graph.
     */
    public static final String ADMIN_REPO = "admin";

    private static ValueFactory vf = SimpleValueFactory.getInstance();

    private static final Logger log = LoggerFactory.getLogger(TripleStore.class);

    private final Map<String, Repository> repositories = new LinkedHashMap<>();

    private String endpointBase = null;
    private String endpointType = null;

    private static TripleStore tripleStoreInstance;

    /**
     * Returns singleton triple store instance.
     *
     * @return Triple store instance
     */
    public static TripleStore get() {
        if (tripleStoreInstance == null) {
            try {
                tripleStoreInstance = new TripleStore();
            } catch (IOException ex) {
                log.info("Could not init TripleStore. ", ex);
            }
        }
        return tripleStoreInstance;
    }

    private TripleStore() throws IOException {
        Map<String, String> env = EnvironmentUtils.getProcEnvironment();
        endpointBase = env.get("ENDPOINT_BASE");
        log.info("Endpoint base: {}", endpointBase);
        endpointType = env.get("ENDPOINT_TYPE");

        getRepository("empty");  // Make sure empty repo exists
    }

    private final CloseableHttpClient httpclient = HttpClients.createDefault();

    @GeneratedFlagForDependentElements
    Repository getRepository(String name) {
        synchronized (this) {
            while (repositories.size() > 100) {
                Entry<String, Repository> e = repositories.entrySet().iterator().next();
                repositories.remove(e.getKey());
                log.info("Shutting down repo: {}", e.getKey());
                e.getValue().shutDown();
                log.info("Shutdown complete");
            }
            if (repositories.containsKey(name)) {
                // Move to the end of the list:
                Repository repo = repositories.remove(name);
                repositories.put(name, repo);
            } else {
                Repository repository = null;
                if (endpointType == null || endpointType.equals("rdf4j")) {
                    HTTPRepository hr = new HTTPRepository(endpointBase + "repositories/" + name);
                    hr.setHttpClient(httpclient);
                    repository = hr;
//                        } else if (endpointType.equals("virtuoso")) {
//                                repository = new VirtuosoRepository(endpointBase + name, username, password);
                } else {
                    throw new RuntimeException("Unknown repository type: " + endpointType);
                }
                repositories.put(name, repository);
                createRepo(name);
                getRepoConnection(name).close();
            }
            return repositories.get(name);
        }
    }

    /**
     * Return the repository connection for the given repository name.
     *
     * @param name repository name
     * @return repository connection
     */
    @GeneratedFlagForDependentElements
    public RepositoryConnection getRepoConnection(String name) {
        Repository repo = getRepository(name);
        if (repo == null) {
            return null;
        }
        return repo.getConnection();
    }

    @GeneratedFlagForDependentElements
    private void createRepo(String repoName) {
        if (!repoName.equals(ADMIN_REPO)) {
            getRepository(ADMIN_REPO);  // make sure admin repo is loaded first
        }
        try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
            //log.info("Trying to creating repo " + name);

            // TODO new syntax somehow doesn't work for the Lucene case:

//                        String createRegularRepoQueryString = "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n"
//                                        + "@prefix config: <tag:rdf4j.org,2023:config/>.\n"
//                                        + "[] a config:Repository ;\n"
//                                        + "    config:rep.id \"" + name + "\" ;\n"
//                                        + "    rdfs:label \"" + name + " native store\" ;\n"
//                                        + "    config:rep.impl [\n"
//                                        + "        config:rep.type \"openrdf:SailRepository\" ;\n"
//                                        + "        config:sail.impl [\n"
//                                        + "            config:sail.type \"openrdf:NativeStore\" ;\n"
//                                        + "            config:sail.iterationCacheSyncThreshold \"10000\";\n"
//                                        + "            config:native.tripleIndexes \"spoc,posc,ospc,opsc,psoc,sopc,spoc,cpos,cosp,cops,cpso,csop\" ;\n"
//                                        + "            config:sail.defaultQueryEvaluationMode \"STANDARD\"\n"
//                                        + "        ]\n"
//                                        + "    ].";
//                        String createTextRepoQueryString = "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n"
//                                        + "@prefix config: <tag:rdf4j.org,2023:config/>.\n"
//                                        + "[] a config:Repository ;\n"
//                                        + "    config:rep.id \"" + name + "\" ;\n"
//                                        + "    rdfs:label \"" + name + " native store\" ;\n"
//                                        + "    config:rep.impl [\n"
//                                        + "        config:rep.type \"openrdf:SailRepository\" ;\n"
//                                        + "        config:sail.impl [\n"
//                                        + "            config:sail.type \"openrdf:LuceneSail\" ;\n"
//                                        + "            config:sail.lucene.indexDir \"index/\" ;\n"
//                                        + "            config:delegate [\n"
//                                        + "                config:rep.type \"openrdf:SailRepository\" ;\n"
//                                        + "                config:sail.impl [\n"
//                                        + "                    config:sail.type \"openrdf:NativeStore\" ;\n"
//                                        + "                    config:sail.iterationCacheSyncThreshold \"10000\";\n"
//                                        + "                    config:native.tripleIndexes \"spoc,posc,ospc,opsc,psoc,sopc,spoc,cpos,cosp,cops,cpso,csop\" ;\n"
//                                        + "                    config:sail.defaultQueryEvaluationMode \"STANDARD\"\n"
//                                        + "                ]\n"
//                                        + "            ]\n"
//                                        + "        ]\n"
//                                        + "    ].";

            String indexTypes = "spoc,posc,ospc,cspo,cpos,cosp";
            if (repoName.startsWith("meta") || repoName.startsWith("text")) {
                indexTypes = "spoc,posc,ospc";
            }

            String createRegularRepoQueryString =
                    "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" +
                    "@prefix rep: <http://www.openrdf.org/config/repository#>.\n" +
                    "@prefix sr: <http://www.openrdf.org/config/repository/sail#>.\n" +
                    "@prefix sail: <http://www.openrdf.org/config/sail#>.\n" +
                    "@prefix sail-luc: <http://www.openrdf.org/config/sail/lucene#>.\n" +
                    "@prefix lmdb: <http://rdf4j.org/config/sail/lmdb#>.\n" +
                    "@prefix sb: <http://www.openrdf.org/config/sail/base#>.\n" +
                    "\n" +
                    "[] a rep:Repository ;\n" +
                    "    rep:repositoryID \"" + repoName + "\" ;\n" +
                    "    rdfs:label \"" + repoName + " LMDB store\" ;\n" +
                    "    rep:repositoryImpl [\n" +
                    "        rep:repositoryType \"openrdf:SailRepository\" ;\n" +
                    "        sr:sailImpl [\n" +
                    "            sail:sailType \"rdf4j:LmdbStore\" ;\n" +
                    "            sail:iterationCacheSyncThreshold \"10000\";\n" +
                    "            lmdb:tripleIndexes \"" + indexTypes + "\" ;\n" +
                    "            sb:defaultQueryEvaluationMode \"STANDARD\"\n" +
                    "        ]\n"
                    + "    ].\n";

            // TODO Index npa:hasFilterLiteral predicate too (see https://groups.google.com/g/rdf4j-users/c/epF4Af1jXGU):
            String createTextRepoQueryString =
                    "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" +
                    "@prefix rep: <http://www.openrdf.org/config/repository#>.\n" +
                    "@prefix sr: <http://www.openrdf.org/config/repository/sail#>.\n" +
                    "@prefix sail: <http://www.openrdf.org/config/sail#>.\n" +
                    "@prefix sail-luc: <http://www.openrdf.org/config/sail/lucene#>.\n" +
                    "@prefix lmdb: <http://rdf4j.org/config/sail/lmdb#>.\n" +
                    "@prefix sb: <http://www.openrdf.org/config/sail/base#>.\n" +
                    "\n"
                    + "[] a rep:Repository ;\n" +
                    "    rep:repositoryID \"" + repoName + "\" ;\n" +
                    "    rdfs:label \"" + repoName + " store\" ;\n" +
                    "    rep:repositoryImpl [\n" +
                    "        rep:repositoryType \"openrdf:SailRepository\" ;\n" +
                    "        sr:sailImpl [\n" +
                    "            sail:sailType \"openrdf:LuceneSail\" ;\n" +
                    "            sail-luc:indexDir \"index/\" ;\n" +
                    "            sail-luc:transactional false ;\n" +
                    "            sail:delegate [\n" +
                    "              sail:sailType \"rdf4j:LmdbStore\" ;\n" +
                    "              sail:iterationCacheSyncThreshold \"10000\";\n" +
                    "              lmdb:tripleIndexes \"" + indexTypes + "\" ;\n" +
                    "              sb:defaultQueryEvaluationMode \"STANDARD\"\n" +
                    "            ]\n" +
                    "        ]\n" +
                    "    ].";

            String createRepoQueryString = createRegularRepoQueryString;
            if (repoName.startsWith("text")) {
                createRepoQueryString = createTextRepoQueryString;
            }

            HttpUriRequest createRepoRequest = RequestBuilder.put().setUri(endpointBase + "repositories/" + repoName).addHeader("Content-Type", "text/turtle").setEntity(new StringEntity(createRepoQueryString)).build();

            HttpResponse response = httpclient.execute(createRepoRequest);
            int statusCode = response.getStatusLine().getStatusCode();
            if (statusCode == 409) {
                //log.info("Already exists.");
                getRepository(repoName).init();
            } else if (statusCode >= 200 && statusCode < 300) {
                //log.info("Successfully created.");
                initNewRepo(repoName);
            } else {
                log.info("Status code: {}", response.getStatusLine().getStatusCode());
                log.info(response.getStatusLine().getReasonPhrase());
                String handledResponse = new BasicResponseHandler().handleResponse(response);
                log.info("Response: {}", handledResponse);
            }
        } catch (IOException ex) {
            log.info("Could not create repo.", ex);
        }
    }

    /**
     * Sends shutdown signal to all repositories.
     */
    @GeneratedFlagForDependentElements
    public void shutdownRepositories() {
        for (Repository repo : repositories.values()) {
            if (repo != null && repo.isInitialized()) {
                repo.shutDown();
            }
        }
    }

    /**
     * Returns admin repo connection.
     *
     * @return repository connection to admin repository
     */
    @GeneratedFlagForDependentElements
    public RepositoryConnection getAdminRepoConnection() {
        return get().getRepoConnection(ADMIN_REPO);
    }

    private Set<String> cachedRepositoryNames = Set.of();
    private boolean repoNamesCacheValid = false;
    private final ReadWriteLock repoNamesCacheLock = new ReentrantReadWriteLock();

    /**
     * Returns set of all repository names.
     *
     * @return Repository name set
     */
    public Set<String> getRepositoryNames() {
        // See if the repository names are cached:
        final var readLock = repoNamesCacheLock.readLock();
        try {
            readLock.lock();
            if (repoNamesCacheValid) {
                return cachedRepositoryNames;
            }
        } finally {
            readLock.unlock();
        }

        // Not cached, get from server:
        final var writeLock = repoNamesCacheLock.writeLock();
        try {
            writeLock.lock();
            // Check again if another thread has already updated the cache:
            if (repoNamesCacheValid) {
                return cachedRepositoryNames;
            }
            Map<String, Boolean> repositoryNames = null;
            try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
                HttpResponse resp = httpclient.execute(RequestBuilder.get()
                        .setUri(endpointBase + "/repositories")
                        .addHeader("Content-Type", "text/csv")
                        .build());
                BufferedReader reader = new BufferedReader(new InputStreamReader(resp.getEntity().getContent()));
                int code = resp.getStatusLine().getStatusCode();
                if (code < 200 || code >= 300) return null;
                repositoryNames = new HashMap<>();
                int lineCount = 0;
                while (true) {
                    String line = reader.readLine();
                    if (line == null) break;
                    if (lineCount > 0) {
                        String repoName = line.split(",")[1];
                        repositoryNames.put(repoName, true);
                    }
                    lineCount = lineCount + 1;
                }
            } catch (IOException ex) {
                log.info("Could not get repository names.", ex);
                return null;
            }
            cachedRepositoryNames = repositoryNames.keySet();
            repoNamesCacheValid = true;
            return cachedRepositoryNames;
        } finally {
            writeLock.unlock();
        }
    }

    /**
     * Invalidates the repository names cache. Call this method when a repository is created or deleted.
     */
    private void invalidateRepositoryNamesCache() {
        final var writeLock = repoNamesCacheLock.writeLock();
        try {
            writeLock.lock();
            repoNamesCacheValid = false;
        } finally {
            writeLock.unlock();
        }
    }

    @GeneratedFlagForDependentElements
    private void initNewRepo(String repoName) {
        String repoInitId = new Random().nextLong() + "";
        getRepository(repoName).init();
        if (!repoName.equals("empty")) {
            RepositoryConnection conn = getRepoConnection(repoName);
            try (conn) {
                // Full isolation, just in case.
                conn.begin(IsolationLevels.SERIALIZABLE);
                conn.add(NPA.THIS_REPO, NPA.HAS_REPO_INIT_ID, vf.createLiteral(repoInitId), NPA.GRAPH);
                if (tracksNanopubCountAndChecksum(repoName)) {
                    conn.add(NPA.THIS_REPO, NPA.HAS_NANOPUB_COUNT, vf.createLiteral(0L), NPA.GRAPH);
                    conn.add(NPA.THIS_REPO, NPA.HAS_NANOPUB_CHECKSUM, vf.createLiteral(NanopubUtils.INIT_CHECKSUM), NPA.GRAPH);
                }
                if (repoName.startsWith("pubkey_") || repoName.startsWith("type_")) {
                    String h = repoName.replaceFirst("^[^_]+_", "");
                    conn.add(NPA.THIS_REPO, NPA.HAS_COVERAGE_ITEM, Utils.getObjectForHash(h), NPA.GRAPH);
                    conn.add(NPA.THIS_REPO, NPA.HAS_COVERAGE_HASH, vf.createLiteral(h), NPA.GRAPH);
                    conn.add(NPA.THIS_REPO, NPA.HAS_COVERAGE_FILTER, vf.createLiteral("_" + repoName), NPA.GRAPH);
                }
                conn.commit();
            }
            // Refresh repository names cache
            invalidateRepositoryNamesCache();
        }
    }

    /**
     * Whether the given repo participates in the cumulative nanopub-count / XOR-checksum
     * tracking. Repos that maintain ad-hoc content (last30d expires entries; trust and
     * spaces hold derived state, not raw nanopubs) skip the {@code npa:hasNanopubCount}
     * and {@code npa:hasNanopubChecksum} initial triples — leaving them at {@code 0} and
     * the empty-XOR placeholder forever would just be misleading.
     */
    private static boolean tracksNanopubCountAndChecksum(String repoName) {
        return !repoName.equals("last30d")
                && !repoName.equals("trust")
                && !repoName.equals("spaces");
    }

}

1	package com.knowledgepixels.query;
2
3	import org.apache.commons.exec.environment.EnvironmentUtils;
4	import org.apache.http.HttpResponse;
5	import org.apache.http.client.methods.HttpUriRequest;
6	import org.apache.http.client.methods.RequestBuilder;
7	import org.apache.http.entity.StringEntity;
8	import org.apache.http.impl.client.BasicResponseHandler;
9	import org.apache.http.impl.client.CloseableHttpClient;
10	import org.apache.http.impl.client.HttpClients;
11	import org.eclipse.rdf4j.common.transaction.IsolationLevels;
12	import org.eclipse.rdf4j.model.ValueFactory;
13	import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
14	import org.eclipse.rdf4j.repository.Repository;
15	import org.eclipse.rdf4j.repository.RepositoryConnection;
16	import org.eclipse.rdf4j.repository.http.HTTPRepository;
17	import org.nanopub.NanopubUtils;
18	import org.nanopub.vocabulary.NPA;
19	import org.slf4j.Logger;
20	import org.slf4j.LoggerFactory;
21
22	import java.io.BufferedReader;
23	import java.io.IOException;
24	import java.io.InputStreamReader;
25	import java.util.*;
26	import java.util.Map.Entry;
27	import java.util.concurrent.locks.ReadWriteLock;
28	import java.util.concurrent.locks.ReentrantReadWriteLock;
29
30	/**
31	* Class to access the database in the form of triple stores.
32	*/
33	public class TripleStore {
34
35	/**
36	* Name of the admin graph.
37	*/
38	public static final String ADMIN_REPO = "admin";
39
40	private static ValueFactory vf = SimpleValueFactory.getInstance();	6✔
41
42	private static final Logger log = LoggerFactory.getLogger(TripleStore.class);	12✔
43
44	private final Map<String, Repository> repositories = new LinkedHashMap<>();	15✔
45
46	private String endpointBase = null;	9✔
47	private String endpointType = null;	9✔
48
49	private static TripleStore tripleStoreInstance;
50
51	/**
52	* Returns singleton triple store instance.
53	*
54	* @return Triple store instance
55	*/
56	public static TripleStore get() {
57	if (tripleStoreInstance == null) {	6!
58	try {
59	tripleStoreInstance = new TripleStore();	×
60	} catch (IOException ex) {	×
61	log.info("Could not init TripleStore. ", ex);	×
62	}	×
63	}
64	return tripleStoreInstance;	×
65	}
66
67	private TripleStore() throws IOException {	6✔
68	Map<String, String> env = EnvironmentUtils.getProcEnvironment();	6✔
69	endpointBase = env.get("ENDPOINT_BASE");	18✔
70	log.info("Endpoint base: {}", endpointBase);	15✔
71	endpointType = env.get("ENDPOINT_TYPE");	18✔
72
73	getRepository("empty"); // Make sure empty repo exists	×
74	}	×
75
76	private final CloseableHttpClient httpclient = HttpClients.createDefault();	9✔
77
78	@GeneratedFlagForDependentElements
79	Repository getRepository(String name) {
80	synchronized (this) {
81	while (repositories.size() > 100) {
82	Entry<String, Repository> e = repositories.entrySet().iterator().next();
83	repositories.remove(e.getKey());
84	log.info("Shutting down repo: {}", e.getKey());
85	e.getValue().shutDown();
86	log.info("Shutdown complete");
87	}
88	if (repositories.containsKey(name)) {
89	// Move to the end of the list:
90	Repository repo = repositories.remove(name);
91	repositories.put(name, repo);
92	} else {
93	Repository repository = null;
94	if (endpointType == null \|\| endpointType.equals("rdf4j")) {
95	HTTPRepository hr = new HTTPRepository(endpointBase + "repositories/" + name);
96	hr.setHttpClient(httpclient);
97	repository = hr;
98	// } else if (endpointType.equals("virtuoso")) {
99	// repository = new VirtuosoRepository(endpointBase + name, username, password);
100	} else {
101	throw new RuntimeException("Unknown repository type: " + endpointType);
102	}
103	repositories.put(name, repository);
104	createRepo(name);
105	getRepoConnection(name).close();
106	}
107	return repositories.get(name);
108	}
109	}
110
111	/**
112	* Return the repository connection for the given repository name.
113	*
114	* @param name repository name
115	* @return repository connection
116	*/
117	@GeneratedFlagForDependentElements
118	public RepositoryConnection getRepoConnection(String name) {
119	Repository repo = getRepository(name);
120	if (repo == null) {
121	return null;
122	}
123	return repo.getConnection();
124	}
125
126	@GeneratedFlagForDependentElements
127	private void createRepo(String repoName) {
128	if (!repoName.equals(ADMIN_REPO)) {
129	getRepository(ADMIN_REPO); // make sure admin repo is loaded first
130	}
131	try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
132	//log.info("Trying to creating repo " + name);
133
134	// TODO new syntax somehow doesn't work for the Lucene case:
135
136	// String createRegularRepoQueryString = "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n"
137	// + "@prefix config: <tag:rdf4j.org,2023:config/>.\n"
138	// + "[] a config:Repository ;\n"
139	// + " config:rep.id \"" + name + "\" ;\n"
140	// + " rdfs:label \"" + name + " native store\" ;\n"
141	// + " config:rep.impl [\n"
142	// + " config:rep.type \"openrdf:SailRepository\" ;\n"
143	// + " config:sail.impl [\n"
144	// + " config:sail.type \"openrdf:NativeStore\" ;\n"
145	// + " config:sail.iterationCacheSyncThreshold \"10000\";\n"
146	// + " config:native.tripleIndexes \"spoc,posc,ospc,opsc,psoc,sopc,spoc,cpos,cosp,cops,cpso,csop\" ;\n"
147	// + " config:sail.defaultQueryEvaluationMode \"STANDARD\"\n"
148	// + " ]\n"
149	// + " ].";
150	// String createTextRepoQueryString = "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n"
151	// + "@prefix config: <tag:rdf4j.org,2023:config/>.\n"
152	// + "[] a config:Repository ;\n"
153	// + " config:rep.id \"" + name + "\" ;\n"
154	// + " rdfs:label \"" + name + " native store\" ;\n"
155	// + " config:rep.impl [\n"
156	// + " config:rep.type \"openrdf:SailRepository\" ;\n"
157	// + " config:sail.impl [\n"
158	// + " config:sail.type \"openrdf:LuceneSail\" ;\n"
159	// + " config:sail.lucene.indexDir \"index/\" ;\n"
160	// + " config:delegate [\n"
161	// + " config:rep.type \"openrdf:SailRepository\" ;\n"
162	// + " config:sail.impl [\n"
163	// + " config:sail.type \"openrdf:NativeStore\" ;\n"
164	// + " config:sail.iterationCacheSyncThreshold \"10000\";\n"
165	// + " config:native.tripleIndexes \"spoc,posc,ospc,opsc,psoc,sopc,spoc,cpos,cosp,cops,cpso,csop\" ;\n"
166	// + " config:sail.defaultQueryEvaluationMode \"STANDARD\"\n"
167	// + " ]\n"
168	// + " ]\n"
169	// + " ]\n"
170	// + " ].";
171
172	String indexTypes = "spoc,posc,ospc,cspo,cpos,cosp";
173	if (repoName.startsWith("meta") \|\| repoName.startsWith("text")) {
174	indexTypes = "spoc,posc,ospc";
175	}
176
177	String createRegularRepoQueryString =
178	"@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" +
179	"@prefix rep: <http://www.openrdf.org/config/repository#>.\n" +
180	"@prefix sr: <http://www.openrdf.org/config/repository/sail#>.\n" +
181	"@prefix sail: <http://www.openrdf.org/config/sail#>.\n" +
182	"@prefix sail-luc: <http://www.openrdf.org/config/sail/lucene#>.\n" +
183	"@prefix lmdb: <http://rdf4j.org/config/sail/lmdb#>.\n" +
184	"@prefix sb: <http://www.openrdf.org/config/sail/base#>.\n" +
185	"\n" +
186	"[] a rep:Repository ;\n" +
187	" rep:repositoryID \"" + repoName + "\" ;\n" +
188	" rdfs:label \"" + repoName + " LMDB store\" ;\n" +
189	" rep:repositoryImpl [\n" +
190	" rep:repositoryType \"openrdf:SailRepository\" ;\n" +
191	" sr:sailImpl [\n" +
192	" sail:sailType \"rdf4j:LmdbStore\" ;\n" +
193	" sail:iterationCacheSyncThreshold \"10000\";\n" +
194	" lmdb:tripleIndexes \"" + indexTypes + "\" ;\n" +
195	" sb:defaultQueryEvaluationMode \"STANDARD\"\n" +
196	" ]\n"
197	+ " ].\n";
198
199	// TODO Index npa:hasFilterLiteral predicate too (see https://groups.google.com/g/rdf4j-users/c/epF4Af1jXGU):
200	String createTextRepoQueryString =
201	"@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" +
202	"@prefix rep: <http://www.openrdf.org/config/repository#>.\n" +
203	"@prefix sr: <http://www.openrdf.org/config/repository/sail#>.\n" +
204	"@prefix sail: <http://www.openrdf.org/config/sail#>.\n" +
205	"@prefix sail-luc: <http://www.openrdf.org/config/sail/lucene#>.\n" +
206	"@prefix lmdb: <http://rdf4j.org/config/sail/lmdb#>.\n" +
207	"@prefix sb: <http://www.openrdf.org/config/sail/base#>.\n" +
208	"\n"
209	+ "[] a rep:Repository ;\n" +
210	" rep:repositoryID \"" + repoName + "\" ;\n" +
211	" rdfs:label \"" + repoName + " store\" ;\n" +
212	" rep:repositoryImpl [\n" +
213	" rep:repositoryType \"openrdf:SailRepository\" ;\n" +
214	" sr:sailImpl [\n" +
215	" sail:sailType \"openrdf:LuceneSail\" ;\n" +
216	" sail-luc:indexDir \"index/\" ;\n" +
217	" sail-luc:transactional false ;\n" +
218	" sail:delegate [\n" +
219	" sail:sailType \"rdf4j:LmdbStore\" ;\n" +
220	" sail:iterationCacheSyncThreshold \"10000\";\n" +
221	" lmdb:tripleIndexes \"" + indexTypes + "\" ;\n" +
222	" sb:defaultQueryEvaluationMode \"STANDARD\"\n" +
223	" ]\n" +
224	" ]\n" +
225	" ].";
226
227	String createRepoQueryString = createRegularRepoQueryString;
228	if (repoName.startsWith("text")) {
229	createRepoQueryString = createTextRepoQueryString;
230	}
231
232	HttpUriRequest createRepoRequest = RequestBuilder.put().setUri(endpointBase + "repositories/" + repoName).addHeader("Content-Type", "text/turtle").setEntity(new StringEntity(createRepoQueryString)).build();
233
234	HttpResponse response = httpclient.execute(createRepoRequest);
235	int statusCode = response.getStatusLine().getStatusCode();
236	if (statusCode == 409) {
237	//log.info("Already exists.");
238	getRepository(repoName).init();
239	} else if (statusCode >= 200 && statusCode < 300) {
240	//log.info("Successfully created.");
241	initNewRepo(repoName);
242	} else {
243	log.info("Status code: {}", response.getStatusLine().getStatusCode());
244	log.info(response.getStatusLine().getReasonPhrase());
245	String handledResponse = new BasicResponseHandler().handleResponse(response);
246	log.info("Response: {}", handledResponse);
247	}
248	} catch (IOException ex) {
249	log.info("Could not create repo.", ex);
250	}
251	}
252
253	/**
254	* Sends shutdown signal to all repositories.
255	*/
256	@GeneratedFlagForDependentElements
257	public void shutdownRepositories() {
258	for (Repository repo : repositories.values()) {
259	if (repo != null && repo.isInitialized()) {
260	repo.shutDown();
261	}
262	}
263	}
264
265	/**
266	* Returns admin repo connection.
267	*
268	* @return repository connection to admin repository
269	*/
270	@GeneratedFlagForDependentElements
271	public RepositoryConnection getAdminRepoConnection() {
272	return get().getRepoConnection(ADMIN_REPO);
273	}
274
275	private Set<String> cachedRepositoryNames = Set.of();	9✔
276	private boolean repoNamesCacheValid = false;	9✔
277	private final ReadWriteLock repoNamesCacheLock = new ReentrantReadWriteLock();	15✔
278
279	/**
280	* Returns set of all repository names.
281	*
282	* @return Repository name set
283	*/
284	public Set<String> getRepositoryNames() {
285	// See if the repository names are cached:
286	final var readLock = repoNamesCacheLock.readLock();	12✔
287	try {
288	readLock.lock();	6✔
289	if (repoNamesCacheValid) {	9✔
290	return cachedRepositoryNames;	15✔
291	}
292	} finally {
293	readLock.unlock();	6✔
294	}
295
296	// Not cached, get from server:
297	final var writeLock = repoNamesCacheLock.writeLock();	12✔
298	try {
299	writeLock.lock();	6✔
300	// Check again if another thread has already updated the cache:
301	if (repoNamesCacheValid) {	9!
302	return cachedRepositoryNames;	×
303	}
304	Map<String, Boolean> repositoryNames = null;	6✔
305	try (CloseableHttpClient httpclient = HttpClients.createDefault()) {	6✔
306	HttpResponse resp = httpclient.execute(RequestBuilder.get()	21✔
307	.setUri(endpointBase + "/repositories")	9✔
308	.addHeader("Content-Type", "text/csv")	3✔
309	.build());	3✔
310	BufferedReader reader = new BufferedReader(new InputStreamReader(resp.getEntity().getContent()));	30✔
311	int code = resp.getStatusLine().getStatusCode();	12✔
312	if (code < 200 \|\| code >= 300) return null;	30!
313	repositoryNames = new HashMap<>();	12✔
314	int lineCount = 0;	6✔
315	while (true) {
316	String line = reader.readLine();	9✔
317	if (line == null) break;	9✔
318	if (lineCount > 0) {	6✔
319	String repoName = line.split(",")[1];	18✔
320	repositoryNames.put(repoName, true);	18✔
321	}
322	lineCount = lineCount + 1;	12✔
323	}	3✔
324	} catch (IOException ex) {	15!
325	log.info("Could not get repository names.", ex);	12✔
326	return null;	12✔
327	}	3✔
328	cachedRepositoryNames = repositoryNames.keySet();	12✔
329	repoNamesCacheValid = true;	9✔
330	return cachedRepositoryNames;	15✔
331	} finally {
332	writeLock.unlock();	6✔
333	}
334	}
335
336	/**
337	* Invalidates the repository names cache. Call this method when a repository is created or deleted.
338	*/
339	private void invalidateRepositoryNamesCache() {
340	final var writeLock = repoNamesCacheLock.writeLock();	×
341	try {
342	writeLock.lock();	×
343	repoNamesCacheValid = false;	×
344	} finally {
345	writeLock.unlock();	×
346	}
347	}	×
348
349	@GeneratedFlagForDependentElements
350	private void initNewRepo(String repoName) {
351	String repoInitId = new Random().nextLong() + "";
352	getRepository(repoName).init();
353	if (!repoName.equals("empty")) {
354	RepositoryConnection conn = getRepoConnection(repoName);
355	try (conn) {
356	// Full isolation, just in case.
357	conn.begin(IsolationLevels.SERIALIZABLE);
358	conn.add(NPA.THIS_REPO, NPA.HAS_REPO_INIT_ID, vf.createLiteral(repoInitId), NPA.GRAPH);
359	if (tracksNanopubCountAndChecksum(repoName)) {
360	conn.add(NPA.THIS_REPO, NPA.HAS_NANOPUB_COUNT, vf.createLiteral(0L), NPA.GRAPH);
361	conn.add(NPA.THIS_REPO, NPA.HAS_NANOPUB_CHECKSUM, vf.createLiteral(NanopubUtils.INIT_CHECKSUM), NPA.GRAPH);
362	}
363	if (repoName.startsWith("pubkey_") \|\| repoName.startsWith("type_")) {
364	String h = repoName.replaceFirst("^[^_]+_", "");
365	conn.add(NPA.THIS_REPO, NPA.HAS_COVERAGE_ITEM, Utils.getObjectForHash(h), NPA.GRAPH);
366	conn.add(NPA.THIS_REPO, NPA.HAS_COVERAGE_HASH, vf.createLiteral(h), NPA.GRAPH);
367	conn.add(NPA.THIS_REPO, NPA.HAS_COVERAGE_FILTER, vf.createLiteral("_" + repoName), NPA.GRAPH);
368	}
369	conn.commit();
370	}
371	// Refresh repository names cache
372	invalidateRepositoryNamesCache();
373	}
374	}
375
376	/**
377	* Whether the given repo participates in the cumulative nanopub-count / XOR-checksum
378	* tracking. Repos that maintain ad-hoc content (last30d expires entries; trust and
379	* spaces hold derived state, not raw nanopubs) skip the {@code npa:hasNanopubCount}
380	* and {@code npa:hasNanopubChecksum} initial triples — leaving them at {@code 0} and
381	* the empty-XOR placeholder forever would just be misleading.
382	*/
383	private static boolean tracksNanopubCountAndChecksum(String repoName) {
384	return !repoName.equals("last30d")	×
385	&& !repoName.equals("trust")	×
386	&& !repoName.equals("spaces");	×
387	}
388
389	}

knowledgepixels / nanopub-query / 24567987783

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous