evolvedbinary / elemental / 982

Committed 29 Apr 2025 08:34PM UTC coverage: 56.409% (+0.007%) from 56.402%

Build # 982

Build Type

push

circleci

Committed by

adamretter

Commit Message

[feature] Improve README.md badges

Run Details

28451 of 55847 branches covered (50.94%)

Branch coverage included in aggregate %.

77468 of 131924 relevant lines covered (58.72%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.25

/exist-core/src/main/java/org/exist/storage/XQueryPool.java

/*
 * Elemental
 * Copyright (C) 2024, Evolved Binary Ltd
 *
 * admin@evolvedbinary.com
 * https://www.evolvedbinary.com | https://www.elemental.xyz
 *
 * Use of this software is governed by the Business Source License 1.1
 * included in the LICENSE file and at www.mariadb.com/bsl11.
 *
 * Change Date: 2028-04-27
 *
 * On the date above, in accordance with the Business Source License, use
 * of this software will be governed by the Apache License, Version 2.0.
 *
 * Additional Use Grant: Production use of the Licensed Work for a permitted
 * purpose. A Permitted Purpose is any purpose other than a Competing Use.
 * A Competing Use means making the Software available to others in a commercial
 * product or service that: substitutes for the Software; substitutes for any
 * other product or service we offer using the Software that exists as of the
 * date we make the Software available; or offers the same or substantially
 * similar functionality as the Software.
 */
package org.exist.storage;

import java.text.NumberFormat;
import java.util.ArrayDeque;
import java.util.Deque;

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import net.jcip.annotations.ThreadSafe;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.exist.security.Permission;
import org.exist.security.PermissionDeniedException;
import org.exist.source.DBSource;
import org.exist.source.Source;
import org.exist.util.Configuration;
import org.exist.util.Holder;
import org.exist.xquery.*;

/**
 * Global pool for compiled XQuery expressions.
 *
 * Expressions are stored and retrieved from the pool by comparing the
 * {@link org.exist.source.Source} objects from which they were created.
 *
 * For each XQuery, a maximum of {@link #DEFAULT_MAX_QUERY_STACK_SIZE} compiled
 * expressions are kept in the pool.
 *
 * @author <a href="mailto:adam@evolvedbinary.com">Adam Retter</a>
 */
@ThreadSafe
public class XQueryPool implements BrokerPoolService {

    private static final Logger LOG = LogManager.getLogger(XQueryPool.class);

    public static final String CONFIGURATION_ELEMENT_NAME = "query-pool";
    public static final String MAX_STACK_SIZE_ATTRIBUTE = "max-stack-size";
    public static final String POOL_SIZE_ATTTRIBUTE = "size";

    public static final String PROPERTY_MAX_STACK_SIZE = "db-connection.query-pool.max-stack-size";
    public static final String PROPERTY_POOL_SIZE = "db-connection.query-pool.size";

    private static final int DEFAULT_MAX_POOL_SIZE = 128;
    private static final int DEFAULT_MAX_QUERY_STACK_SIZE = 64;

    private int maxPoolSize = DEFAULT_MAX_POOL_SIZE;
    private int maxQueryStackSize = DEFAULT_MAX_QUERY_STACK_SIZE;

    /**
     * Source -> Deque of compiled Queries
     */
    private Cache<Source, Deque<CompiledXQuery>> cache;

    @Override
    public void configure(final Configuration configuration) {
        final Integer maxStSz = (Integer) configuration.getProperty(PROPERTY_MAX_STACK_SIZE);
        final Integer maxPoolSz = (Integer) configuration.getProperty(PROPERTY_POOL_SIZE);
        final NumberFormat nf = NumberFormat.getNumberInstance();

        if (maxPoolSz != null) {
            this.maxPoolSize = maxPoolSz;
        } else {
            this.maxPoolSize = DEFAULT_MAX_POOL_SIZE;
        }

        if (maxStSz != null) {
            this.maxQueryStackSize = maxStSz;
        } else {
            this.maxQueryStackSize = DEFAULT_MAX_QUERY_STACK_SIZE;
        }

        this.cache = Caffeine.newBuilder()
                .maximumSize(maxPoolSize)
                .build();

        LOG.info("QueryPool: size = {}; maxQueryStackSize = {}", nf.format(maxPoolSize), nf.format(maxQueryStackSize));
    }

    /**
     * Returns a compiled XQuery to the XQuery pool.
     *
     * @param source The source of the compiled XQuery.
     * @param compiledXQuery The compiled XQuery to add to the XQuery pool.
     */
    public void returnCompiledXQuery(final Source source, final CompiledXQuery compiledXQuery) {
        if (compiledXQuery == null) {
            return;
        }

        cache.asMap().compute(source, (key, value) -> {
            final Deque<CompiledXQuery> deque;
            if (value != null) {
                deque = value;
            } else {
                deque = new ArrayDeque<>(maxQueryStackSize);
            }

            deque.offerFirst(compiledXQuery);

            return deque;
        });
    }

    /**
     * Borrows a compiled XQuery from the XQuery pool.
     *
     * @param broker A database broker.
     * @param source The source identifying the XQuery to borrow.
     *
     * @return The compiled XQuery identified by the source, or null if
     *     there is no valid compiled representation in the XQuery pool.
     *
     * @throws PermissionDeniedException if the caller does not have execute
     *     permission for the compiled XQuery.
     */
    public CompiledXQuery borrowCompiledXQuery(final DBBroker broker, final Source source)
            throws PermissionDeniedException {
        if (broker == null || source == null) {
            return null;
        }

        // this will be set to non-null if we can borrow a query... allows us to escape the lamba, see https://github.com/ben-manes/caffeine/issues/192#issuecomment-337365618
        final Holder<CompiledXQuery> borrowedCompiledQuery = new Holder<>();

        // get (compute by checking validity) the stack of compiled XQuerys for the source
        final Deque<CompiledXQuery> deque = cache.asMap().computeIfPresent(source, (key, value) -> {
            final CompiledXQuery firstCompiledXQuery = value.pollFirst();
            if (firstCompiledXQuery == null) {
                // deque is empty, returning null will remove the entry from the cache
                return null;
            }

            if (!isCompiledQueryValid(firstCompiledXQuery)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("{} is invalid, removing from XQuery Pool...", source.pathOrShortIdentifier());
                }

                // query is invalid, returning null will remove the entry from the cache
                return null;
            }

            // escape the result from the lambda
            borrowedCompiledQuery.value = firstCompiledXQuery;

            // query is ok, preserve the tail of the deque
            return value;
        });

        if (deque == null) {
            return null;
        }

        //check execution permission
        if (source instanceof DBSource) {
            ((DBSource) source).validate(Permission.EXECUTE);
        }

        return borrowedCompiledQuery.value;
    }

    /**
     * Determines if a compiled XQuery is still valid.
     *
     * @param broker the database broker
     * @param source the source of the query
     * @param compiledXQuery the compiled query
     *
     * @return true if the compiled query is still valid, false otherwise.
     */
    private static boolean isCompiledQueryValid(final CompiledXQuery compiledXQuery) {
        final Source cachedSource = compiledXQuery.getSource();
        final Source.Validity validity = cachedSource.isValid();

        if (validity == Source.Validity.INVALID) {
            return false;    // returning false will remove the entry from the cache
        }

        // the compiled query is no longer valid if one of the imported
        // modules may have changed
        return compiledXQuery.isValid();
    }

    /**
     * Removes all entries from the XQuery Pool.
     */
    public void clear() {
        cache.invalidateAll();
    }
}

1	/*
2	* Elemental
3	* Copyright (C) 2024, Evolved Binary Ltd
4	*
5	* admin@evolvedbinary.com
6	* https://www.evolvedbinary.com \| https://www.elemental.xyz
7	*
8	* Use of this software is governed by the Business Source License 1.1
9	* included in the LICENSE file and at www.mariadb.com/bsl11.
10	*
11	* Change Date: 2028-04-27
12	*
13	* On the date above, in accordance with the Business Source License, use
14	* of this software will be governed by the Apache License, Version 2.0.
15	*
16	* Additional Use Grant: Production use of the Licensed Work for a permitted
17	* purpose. A Permitted Purpose is any purpose other than a Competing Use.
18	* A Competing Use means making the Software available to others in a commercial
19	* product or service that: substitutes for the Software; substitutes for any
20	* other product or service we offer using the Software that exists as of the
21	* date we make the Software available; or offers the same or substantially
22	* similar functionality as the Software.
23	*/
24	package org.exist.storage;
25
26	import java.text.NumberFormat;
27	import java.util.ArrayDeque;
28	import java.util.Deque;
29
30	import com.github.benmanes.caffeine.cache.Cache;
31	import com.github.benmanes.caffeine.cache.Caffeine;
32	import net.jcip.annotations.ThreadSafe;
33	import org.apache.logging.log4j.LogManager;
34	import org.apache.logging.log4j.Logger;
35	import org.exist.security.Permission;
36	import org.exist.security.PermissionDeniedException;
37	import org.exist.source.DBSource;
38	import org.exist.source.Source;
39	import org.exist.util.Configuration;
40	import org.exist.util.Holder;
41	import org.exist.xquery.*;
42
43	/**
44	* Global pool for compiled XQuery expressions.
45	*
46	* Expressions are stored and retrieved from the pool by comparing the
47	* {@link org.exist.source.Source} objects from which they were created.
48	*
49	* For each XQuery, a maximum of {@link #DEFAULT_MAX_QUERY_STACK_SIZE} compiled
50	* expressions are kept in the pool.
51	*
52	* @author <a href="mailto:adam@evolvedbinary.com">Adam Retter</a>
53	*/
54	@ThreadSafe
55	public class XQueryPool implements BrokerPoolService {	1✔
56
57	private static final Logger LOG = LogManager.getLogger(XQueryPool.class);	1✔
58
59	public static final String CONFIGURATION_ELEMENT_NAME = "query-pool";
60	public static final String MAX_STACK_SIZE_ATTRIBUTE = "max-stack-size";
61	public static final String POOL_SIZE_ATTTRIBUTE = "size";
62
63	public static final String PROPERTY_MAX_STACK_SIZE = "db-connection.query-pool.max-stack-size";
64	public static final String PROPERTY_POOL_SIZE = "db-connection.query-pool.size";
65
66	private static final int DEFAULT_MAX_POOL_SIZE = 128;
67	private static final int DEFAULT_MAX_QUERY_STACK_SIZE = 64;	1✔
68
69	private int maxPoolSize = DEFAULT_MAX_POOL_SIZE;	1✔
70	private int maxQueryStackSize = DEFAULT_MAX_QUERY_STACK_SIZE;	1✔
71
72	/**
73	* Source -> Deque of compiled Queries
74	*/
75	private Cache<Source, Deque<CompiledXQuery>> cache;
76
77	@Override
78	public void configure(final Configuration configuration) {
79	final Integer maxStSz = (Integer) configuration.getProperty(PROPERTY_MAX_STACK_SIZE);	1✔
80	final Integer maxPoolSz = (Integer) configuration.getProperty(PROPERTY_POOL_SIZE);	1✔
81	final NumberFormat nf = NumberFormat.getNumberInstance();	1✔
82
83	if (maxPoolSz != null) {	1✔
84	this.maxPoolSize = maxPoolSz;	1✔
85	} else {	1✔
86	this.maxPoolSize = DEFAULT_MAX_POOL_SIZE;	1✔
87	}
88
89	if (maxStSz != null) {	1✔
90	this.maxQueryStackSize = maxStSz;	1✔
91	} else {	1✔
92	this.maxQueryStackSize = DEFAULT_MAX_QUERY_STACK_SIZE;	1✔
93	}
94
95	this.cache = Caffeine.newBuilder()	1✔
96	.maximumSize(maxPoolSize)	1✔
97	.build();	1✔
98
99	LOG.info("QueryPool: size = {}; maxQueryStackSize = {}", nf.format(maxPoolSize), nf.format(maxQueryStackSize));	1✔
100	}	1✔
101
102	/**
103	* Returns a compiled XQuery to the XQuery pool.
104	*
105	* @param source The source of the compiled XQuery.
106	* @param compiledXQuery The compiled XQuery to add to the XQuery pool.
107	*/
108	public void returnCompiledXQuery(final Source source, final CompiledXQuery compiledXQuery) {
109	if (compiledXQuery == null) {	1!
110	return;	×
111	}
112
113	cache.asMap().compute(source, (key, value) -> {	1✔
114	final Deque<CompiledXQuery> deque;
115	if (value != null) {	1✔
116	deque = value;	1✔
117	} else {	1✔
118	deque = new ArrayDeque<>(maxQueryStackSize);	1✔
119	}
120
121	deque.offerFirst(compiledXQuery);	1✔
122
123	return deque;	1✔
124	});
125	}	1✔
126
127	/**
128	* Borrows a compiled XQuery from the XQuery pool.
129	*
130	* @param broker A database broker.
131	* @param source The source identifying the XQuery to borrow.
132	*
133	* @return The compiled XQuery identified by the source, or null if
134	* there is no valid compiled representation in the XQuery pool.
135	*
136	* @throws PermissionDeniedException if the caller does not have execute
137	* permission for the compiled XQuery.
138	*/
139	public CompiledXQuery borrowCompiledXQuery(final DBBroker broker, final Source source)
140	throws PermissionDeniedException {
141	if (broker == null \|\| source == null) {	1!
142	return null;	×
143	}
144
145	// this will be set to non-null if we can borrow a query... allows us to escape the lamba, see https://github.com/ben-manes/caffeine/issues/192#issuecomment-337365618
146	final Holder<CompiledXQuery> borrowedCompiledQuery = new Holder<>();	1✔
147
148	// get (compute by checking validity) the stack of compiled XQuerys for the source
149	final Deque<CompiledXQuery> deque = cache.asMap().computeIfPresent(source, (key, value) -> {	1✔
150	final CompiledXQuery firstCompiledXQuery = value.pollFirst();	1✔
151	if (firstCompiledXQuery == null) {	1✔
152	// deque is empty, returning null will remove the entry from the cache
153	return null;	1✔
154	}
155
156	if (!isCompiledQueryValid(firstCompiledXQuery)) {	1✔
157	if (LOG.isDebugEnabled()) {	1!
158	LOG.debug("{} is invalid, removing from XQuery Pool...", source.pathOrShortIdentifier());	×
159	}
160
161	// query is invalid, returning null will remove the entry from the cache
162	return null;	1✔
163	}
164
165	// escape the result from the lambda
166	borrowedCompiledQuery.value = firstCompiledXQuery;	1✔
167
168	// query is ok, preserve the tail of the deque
169	return value;	1✔
170	});
171
172	if (deque == null) {	1✔
173	return null;	1✔
174	}
175
176	//check execution permission
177	if (source instanceof DBSource) {	1✔
178	((DBSource) source).validate(Permission.EXECUTE);	1✔
179	}
180
181	return borrowedCompiledQuery.value;	1✔
182	}
183
184	/**
185	* Determines if a compiled XQuery is still valid.
186	*
187	* @param broker the database broker
188	* @param source the source of the query
189	* @param compiledXQuery the compiled query
190	*
191	* @return true if the compiled query is still valid, false otherwise.
192	*/
193	private static boolean isCompiledQueryValid(final CompiledXQuery compiledXQuery) {
194	final Source cachedSource = compiledXQuery.getSource();	1✔
195	final Source.Validity validity = cachedSource.isValid();	1✔
196
197	if (validity == Source.Validity.INVALID) {	1✔
198	return false; // returning false will remove the entry from the cache	1✔
199	}
200
201	// the compiled query is no longer valid if one of the imported
202	// modules may have changed
203	return compiledXQuery.isValid();	1✔
204	}
205
206	/**
207	* Removes all entries from the XQuery Pool.
208	*/
209	public void clear() {
210	cache.invalidateAll();	1✔
211	}	1✔
212	}

evolvedbinary / elemental / 982

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous