evolvedbinary / elemental / 982

Committed 29 Apr 2025 08:34PM UTC coverage: 56.409% (+0.007%) from 56.402%

Build # 982

Build Type

push

circleci

Committed by

adamretter

Commit Message

[feature] Improve README.md badges

Run Details

28451 of 55847 branches covered (50.94%)

Branch coverage included in aggregate %.

77468 of 131924 relevant lines covered (58.72%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.45

/exist-core/src/main/java/org/exist/indexing/IndexManager.java

/*
 * Elemental
 * Copyright (C) 2024, Evolved Binary Ltd
 *
 * admin@evolvedbinary.com
 * https://www.evolvedbinary.com | https://www.elemental.xyz
 *
 * Use of this software is governed by the Business Source License 1.1
 * included in the LICENSE file and at www.mariadb.com/bsl11.
 *
 * Change Date: 2028-04-27
 *
 * On the date above, in accordance with the Business Source License, use
 * of this software will be governed by the Apache License, Version 2.0.
 *
 * Additional Use Grant: Production use of the Licensed Work for a permitted
 * purpose. A Permitted Purpose is any purpose other than a Competing Use.
 * A Competing Use means making the Software available to others in a commercial
 * product or service that: substitutes for the Software; substitutes for any
 * other product or service we offer using the Software that exists as of the
 * date we make the Software available; or offers the same or substantially
 * similar functionality as the Software.
 *
 * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
 *       The original license header is included below.
 *
 * =====================================================================
 *
 * eXist-db Open Source Native XML Database
 * Copyright (C) 2001 The eXist-db Authors
 *
 * info@exist-db.org
 * http://www.exist-db.org
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
package org.exist.indexing;

import net.jcip.annotations.ThreadSafe;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.exist.backup.RawDataBackup;
import org.exist.storage.BrokerPool;
import org.exist.storage.BrokerPoolService;
import org.exist.storage.BrokerPoolServiceException;
import org.exist.storage.DBBroker;
import org.exist.storage.btree.DBException;
import org.exist.util.Configuration;
import org.exist.util.DatabaseConfigurationException;
import org.w3c.dom.Element;

import java.io.IOException;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;

/**
 * Manages all custom indexes registered with the database instance.
 */
@ThreadSafe
public class IndexManager implements BrokerPoolService {

    private final static Logger LOG = LogManager.getLogger(IndexManager.class);

    public static final String CONFIGURATION_ELEMENT_NAME = "modules";
    public static final String CONFIGURATION_MODULE_ELEMENT_NAME = "module";
    public static final String INDEXER_MODULES_CLASS_ATTRIBUTE = "class";
    public static final String INDEXER_MODULES_ID_ATTRIBUTE = "id";

    public final static String PROPERTY_INDEXER_MODULES = "indexer.modules";

    private final BrokerPool pool;

    private final Map<String, Index> indexers = new ConcurrentHashMap<>();

    private Configuration.IndexModuleConfig modConfigs[];
    private Path dataDir;

    private AtomicLong configurationTimestamp = new AtomicLong(System.currentTimeMillis());

    /**
     * @param pool   the BrokerPool representing the current database instance
     */
    public IndexManager(final BrokerPool pool) {
        this.pool = pool;
    }

    private void configurationChanged() {
        while (true) {
            long prev = configurationTimestamp.get();
            long now = System.currentTimeMillis();

            if(now > prev && configurationTimestamp.compareAndSet(prev, now)) {
                return;
            }
        }
    }

    /**
     * Get the timestamp of when the index manager's configuration was last
     * updated.
     *
     * @return the timestamp of when the index managers configuration was
     *      last updated.
     */
    public long getConfigurationTimestamp() {
        return configurationTimestamp.get();
    }

    @Override
    public void configure(final Configuration configuration) throws BrokerPoolServiceException {
        this.modConfigs = (Configuration.IndexModuleConfig[])
                configuration.getProperty(PROPERTY_INDEXER_MODULES);
        this.dataDir = (Path) configuration.getProperty(BrokerPool.PROPERTY_DATA_DIR);
        configurationChanged();
    }

    /**
     * Registers the indexes specified in
     * the global configuration object, i.e. in the :
     * <pre>
     * &lt;modules&gt;
     *   &lt;module id="foo" class="bar" foo1="bar1" ... /&gt;
     * &lt;/modules&gt;
     * </pre>
     * section of the configuration file.
     */
    @Override
    public void prepare(final BrokerPool brokerPool) throws BrokerPoolServiceException {
        try {
            if (modConfigs != null) {
                for (final Configuration.IndexModuleConfig modConfig : modConfigs) {
                    final String className = modConfig.className();
                    initIndex(pool, modConfig.id(), modConfig.config(), dataDir, className);
                }
            }
            // check if a structural index was configured. If not, create one based on default settings.
            AbstractIndex structural = (AbstractIndex) indexers.get(StructuralIndex.STRUCTURAL_INDEX_ID);
            if (structural == null) {
                structural = initIndex(pool, StructuralIndex.STRUCTURAL_INDEX_ID, null, dataDir, StructuralIndex.DEFAULT_CLASS);
                if (structural != null) {
                    structural.setName(StructuralIndex.STRUCTURAL_INDEX_ID);
                }
            }
        } catch(final DatabaseConfigurationException e) {
            throw new BrokerPoolServiceException(e);
        } finally {
            configurationChanged();
        }
    }

    private AbstractIndex initIndex(final BrokerPool pool, final String id, final Element config, final Path dataDir, final String className) throws DatabaseConfigurationException {
        try {
            final Class<?> clazz = Class.forName(className);
            if (!AbstractIndex.class.isAssignableFrom(clazz)) {
                throw new DatabaseConfigurationException("Class " + className + " does not implement " +
                        AbstractIndex.class.getName());
            }
            final AbstractIndex index = (AbstractIndex) clazz.newInstance();
            index.configure(pool, dataDir, config);
            index.open();
            indexers.put(id, index);
            if (LOG.isInfoEnabled()) {
                LOG.info("Registered index {} as {}", className, id);
            }
            return index;
        } catch (final ClassNotFoundException e) {
            LOG.warn("Class {} not found. Cannot configure index.", className);
        } catch (final IllegalAccessException | InstantiationException e) {
            LOG.warn("Exception while configuring index {}: {}", className, e.getMessage(), e);
        }
        return null;
    }

    public Index registerIndex(final Index index) throws DatabaseConfigurationException {
        index.open();
        indexers.put(index.getIndexId(), index);
        if (LOG.isInfoEnabled()) {
            LOG.info("Registered index {} as {}", index.getClass(), index.getIndexId());
        }

        configurationChanged();

        return index;
    }

    public void unregisterIndex(final Index index) throws DBException {
        indexers.remove(index.getIndexId(), index);
        index.close();
        if (LOG.isInfoEnabled()) {
            LOG.info("Unregistered index {} as {}", index.getClass(), index.getIndexId());
        }

        configurationChanged();
    }

    /**
     * Returns the {@link org.exist.storage.BrokerPool} on with this IndexManager operates.
     *
     * @return the broker pool
     */
    public BrokerPool getBrokerPool() {
        return pool;
    }

    /**
     * Returns an iterator over the registered indexes.
     *
     * @return the iterator
     */
    protected Iterator<Index> iterator() {
        return indexers.values().iterator();
    }

    /**
     * Returns the index registered with the provided ID.
     *
     * @param indexId the ID
     * @return the index
     */
    public synchronized Index getIndexById(final String indexId) {
        return indexers.values().stream()
                .filter(indexer -> indexer.getIndexId().equals(indexId))
                .findFirst()
                .orElse(null);
    }

    /**
     * Returns the index registered with the provided human-readable name.
     *
     * @param indexName the name
     * @return the index
     */
    public synchronized Index getIndexByName(final String indexName) {
        return indexers.get(indexName);
    }

    /**
     * Returns a set of IndexWorkers, one for each registered index. The
     * returned IndexWorkers are used by the DBBroker instances to perform the
     * actual indexing work.
     *
     * @return set of IndexWorkers
     */
    synchronized List<IndexWorker> getWorkers(final DBBroker broker) {
        return indexers.values().stream()
                .map(index -> index.getWorker(broker))
                .filter(Objects::nonNull)
                .collect(Collectors.toList());
    }

    /**
     * Shutdowns all registered indexes by calling {@link org.exist.indexing.Index#close()}
     * on them.
     *
     * @param systemBroker The broker that will perform the operation
     * @throws BrokerPoolServiceException in case of an error in the BrookerPoolService
     */
    @Override
    public void stopSystem(final DBBroker systemBroker) throws BrokerPoolServiceException {
        for (final Iterator<Index> i = iterator(); i.hasNext(); ) {
            final Index index = i.next();
            try {
                index.close();
            } catch(final DBException e) {
                throw new BrokerPoolServiceException(e);
            }
        }
    }

    /**
     * Call indexes to flush all data to disk.
     *
     * @throws DBException in case of an error
     */
    public void sync() throws DBException {
        for (final Iterator<Index> i = iterator(); i.hasNext(); ) {
            final Index index = i.next();
            index.sync();
        }
    }

    /**
     * Physically destroy the registered indexes by calling {@link org.exist.indexing.Index#remove()}
     * on them.
     *
     * @throws DBException in case of an error
     */
    public void removeIndexes() throws DBException {
        for (final Iterator<Index> i = iterator(); i.hasNext(); ) {
            final Index index = i.next();
            index.remove();
        }
    }

    /**
     * Reopens the registered index in case they have been closed by a previous operation
     * such as {@link org.exist.indexing.Index#close()} by calling {@link org.exist.indexing.Index#open()}
     * on them.
     *
     * @throws DatabaseConfigurationException in cse of an database configuration error
     */
    public void reopenIndexes() throws DatabaseConfigurationException {
        for (final Iterator<Index> i = iterator(); i.hasNext(); ) {
            final Index index = i.next();
            index.open();
        }
    }

    public void backupToArchive(final RawDataBackup backup) throws IOException {
        for (final Iterator<Index> i = iterator(); i.hasNext(); ) {
            final Index index = i.next();
            if (index instanceof RawBackupSupport) {
                ((RawBackupSupport) index).backupToArchive(backup);
            }
        }
    }
}

1	/*
2	* Elemental
3	* Copyright (C) 2024, Evolved Binary Ltd
4	*
5	* admin@evolvedbinary.com
6	* https://www.evolvedbinary.com \| https://www.elemental.xyz
7	*
8	* Use of this software is governed by the Business Source License 1.1
9	* included in the LICENSE file and at www.mariadb.com/bsl11.
10	*
11	* Change Date: 2028-04-27
12	*
13	* On the date above, in accordance with the Business Source License, use
14	* of this software will be governed by the Apache License, Version 2.0.
15	*
16	* Additional Use Grant: Production use of the Licensed Work for a permitted
17	* purpose. A Permitted Purpose is any purpose other than a Competing Use.
18	* A Competing Use means making the Software available to others in a commercial
19	* product or service that: substitutes for the Software; substitutes for any
20	* other product or service we offer using the Software that exists as of the
21	* date we make the Software available; or offers the same or substantially
22	* similar functionality as the Software.
23	*
24	* NOTE: Parts of this file contain code from 'The eXist-db Authors'.
25	* The original license header is included below.
26	*
27	* =====================================================================
28	*
29	* eXist-db Open Source Native XML Database
30	* Copyright (C) 2001 The eXist-db Authors
31	*
32	* info@exist-db.org
33	* http://www.exist-db.org
34	*
35	* This library is free software; you can redistribute it and/or
36	* modify it under the terms of the GNU Lesser General Public
37	* License as published by the Free Software Foundation; either
38	* version 2.1 of the License, or (at your option) any later version.
39	*
40	* This library is distributed in the hope that it will be useful,
41	* but WITHOUT ANY WARRANTY; without even the implied warranty of
42	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
43	* Lesser General Public License for more details.
44	*
45	* You should have received a copy of the GNU Lesser General Public
46	* License along with this library; if not, write to the Free Software
47	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
48	*/
49	package org.exist.indexing;
50
51	import net.jcip.annotations.ThreadSafe;
52	import org.apache.logging.log4j.LogManager;
53	import org.apache.logging.log4j.Logger;
54	import org.exist.backup.RawDataBackup;
55	import org.exist.storage.BrokerPool;
56	import org.exist.storage.BrokerPoolService;
57	import org.exist.storage.BrokerPoolServiceException;
58	import org.exist.storage.DBBroker;
59	import org.exist.storage.btree.DBException;
60	import org.exist.util.Configuration;
61	import org.exist.util.DatabaseConfigurationException;
62	import org.w3c.dom.Element;
63
64	import java.io.IOException;
65	import java.nio.file.Path;
66	import java.util.*;
67	import java.util.concurrent.ConcurrentHashMap;
68	import java.util.concurrent.atomic.AtomicLong;
69	import java.util.stream.Collectors;
70
71	/**
72	* Manages all custom indexes registered with the database instance.
73	*/
74	@ThreadSafe
75	public class IndexManager implements BrokerPoolService {
76
77	private final static Logger LOG = LogManager.getLogger(IndexManager.class);	1✔
78
79	public static final String CONFIGURATION_ELEMENT_NAME = "modules";
80	public static final String CONFIGURATION_MODULE_ELEMENT_NAME = "module";
81	public static final String INDEXER_MODULES_CLASS_ATTRIBUTE = "class";
82	public static final String INDEXER_MODULES_ID_ATTRIBUTE = "id";
83
84	public final static String PROPERTY_INDEXER_MODULES = "indexer.modules";	1✔
85
86	private final BrokerPool pool;
87
88	private final Map<String, Index> indexers = new ConcurrentHashMap<>();	1✔
89
90	private Configuration.IndexModuleConfig modConfigs[];
91	private Path dataDir;
92
93	private AtomicLong configurationTimestamp = new AtomicLong(System.currentTimeMillis());	1✔
94
95	/**
96	* @param pool the BrokerPool representing the current database instance
97	*/
98	public IndexManager(final BrokerPool pool) {	1✔
99	this.pool = pool;	1✔
100	}	1✔
101
102	private void configurationChanged() {
103	while (true) {
104	long prev = configurationTimestamp.get();	1✔
105	long now = System.currentTimeMillis();	1✔
106
107	if(now > prev && configurationTimestamp.compareAndSet(prev, now)) {	1!
108	return;	1✔
109	}
110	}
111	}
112
113	/**
114	* Get the timestamp of when the index manager's configuration was last
115	* updated.
116	*
117	* @return the timestamp of when the index managers configuration was
118	* last updated.
119	*/
120	public long getConfigurationTimestamp() {
121	return configurationTimestamp.get();	1✔
122	}
123
124	@Override
125	public void configure(final Configuration configuration) throws BrokerPoolServiceException {
126	this.modConfigs = (Configuration.IndexModuleConfig[])	1✔
127	configuration.getProperty(PROPERTY_INDEXER_MODULES);	1✔
128	this.dataDir = (Path) configuration.getProperty(BrokerPool.PROPERTY_DATA_DIR);	1✔
129	configurationChanged();	1✔
130	}	1✔
131
132	/**
133	* Registers the indexes specified in
134	* the global configuration object, i.e. in the :
135	* <pre>
136	* <modules>
137	* <module id="foo" class="bar" foo1="bar1" ... />
138	* </modules>
139	* </pre>
140	* section of the configuration file.
141	*/
142	@Override
143	public void prepare(final BrokerPool brokerPool) throws BrokerPoolServiceException {
144	try {
145	if (modConfigs != null) {	1✔
146	for (final Configuration.IndexModuleConfig modConfig : modConfigs) {	1✔
147	final String className = modConfig.className();	1✔
148	initIndex(pool, modConfig.id(), modConfig.config(), dataDir, className);	1✔
149	}
150	}
151	// check if a structural index was configured. If not, create one based on default settings.
152	AbstractIndex structural = (AbstractIndex) indexers.get(StructuralIndex.STRUCTURAL_INDEX_ID);	1✔
153	if (structural == null) {	1!
154	structural = initIndex(pool, StructuralIndex.STRUCTURAL_INDEX_ID, null, dataDir, StructuralIndex.DEFAULT_CLASS);	1✔
155	if (structural != null) {	1!
156	structural.setName(StructuralIndex.STRUCTURAL_INDEX_ID);	1✔
157	}
158	}
159	} catch(final DatabaseConfigurationException e) {	1✔
160	throw new BrokerPoolServiceException(e);	×
161	} finally {
162	configurationChanged();	1✔
163	}
164	}	1✔
165
166	private AbstractIndex initIndex(final BrokerPool pool, final String id, final Element config, final Path dataDir, final String className) throws DatabaseConfigurationException {
167	try {
168	final Class<?> clazz = Class.forName(className);	1✔
169	if (!AbstractIndex.class.isAssignableFrom(clazz)) {	1!
170	throw new DatabaseConfigurationException("Class " + className + " does not implement " +	×
171	AbstractIndex.class.getName());	×
172	}
173	final AbstractIndex index = (AbstractIndex) clazz.newInstance();	1✔
174	index.configure(pool, dataDir, config);	1✔
175	index.open();	1✔
176	indexers.put(id, index);	1✔
177	if (LOG.isInfoEnabled()) {	1!
178	LOG.info("Registered index {} as {}", className, id);	1✔
179	}
180	return index;	1✔
181	} catch (final ClassNotFoundException e) {	×
182	LOG.warn("Class {} not found. Cannot configure index.", className);	×
183	} catch (final IllegalAccessException \| InstantiationException e) {	×
184	LOG.warn("Exception while configuring index {}: {}", className, e.getMessage(), e);	×
185	}
186	return null;	×
187	}
188
189	public Index registerIndex(final Index index) throws DatabaseConfigurationException {
190	index.open();	1✔
191	indexers.put(index.getIndexId(), index);	1✔
192	if (LOG.isInfoEnabled()) {	1!
193	LOG.info("Registered index {} as {}", index.getClass(), index.getIndexId());	1✔
194	}
195
196	configurationChanged();	1✔
197
198	return index;	1✔
199	}
200
201	public void unregisterIndex(final Index index) throws DBException {
202	indexers.remove(index.getIndexId(), index);	1✔
203	index.close();	1✔
204	if (LOG.isInfoEnabled()) {	1!
205	LOG.info("Unregistered index {} as {}", index.getClass(), index.getIndexId());	1✔
206	}
207
208	configurationChanged();	1✔
209	}	1✔
210
211	/**
212	* Returns the {@link org.exist.storage.BrokerPool} on with this IndexManager operates.
213	*
214	* @return the broker pool
215	*/
216	public BrokerPool getBrokerPool() {
217	return pool;	×
218	}
219
220	/**
221	* Returns an iterator over the registered indexes.
222	*
223	* @return the iterator
224	*/
225	protected Iterator<Index> iterator() {
226	return indexers.values().iterator();	1✔
227	}
228
229	/**
230	* Returns the index registered with the provided ID.
231	*
232	* @param indexId the ID
233	* @return the index
234	*/
235	public synchronized Index getIndexById(final String indexId) {
236	return indexers.values().stream()	×
237	.filter(indexer -> indexer.getIndexId().equals(indexId))	×
238	.findFirst()	×
239	.orElse(null);	×
240	}
241
242	/**
243	* Returns the index registered with the provided human-readable name.
244	*
245	* @param indexName the name
246	* @return the index
247	*/
248	public synchronized Index getIndexByName(final String indexName) {
249	return indexers.get(indexName);	×
250	}
251
252	/**
253	* Returns a set of IndexWorkers, one for each registered index. The
254	* returned IndexWorkers are used by the DBBroker instances to perform the
255	* actual indexing work.
256	*
257	* @return set of IndexWorkers
258	*/
259	synchronized List<IndexWorker> getWorkers(final DBBroker broker) {
260	return indexers.values().stream()	1✔
261	.map(index -> index.getWorker(broker))	1✔
262	.filter(Objects::nonNull)	1✔
263	.collect(Collectors.toList());	1✔
264	}
265
266	/**
267	* Shutdowns all registered indexes by calling {@link org.exist.indexing.Index#close()}
268	* on them.
269	*
270	* @param systemBroker The broker that will perform the operation
271	* @throws BrokerPoolServiceException in case of an error in the BrookerPoolService
272	*/
273	@Override
274	public void stopSystem(final DBBroker systemBroker) throws BrokerPoolServiceException {
275	for (final Iterator<Index> i = iterator(); i.hasNext(); ) {	1✔
276	final Index index = i.next();	1✔
277	try {
278	index.close();	1✔
279	} catch(final DBException e) {	1✔
280	throw new BrokerPoolServiceException(e);	×
281	}
282	}
283	}	1✔
284
285	/**
286	* Call indexes to flush all data to disk.
287	*
288	* @throws DBException in case of an error
289	*/
290	public void sync() throws DBException {
291	for (final Iterator<Index> i = iterator(); i.hasNext(); ) {	1✔
292	final Index index = i.next();	1✔
293	index.sync();	1✔
294	}
295	}	1✔
296
297	/**
298	* Physically destroy the registered indexes by calling {@link org.exist.indexing.Index#remove()}
299	* on them.
300	*
301	* @throws DBException in case of an error
302	*/
303	public void removeIndexes() throws DBException {
304	for (final Iterator<Index> i = iterator(); i.hasNext(); ) {	1✔
305	final Index index = i.next();	1✔
306	index.remove();	1✔
307	}
308	}	1✔
309
310	/**
311	* Reopens the registered index in case they have been closed by a previous operation
312	* such as {@link org.exist.indexing.Index#close()} by calling {@link org.exist.indexing.Index#open()}
313	* on them.
314	*
315	* @throws DatabaseConfigurationException in cse of an database configuration error
316	*/
317	public void reopenIndexes() throws DatabaseConfigurationException {
318	for (final Iterator<Index> i = iterator(); i.hasNext(); ) {	1✔
319	final Index index = i.next();	1✔
320	index.open();	1✔
321	}
322	}	1✔
323
324	public void backupToArchive(final RawDataBackup backup) throws IOException {
325	for (final Iterator<Index> i = iterator(); i.hasNext(); ) {	1✔
326	final Index index = i.next();	1✔
327	if (index instanceof RawBackupSupport) {	1!
328	((RawBackupSupport) index).backupToArchive(backup);	1✔
329	}
330	}
331	}	1✔
332	}

evolvedbinary / elemental / 982

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous