evolvedbinary / elemental / 982

Committed 29 Apr 2025 08:34PM UTC coverage: 56.409% (+0.007%) from 56.402%

Build # 982

Build Type

push

circleci

Committed by

adamretter

Commit Message

[feature] Improve README.md badges

Run Details

28451 of 55847 branches covered (50.94%)

Branch coverage included in aggregate %.

77468 of 131924 relevant lines covered (58.72%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

39.06

/exist-core/src/main/java/org/exist/storage/lock/FileLock.java

/*
 * Elemental
 * Copyright (C) 2024, Evolved Binary Ltd
 *
 * admin@evolvedbinary.com
 * https://www.evolvedbinary.com | https://www.elemental.xyz
 *
 * Use of this software is governed by the Business Source License 1.1
 * included in the LICENSE file and at www.mariadb.com/bsl11.
 *
 * Change Date: 2028-04-27
 *
 * On the date above, in accordance with the Business Source License, use
 * of this software will be governed by the Apache License, Version 2.0.
 *
 * Additional Use Grant: Production use of the Licensed Work for a permitted
 * purpose. A Permitted Purpose is any purpose other than a Competing Use.
 * A Competing Use means making the Software available to others in a commercial
 * product or service that: substitutes for the Software; substitutes for any
 * other product or service we offer using the Software that exists as of the
 * date we make the Software available; or offers the same or substantially
 * similar functionality as the Software.
 *
 * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
 *       The original license header is included below.
 *
 * =====================================================================
 *
 * eXist-db Open Source Native XML Database
 * Copyright (C) 2001 The eXist-db Authors
 *
 * info@exist-db.org
 * http://www.exist-db.org
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
package org.exist.storage.lock;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.exist.storage.BrokerPool;
import org.exist.util.FileUtils;
import org.exist.util.ReadOnlyException;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.DateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.Properties;
import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.SYNC;
import static java.nio.file.StandardOpenOption.WRITE;

/**
 * Cooperative inter-process file locking, used to synchronize access to database files across
 * processes, i.e. across different Java VMs or separate database instances within one
 * VM. This is similar to the native file locks provided by Java NIO. However, the NIO
 * implementation has various problems. Among other things, we observed that locks
 * were not properly released on WinXP.
 * 
 * FileLock implements a cooperative approach. The class attempts to write a lock file
 * at the specified location. Every lock file stores 1) a magic word to make sure that the
 * file was really written by eXist, 2) a heartbeat timestamp. The procedure for acquiring the
 * lock in {@link #tryLock()} is as follows:
 * 
 * If a lock file does already exist in the specified location, we check its heartbeat timestamp.
 * If the timestamp is more than {@link #HEARTBEAT} milliseconds in the past, we assume
 * that the lock is stale and its owner process has died. The lock file is removed and we create
 * a new one.
 * 
 * If the heartbeat indicates that the owner process is still alive, the lock
 * attempt is aborted and {@link #tryLock()} returns false.
 * 
 * Otherwise, we create a new lock file and start a daemon thread to periodically update
 * the lock file's heart-beat value.
 * 
 * @author Wolfgang Meier
 * 
 */
public class FileLock {

    private final static Logger LOG = LogManager.getLogger(FileLock.class);

    /** The heartbeat period in milliseconds */
    private final long HEARTBEAT = 10100;

    /** Magic word to be written to the start of the lock file */
    private final static byte[] MAGIC =
        { 0x65, 0x58, 0x69, 0x73, 0x74, 0x2D, 0x64, 0x62 }; // "eXist-db"

    // TODO(AR) modify for Elemental release 8.0.0
//    private final static byte[] MAGIC =
//        { 0x45, 0x6C, 0x65, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C }; // "Elemental"

    /** BrokerPool provides access the SyncDaemon */
    private BrokerPool pool;

    /** The lock file */
    private Path lockFile;

    /** An open channel to the lock file */
    private SeekableByteChannel channel = null;

    /** Temporary buffer used for writing */
    private final ByteBuffer buf = ByteBuffer.allocate(MAGIC.length + 8);

    /** The time (in milliseconds) of the last heartbeat written to the lock file */
    private long lastHeartbeat = -1L;

    public FileLock(final BrokerPool pool, final Path path) {
        this.pool = pool;
        this.lockFile = path;
    }

    /**
     * Attempt to create the lock file and thus acquire a lock.
     * 
     * @return false if another process holds the lock
     * @throws ReadOnlyException if the lock file could not be created or saved
     * due to IO errors. The caller may want to switch to read-only mode.
     */
    public boolean tryLock() throws ReadOnlyException {
        int attempt = 0;
        while (Files.exists(lockFile)) {
            if (++attempt > 2) {
                return false;
            }
            
            try {
                read();
            } catch (final IOException e) {
                message("Failed to read lock file", null);
                e.printStackTrace();
            }
            
            //Check if there's a heart-beat. If not, remove the stale .lck file and try again
            if (checkHeartbeat()) {
                //There seems to be a heart-beat...
                //Sometimes Java does not properly delete files, so we may have an old
                //lock file from a previous db run, which has not timed out yet. We thus
                //give the db a second chance and wait for HEARTBEAT + 100 milliseconds
                //before we check the heart-beat a second time.
                synchronized (this) {
                    try {
                        message("Waiting a short time for the lock to be released...", null);
                        wait(HEARTBEAT + 100);
                    } catch (final InterruptedException e) {
                        //Nothing to do
                    }
                }
                
                try {
                    //Close the open channel, so it can be read again
                    if (channel.isOpen()) {
                        channel.close();
                    }
                    channel = null;
                } catch (final IOException e) {
                    //Nothing to do
                }
            }
        }



        try {
            this.lockFile = Files.createFile(lockFile);
        } catch (final IOException e) {
            throw new ReadOnlyException(message("Could not create lock file", e));
        }
        
        try {
            save();
        } catch (final IOException e) {
            throw new ReadOnlyException(message("Caught exception while trying to write lock file", e));
        }
        
        //Schedule the heart-beat for the file lock
        final Properties params = new Properties();
        params.put(FileLock.class.getName(), this);
        pool.getScheduler().createPeriodicJob(HEARTBEAT,
                new FileLockHeartBeat(lockFile.toAbsolutePath().toString()), -1, params);
        
        return true;
    }

    /**
     * Release the lock. Removes the lock file and closes all
     * open channels.
     */
    public void release() {
        try {
            if (channel.isOpen()) {
                channel.close();
            }
            channel = null;
            
        } catch (final Exception e) {
            message("Failed to close lock file", e);
        }

        if(Files.exists(lockFile)) {
            LOG.info("Deleting lock file: {}", lockFile.toAbsolutePath().toString());
            FileUtils.deleteQuietly(lockFile);
        }
    }

    /**
     * Returns the last heartbeat written to the lock file.
     * 
     * @return last heartbeat
     */
    public Date getLastHeartbeat() {
        return new Date(lastHeartbeat);
    }

    /**
     * Returns the lock file that represents the active lock held by
     * the FileLock.
     * 
     * @return lock file
     */
    public Path getFile() {
        return lockFile;
    }

    /**
     * Check if the lock has an active heartbeat, i.e. if it was updated
     * during the past {@link #HEARTBEAT} milliseconds.
     * 
     * @return true if there's an active heartbeat
     */
    private boolean checkHeartbeat() {
        final long now = System.currentTimeMillis();
        if (lastHeartbeat < 0 || now - lastHeartbeat > HEARTBEAT) {
            message("Found a stale lockfile. Trying to remove it: ", null);
            release();
            return false;
        }
        
        return true;
    }

    private void open() throws IOException {
        this.channel = Files.newByteChannel(lockFile, READ, WRITE, SYNC);
    }

    protected void save() throws IOException {
        try {
            if (channel == null) {
                open();
            }
            
            long now = System.currentTimeMillis();
            buf.clear();
            buf.put(MAGIC);
            buf.putLong(now);
            buf.flip();
            channel.position(0);
            channel.write(buf);
            //channel.force(true); //handled by SYNC on open option
            lastHeartbeat = now;
            
        } catch(final NullPointerException npe) {
            if(pool.isShuttingDown()) {
                LOG.info("No need to save FileLock, database is shutting down");
            } else {
                throw npe;
            }
        }
    }

    private void read() throws IOException {
        if (channel == null) {
            open();
        }
        
        channel.read(buf);
        buf.flip();
        if (buf.limit() < 16) {
            buf.clear();
            throw new IOException(message("Could not read file lock.", null));
        }
        
        final byte[] magic = new byte[8];
        buf.get(magic);
        if (!Arrays.equals(magic, MAGIC)) {
            throw new IOException(message("Bad signature in lock file. It does not seem to be an eXist lock file", null));
        }
        
        lastHeartbeat = buf.getLong();
        buf.clear();
        
        final DateFormat df = DateFormat.getDateInstance();
        message("File lock last access timestamp: " + df.format(getLastHeartbeat()), null);
    }

    protected String message(String message, final Exception e) {
        final StringBuilder str = new StringBuilder(message);
        str.append(' ').append(lockFile.toAbsolutePath().toString());
        if (e != null) {
            str.append(": ").append(e.getMessage());
        }
        
        message = str.toString();
        if (LOG.isInfoEnabled()) {
            LOG.info(message);
        }

        return message;
    }
}

1	/*
2	* Elemental
3	* Copyright (C) 2024, Evolved Binary Ltd
4	*
5	* admin@evolvedbinary.com
6	* https://www.evolvedbinary.com \| https://www.elemental.xyz
7	*
8	* Use of this software is governed by the Business Source License 1.1
9	* included in the LICENSE file and at www.mariadb.com/bsl11.
10	*
11	* Change Date: 2028-04-27
12	*
13	* On the date above, in accordance with the Business Source License, use
14	* of this software will be governed by the Apache License, Version 2.0.
15	*
16	* Additional Use Grant: Production use of the Licensed Work for a permitted
17	* purpose. A Permitted Purpose is any purpose other than a Competing Use.
18	* A Competing Use means making the Software available to others in a commercial
19	* product or service that: substitutes for the Software; substitutes for any
20	* other product or service we offer using the Software that exists as of the
21	* date we make the Software available; or offers the same or substantially
22	* similar functionality as the Software.
23	*
24	* NOTE: Parts of this file contain code from 'The eXist-db Authors'.
25	* The original license header is included below.
26	*
27	* =====================================================================
28	*
29	* eXist-db Open Source Native XML Database
30	* Copyright (C) 2001 The eXist-db Authors
31	*
32	* info@exist-db.org
33	* http://www.exist-db.org
34	*
35	* This library is free software; you can redistribute it and/or
36	* modify it under the terms of the GNU Lesser General Public
37	* License as published by the Free Software Foundation; either
38	* version 2.1 of the License, or (at your option) any later version.
39	*
40	* This library is distributed in the hope that it will be useful,
41	* but WITHOUT ANY WARRANTY; without even the implied warranty of
42	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
43	* Lesser General Public License for more details.
44	*
45	* You should have received a copy of the GNU Lesser General Public
46	* License along with this library; if not, write to the Free Software
47	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
48	*/
49	package org.exist.storage.lock;
50
51	import org.apache.logging.log4j.LogManager;
52	import org.apache.logging.log4j.Logger;
53	import org.exist.storage.BrokerPool;
54	import org.exist.util.FileUtils;
55	import org.exist.util.ReadOnlyException;
56
57	import java.io.IOException;
58	import java.nio.ByteBuffer;
59	import java.nio.channels.SeekableByteChannel;
60	import java.nio.file.Files;
61	import java.nio.file.Path;
62	import java.text.DateFormat;
63	import java.util.Arrays;
64	import java.util.Date;
65	import java.util.Properties;
66	import static java.nio.file.StandardOpenOption.READ;
67	import static java.nio.file.StandardOpenOption.SYNC;
68	import static java.nio.file.StandardOpenOption.WRITE;
69
70	/**
71	* Cooperative inter-process file locking, used to synchronize access to database files across
72	* processes, i.e. across different Java VMs or separate database instances within one
73	* VM. This is similar to the native file locks provided by Java NIO. However, the NIO
74	* implementation has various problems. Among other things, we observed that locks
75	* were not properly released on WinXP.
76	*
77	* FileLock implements a cooperative approach. The class attempts to write a lock file
78	* at the specified location. Every lock file stores 1) a magic word to make sure that the
79	* file was really written by eXist, 2) a heartbeat timestamp. The procedure for acquiring the
80	* lock in {@link #tryLock()} is as follows:
81	*
82	* If a lock file does already exist in the specified location, we check its heartbeat timestamp.
83	* If the timestamp is more than {@link #HEARTBEAT} milliseconds in the past, we assume
84	* that the lock is stale and its owner process has died. The lock file is removed and we create
85	* a new one.
86	*
87	* If the heartbeat indicates that the owner process is still alive, the lock
88	* attempt is aborted and {@link #tryLock()} returns false.
89	*
90	* Otherwise, we create a new lock file and start a daemon thread to periodically update
91	* the lock file's heart-beat value.
92	*
93	* @author Wolfgang Meier
94	*
95	*/
96	public class FileLock {
97
98	private final static Logger LOG = LogManager.getLogger(FileLock.class);	1✔
99
100	/** The heartbeat period in milliseconds */
101	private final long HEARTBEAT = 10100;	1✔
102
103	/** Magic word to be written to the start of the lock file */
104	private final static byte[] MAGIC =	1✔
105	{ 0x65, 0x58, 0x69, 0x73, 0x74, 0x2D, 0x64, 0x62 }; // "eXist-db"	1✔
106
107	// TODO(AR) modify for Elemental release 8.0.0
108	// private final static byte[] MAGIC =
109	// { 0x45, 0x6C, 0x65, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C }; // "Elemental"
110
111	/** BrokerPool provides access the SyncDaemon */
112	private BrokerPool pool;
113
114	/** The lock file */
115	private Path lockFile;
116
117	/** An open channel to the lock file */
118	private SeekableByteChannel channel = null;	1✔
119
120	/** Temporary buffer used for writing */
121	private final ByteBuffer buf = ByteBuffer.allocate(MAGIC.length + 8);	1✔
122
123	/** The time (in milliseconds) of the last heartbeat written to the lock file */
124	private long lastHeartbeat = -1L;	1✔
125
126	public FileLock(final BrokerPool pool, final Path path) {	1✔
127	this.pool = pool;	1✔
128	this.lockFile = path;	1✔
129	}	1✔
130
131	/**
132	* Attempt to create the lock file and thus acquire a lock.
133	*
134	* @return false if another process holds the lock
135	* @throws ReadOnlyException if the lock file could not be created or saved
136	* due to IO errors. The caller may want to switch to read-only mode.
137	*/
138	public boolean tryLock() throws ReadOnlyException {
139	int attempt = 0;	1✔
140	while (Files.exists(lockFile)) {	1!
141	if (++attempt > 2) {	×
142	return false;	×
143	}
144
145	try {
146	read();	×
147	} catch (final IOException e) {	×
148	message("Failed to read lock file", null);	×
149	e.printStackTrace();	×
150	}
151
152	//Check if there's a heart-beat. If not, remove the stale .lck file and try again
153	if (checkHeartbeat()) {	×
154	//There seems to be a heart-beat...
155	//Sometimes Java does not properly delete files, so we may have an old
156	//lock file from a previous db run, which has not timed out yet. We thus
157	//give the db a second chance and wait for HEARTBEAT + 100 milliseconds
158	//before we check the heart-beat a second time.
159	synchronized (this) {	×
160	try {
161	message("Waiting a short time for the lock to be released...", null);	×
162	wait(HEARTBEAT + 100);	×
163	} catch (final InterruptedException e) {	×
164	//Nothing to do
165	}
166	}
167
168	try {
169	//Close the open channel, so it can be read again
170	if (channel.isOpen()) {	×
171	channel.close();	×
172	}
173	channel = null;	×
174	} catch (final IOException e) {	×
175	//Nothing to do
176	}
177	}
178	}
179
180
181
182	try {
183	this.lockFile = Files.createFile(lockFile);	1✔
184	} catch (final IOException e) {	1✔
185	throw new ReadOnlyException(message("Could not create lock file", e));	×
186	}
187
188	try {
189	save();	1✔
190	} catch (final IOException e) {	1✔
191	throw new ReadOnlyException(message("Caught exception while trying to write lock file", e));	×
192	}
193
194	//Schedule the heart-beat for the file lock
195	final Properties params = new Properties();	1✔
196	params.put(FileLock.class.getName(), this);	1✔
197	pool.getScheduler().createPeriodicJob(HEARTBEAT,	1✔
198	new FileLockHeartBeat(lockFile.toAbsolutePath().toString()), -1, params);	1✔
199
200	return true;	1✔
201	}
202
203	/**
204	* Release the lock. Removes the lock file and closes all
205	* open channels.
206	*/
207	public void release() {
208	try {
209	if (channel.isOpen()) {	1!
210	channel.close();	1✔
211	}
212	channel = null;	1✔
213
214	} catch (final Exception e) {	1✔
215	message("Failed to close lock file", e);	×
216	}
217
218	if(Files.exists(lockFile)) {	1!
219	LOG.info("Deleting lock file: {}", lockFile.toAbsolutePath().toString());	1✔
220	FileUtils.deleteQuietly(lockFile);	1✔
221	}
222	}	1✔
223
224	/**
225	* Returns the last heartbeat written to the lock file.
226	*
227	* @return last heartbeat
228	*/
229	public Date getLastHeartbeat() {
230	return new Date(lastHeartbeat);	×
231	}
232
233	/**
234	* Returns the lock file that represents the active lock held by
235	* the FileLock.
236	*
237	* @return lock file
238	*/
239	public Path getFile() {
240	return lockFile;	1✔
241	}
242
243	/**
244	* Check if the lock has an active heartbeat, i.e. if it was updated
245	* during the past {@link #HEARTBEAT} milliseconds.
246	*
247	* @return true if there's an active heartbeat
248	*/
249	private boolean checkHeartbeat() {
250	final long now = System.currentTimeMillis();	×
251	if (lastHeartbeat < 0 \|\| now - lastHeartbeat > HEARTBEAT) {	×
252	message("Found a stale lockfile. Trying to remove it: ", null);	×
253	release();	×
254	return false;	×
255	}
256
257	return true;	×
258	}
259
260	private void open() throws IOException {
261	this.channel = Files.newByteChannel(lockFile, READ, WRITE, SYNC);	1✔
262	}	1✔
263
264	protected void save() throws IOException {
265	try {
266	if (channel == null) {	1✔
267	open();	1✔
268	}
269
270	long now = System.currentTimeMillis();	1✔
271	buf.clear();	1✔
272	buf.put(MAGIC);	1✔
273	buf.putLong(now);	1✔
274	buf.flip();	1✔
275	channel.position(0);	1✔
276	channel.write(buf);	1✔
277	//channel.force(true); //handled by SYNC on open option
278	lastHeartbeat = now;	1✔
279
280	} catch(final NullPointerException npe) {	1✔
281	if(pool.isShuttingDown()) {	×
282	LOG.info("No need to save FileLock, database is shutting down");	×
283	} else {	×
284	throw npe;	×
285	}
286	}
287	}	1✔
288
289	private void read() throws IOException {
290	if (channel == null) {	×
291	open();	×
292	}
293
294	channel.read(buf);	×
295	buf.flip();	×
296	if (buf.limit() < 16) {	×
297	buf.clear();	×
298	throw new IOException(message("Could not read file lock.", null));	×
299	}
300
301	final byte[] magic = new byte[8];	×
302	buf.get(magic);	×
303	if (!Arrays.equals(magic, MAGIC)) {	×
304	throw new IOException(message("Bad signature in lock file. It does not seem to be an eXist lock file", null));	×
305	}
306
307	lastHeartbeat = buf.getLong();	×
308	buf.clear();	×
309
310	final DateFormat df = DateFormat.getDateInstance();	×
311	message("File lock last access timestamp: " + df.format(getLastHeartbeat()), null);	×
312	}	×
313
314	protected String message(String message, final Exception e) {
315	final StringBuilder str = new StringBuilder(message);	×
316	str.append(' ').append(lockFile.toAbsolutePath().toString());	×
317	if (e != null) {	×
318	str.append(": ").append(e.getMessage());	×
319	}
320
321	message = str.toString();	×
322	if (LOG.isInfoEnabled()) {	×
323	LOG.info(message);	×
324	}
325
326	return message;	×
327	}
328	}

evolvedbinary / elemental / 982

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous