evolvedbinary / elemental / 982

Committed 29 Apr 2025 08:34PM UTC coverage: 56.409% (+0.007%) from 56.402%

Build # 982

Build Type

push

circleci

Committed by

adamretter

Commit Message

[feature] Improve README.md badges

Run Details

28451 of 55847 branches covered (50.94%)

Branch coverage included in aggregate %.

77468 of 131924 relevant lines covered (58.72%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

64.29

/exist-core/src/main/java/org/exist/util/io/TemporaryFileManager.java

/*
 * Elemental
 * Copyright (C) 2024, Evolved Binary Ltd
 *
 * admin@evolvedbinary.com
 * https://www.evolvedbinary.com | https://www.elemental.xyz
 *
 * Use of this software is governed by the Business Source License 1.1
 * included in the LICENSE file and at www.mariadb.com/bsl11.
 *
 * Change Date: 2028-04-27
 *
 * On the date above, in accordance with the Business Source License, use
 * of this software will be governed by the Apache License, Version 2.0.
 *
 * Additional Use Grant: Production use of the Licensed Work for a permitted
 * purpose. A Permitted Purpose is any purpose other than a Competing Use.
 * A Competing Use means making the Software available to others in a commercial
 * product or service that: substitutes for the Software; substitutes for any
 * other product or service we offer using the Software that exists as of the
 * date we make the Software available; or offers the same or substantially
 * similar functionality as the Software.
 */
package org.exist.util.io;

import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.exist.util.FileUtils;

import static java.nio.file.StandardOpenOption.CREATE_NEW;
import static java.nio.file.StandardOpenOption.DELETE_ON_CLOSE;
import static java.nio.file.StandardOpenOption.WRITE;

/**
 * Temporary File Manager.
 *
 * Provides temporary files for use by eXist-db and deals with cleaning them
 * up.
 *
 * Previously when returning a temporary file if it could not be deleted
 * (which often occurred on Microsoft Windows) we would add it to a queue
 * for reuse the next time a temporary file was required.
 *
 * On Microsoft Windows platforms this was shown to be unreliable. If the
 * temporary file had been Memory Mapped, there would be a lingering open file
 * handle which would only be closed when the GC reclaims the ByteBuffer
 * objects resulting from the mapping. This exhibited two problems:
 *     1. The previously memory mapped file could only be reused for further
 *         memory mapped I/O. Any traditional I/O or file system operations
 *         (e.g. copy, move, etc.) would result in a
 *         java.nio.file.FileSystemException.
 *     2. Keeping the previously memory mapped file in a queue, may result in
 *     strong indirect references to the ByteBuffer objects meaning that they
 *     will never be subject to GC, and therefore the file handles would never
 *     be released.
 * As such, we now never recycle temporary file objects. Instead we rely on the
 * GC to eventually close the file handles of any previously memory mapped files
 * and the Operating System to manage it's temporary file space.
 *
 * Relevant articles on the above described problems are:
 *     1. https://bugs.java.com/view_bug.do?bug_id=4715154
 *     2. https://bugs.openjdk.java.net/browse/JDK-8028683
 *     3. https://bugs.java.com/view_bug.do?bug_id=4724038
 *
 * @version 2.0
 *
 * @author <a href="mailto:adam.retter@googlemail.com">Adam Retter</a>
 */
public class TemporaryFileManager {

    private final static Logger LOG = LogManager.getLogger(TemporaryFileManager.class);

    private static final String FOLDER_PREFIX = "exist-db-temp-file-manager";
    private static final String FILE_PREFIX = "exist-db-temp";
    private static final String LOCK_FILENAME = FOLDER_PREFIX + ".lck";
    private final Path tmpFolder;
    private final FileChannel lockChannel;

    private static final TemporaryFileManager instance = new TemporaryFileManager();

    public static TemporaryFileManager getInstance() {
        return instance;
    }

    private TemporaryFileManager() {
        cleanupOldTempFolders();
        try {
            this.tmpFolder = Files.createTempDirectory(FOLDER_PREFIX + '-');
            this.lockChannel = FileChannel.open(tmpFolder.resolve(LOCK_FILENAME), CREATE_NEW, WRITE, DELETE_ON_CLOSE);
            lockChannel.lock();
            Runtime.getRuntime().addShutdownHook(new Thread(this::cleanUpTempFolders));
        } catch(final IOException ioe) {
            throw new RuntimeException("Unable to create temporary folder", ioe);
        }
        LOG.info("Temporary folder is: {}", tmpFolder);
    }

    private void cleanUpTempFolders() {
        LOG.info("Removing temporary folder is: {}", tmpFolder);
        try {
            lockChannel.close();  // will release the lock on the lock file, and the lock file should be deleted
            //try and remove our temporary folder
            FileUtils.deleteQuietly(tmpFolder);
        } catch(final IOException ioe) {
            LOG.error("Feiled to cleanup {}", tmpFolder, ioe);
        }
    }

    public final Path getTemporaryFile() throws IOException {

        // Be sure that the temp directory exists, create otherwise. #3826
        if (!Files.exists(tmpFolder)) {
            LOG.debug("Recreating {}", tmpFolder);
            Files.createDirectories(tmpFolder);
        }

        final Path tempFile = Files.createTempFile(tmpFolder, FILE_PREFIX + '-', ".tmp");

        /*
        add hook to JVM to delete the file on exit
        unfortunately this does not always work on all (e.g. Windows) platforms
         */
        tempFile.toFile().deleteOnExit();

        return tempFile;
    }

    public void returnTemporaryFile(final Path tempFile) {
        try {
            if (Files.deleteIfExists(tempFile)) {
                LOG.debug("Deleted temporary file: {}", tempFile);
            }
        } catch (final IOException e) {
            // this can often occur on Microsoft Windows (especially if the file was memory mapped!) :-/
            LOG.warn("Unable to delete temporary file: {} due to: {}", tempFile, e.getMessage());
        }
    }

    /**
     * Called at startup to attempt to cleanup
     * any left-over temporary folders
     * from the last time this was run
     */
    private void cleanupOldTempFolders() {
        final Path tmpDir = Paths.get(System.getProperty("java.io.tmpdir"));
        try {
            for (final Path dir : FileUtils.list(tmpDir, path -> Files.isDirectory(path) && path.startsWith(FOLDER_PREFIX))) {
                final Path lockPath = dir.resolve(LOCK_FILENAME);
                if (!Files.exists(lockPath)) {
                    // no lock file present, so not in use
                    FileUtils.deleteQuietly(dir);
                } else {
                    // there is a lock file present, we must determine if it is locked (by another eXist-db instance)
                    try (final FileChannel otherLockChannel = FileChannel.open(lockPath, WRITE)) {
                        if (otherLockChannel.tryLock() != null) {
                            // not locked... so we now have the lock
                            FileUtils.deleteQuietly(dir);
                        }
                    }
                    // will release the lock
                }
            }
        } catch (final IOException ioe) {
            LOG.warn("Unable to delete old temporary folders", ioe);
        }
    }
}

1	/*
2	* Elemental
3	* Copyright (C) 2024, Evolved Binary Ltd
4	*
5	* admin@evolvedbinary.com
6	* https://www.evolvedbinary.com \| https://www.elemental.xyz
7	*
8	* Use of this software is governed by the Business Source License 1.1
9	* included in the LICENSE file and at www.mariadb.com/bsl11.
10	*
11	* Change Date: 2028-04-27
12	*
13	* On the date above, in accordance with the Business Source License, use
14	* of this software will be governed by the Apache License, Version 2.0.
15	*
16	* Additional Use Grant: Production use of the Licensed Work for a permitted
17	* purpose. A Permitted Purpose is any purpose other than a Competing Use.
18	* A Competing Use means making the Software available to others in a commercial
19	* product or service that: substitutes for the Software; substitutes for any
20	* other product or service we offer using the Software that exists as of the
21	* date we make the Software available; or offers the same or substantially
22	* similar functionality as the Software.
23	*/
24	package org.exist.util.io;
25
26	import java.io.IOException;
27	import java.nio.channels.FileChannel;
28	import java.nio.file.Files;
29	import java.nio.file.Path;
30	import java.nio.file.Paths;
31
32	import org.apache.logging.log4j.LogManager;
33	import org.apache.logging.log4j.Logger;
34	import org.exist.util.FileUtils;
35
36	import static java.nio.file.StandardOpenOption.CREATE_NEW;
37	import static java.nio.file.StandardOpenOption.DELETE_ON_CLOSE;
38	import static java.nio.file.StandardOpenOption.WRITE;
39
40	/**
41	* Temporary File Manager.
42	*
43	* Provides temporary files for use by eXist-db and deals with cleaning them
44	* up.
45	*
46	* Previously when returning a temporary file if it could not be deleted
47	* (which often occurred on Microsoft Windows) we would add it to a queue
48	* for reuse the next time a temporary file was required.
49	*
50	* On Microsoft Windows platforms this was shown to be unreliable. If the
51	* temporary file had been Memory Mapped, there would be a lingering open file
52	* handle which would only be closed when the GC reclaims the ByteBuffer
53	* objects resulting from the mapping. This exhibited two problems:
54	* 1. The previously memory mapped file could only be reused for further
55	* memory mapped I/O. Any traditional I/O or file system operations
56	* (e.g. copy, move, etc.) would result in a
57	* java.nio.file.FileSystemException.
58	* 2. Keeping the previously memory mapped file in a queue, may result in
59	* strong indirect references to the ByteBuffer objects meaning that they
60	* will never be subject to GC, and therefore the file handles would never
61	* be released.
62	* As such, we now never recycle temporary file objects. Instead we rely on the
63	* GC to eventually close the file handles of any previously memory mapped files
64	* and the Operating System to manage it's temporary file space.
65	*
66	* Relevant articles on the above described problems are:
67	* 1. https://bugs.java.com/view_bug.do?bug_id=4715154
68	* 2. https://bugs.openjdk.java.net/browse/JDK-8028683
69	* 3. https://bugs.java.com/view_bug.do?bug_id=4724038
70	*
71	* @version 2.0
72	*
73	* @author <a href="mailto:adam.retter@googlemail.com">Adam Retter</a>
74	*/
75	public class TemporaryFileManager {
76
77	private final static Logger LOG = LogManager.getLogger(TemporaryFileManager.class);	1✔
78
79	private static final String FOLDER_PREFIX = "exist-db-temp-file-manager";
80	private static final String FILE_PREFIX = "exist-db-temp";
81	private static final String LOCK_FILENAME = FOLDER_PREFIX + ".lck";
82	private final Path tmpFolder;
83	private final FileChannel lockChannel;
84
85	private static final TemporaryFileManager instance = new TemporaryFileManager();	1✔
86
87	public static TemporaryFileManager getInstance() {
88	return instance;	1✔
89	}
90
91	private TemporaryFileManager() {	1✔
92	cleanupOldTempFolders();	1✔
93	try {
94	this.tmpFolder = Files.createTempDirectory(FOLDER_PREFIX + '-');	1✔
95	this.lockChannel = FileChannel.open(tmpFolder.resolve(LOCK_FILENAME), CREATE_NEW, WRITE, DELETE_ON_CLOSE);	1✔
96	lockChannel.lock();	1✔
97	Runtime.getRuntime().addShutdownHook(new Thread(this::cleanUpTempFolders));	1✔
98	} catch(final IOException ioe) {	1✔
99	throw new RuntimeException("Unable to create temporary folder", ioe);	×
100	}
101	LOG.info("Temporary folder is: {}", tmpFolder);	1✔
102	}	1✔
103
104	private void cleanUpTempFolders() {
105	LOG.info("Removing temporary folder is: {}", tmpFolder);	1✔
106	try {
107	lockChannel.close(); // will release the lock on the lock file, and the lock file should be deleted	1✔
108	//try and remove our temporary folder
109	FileUtils.deleteQuietly(tmpFolder);	1✔
110	} catch(final IOException ioe) {	1✔
111	LOG.error("Feiled to cleanup {}", tmpFolder, ioe);	×
112	}
113	}	1✔
114
115	public final Path getTemporaryFile() throws IOException {
116
117	// Be sure that the temp directory exists, create otherwise. #3826
118	if (!Files.exists(tmpFolder)) {	1!
119	LOG.debug("Recreating {}", tmpFolder);	×
120	Files.createDirectories(tmpFolder);	×
121	}
122
123	final Path tempFile = Files.createTempFile(tmpFolder, FILE_PREFIX + '-', ".tmp");	1✔
124
125	/*
126	add hook to JVM to delete the file on exit
127	unfortunately this does not always work on all (e.g. Windows) platforms
128	*/
129	tempFile.toFile().deleteOnExit();	1✔
130
131	return tempFile;	1✔
132	}
133
134	public void returnTemporaryFile(final Path tempFile) {
135	try {
136	if (Files.deleteIfExists(tempFile)) {	1✔
137	LOG.debug("Deleted temporary file: {}", tempFile);	1✔
138	}
139	} catch (final IOException e) {	1✔
140	// this can often occur on Microsoft Windows (especially if the file was memory mapped!) :-/
141	LOG.warn("Unable to delete temporary file: {} due to: {}", tempFile, e.getMessage());	×
142	}
143	}	1✔
144
145	/**
146	* Called at startup to attempt to cleanup
147	* any left-over temporary folders
148	* from the last time this was run
149	*/
150	private void cleanupOldTempFolders() {
151	final Path tmpDir = Paths.get(System.getProperty("java.io.tmpdir"));	1✔
152	try {
153	for (final Path dir : FileUtils.list(tmpDir, path -> Files.isDirectory(path) && path.startsWith(FOLDER_PREFIX))) {	1!
154	final Path lockPath = dir.resolve(LOCK_FILENAME);	×
155	if (!Files.exists(lockPath)) {	×
156	// no lock file present, so not in use
157	FileUtils.deleteQuietly(dir);	×
158	} else {	×
159	// there is a lock file present, we must determine if it is locked (by another eXist-db instance)
160	try (final FileChannel otherLockChannel = FileChannel.open(lockPath, WRITE)) {	×
161	if (otherLockChannel.tryLock() != null) {	×
162	// not locked... so we now have the lock
163	FileUtils.deleteQuietly(dir);	×
164	}
165	}
166	// will release the lock
167	}
168	}
169	} catch (final IOException ioe) {	1✔
170	LOG.warn("Unable to delete old temporary folders", ioe);	×
171	}
172	}	1✔
173	}

evolvedbinary / elemental / 982

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous