7087

Committed 29 Sep 2025 04:54PM UTC coverage: 36.248% (-0.02%) from 36.264%

Build # 7087

Build Type

Pull #1328

github

Committed by

web-flow

Commit Message

Bump org.jboss.resteasy:resteasy-client from 6.2.12.Final to 7.0.0.Final

Bumps [org.jboss.resteasy:resteasy-client](https://github.com/resteasy/resteasy) from 6.2.12.Final to 7.0.0.Final.
- [Release notes](https://github.com/resteasy/resteasy/releases)
- [Commits](https://github.com/resteasy/resteasy/compare/v6.2.12.Final...v7.0.0.Final)

---
updated-dependencies:
- dependency-name: org.jboss.resteasy:resteasy-client
  dependency-version: 7.0.0.Final
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Pull Request Pull Request #1328: Bump org.jboss.resteasy:resteasy-client from 6.2.12.Final to 7.0.0.Final

Run Details

1908 of 5898 branches covered (32.35%)

Branch coverage included in aggregate %.

8967 of 24104 relevant lines covered (37.2%)

0.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

7.48

/SpiNNaker-nmpiserv/src/main/java/uk/ac/manchester/spinnaker/nmpi/jobmanager/OutputManagerImpl.java

/*
 * Copyright (c) 2014 The University of Manchester
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package uk.ac.manchester.spinnaker.nmpi.jobmanager;

import static java.lang.System.currentTimeMillis;
import static java.nio.file.Files.move;
import static java.nio.file.Files.probeContentType;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import static java.util.concurrent.Executors.newScheduledThreadPool;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static jakarta.ws.rs.core.Response.ok;
import static jakarta.ws.rs.core.Response.serverError;
import static jakarta.ws.rs.core.Response.status;
import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
import static org.slf4j.LoggerFactory.getLogger;
import static uk.ac.manchester.spinnaker.utils.ThreadUtils.waitfor;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;

import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import jakarta.ws.rs.WebApplicationException;
import jakarta.ws.rs.core.Response;

import org.slf4j.Logger;
import org.springframework.beans.factory.annotation.Value;

import uk.ac.manchester.spinnaker.nmpi.model.job.nmpi.DataItem;
import uk.ac.manchester.spinnaker.nmpi.rest.OutputManager;
import uk.ac.manchester.spinnaker.nmpi.rest.UnicoreFileClient;

/**
 * Service for managing Job output files.
 */
//TODO needs security; Role = OutputHandler
public class OutputManagerImpl implements OutputManager {
        /** Indicates that a file has been removed. */
        private static final String PURGED_FILE = ".purged_";

        /** The directory to store files in. */
        @Value("${results.directory}")
        private File resultsDirectory;

        /** The URL of the server. */
        private final URL baseServerUrl;

        /** The amount of time results should be kept, in milliseconds. */
        private long timeToKeepResults;

        /** Map of locks for files. */
        private final Map<File, LockToken> synchronizers = new HashMap<>();

        /** The logger. */
        private static final Logger logger = getLogger(OutputManagerImpl.class);

        /**
         * A lock token. Initially locked.
         */
        private static final class LockToken {
                /** True if the token is locked. */
                private boolean locked = true;

                /** True if the token is waiting for a lock. */
                private boolean waiting = false;

                /**
                 * Wait until the token is unlocked.
                 */
                private synchronized void waitForUnlock() {
                        waiting = true;

                        // Wait until unlocked
                        while (locked) {
                                waitfor(this);
                        }

                        // Now lock again
                        locked = true;
                        waiting = false;
                }

                /**
                 * Unlock the token.
                 *
                 * @return True if the token is waiting again.
                 */
                private synchronized boolean unlock() {
                        locked = false;
                        notifyAll();
                        return waiting;
                }
        }

        /**
         * A class to lock a job.
         */
        private class JobLock implements AutoCloseable {
                /** The directory being locked by this token. */
                private File dir;

                /**
                 * Create a new lock for a directory.
                 *
                 * @param dir
                 *            The directory to lock
                 */
                JobLock(final File dir) {
                        this.dir = dir;

                        LockToken lock;
                        synchronized (synchronizers) {
                                if (!synchronizers.containsKey(dir)) {
                                        // Constructed pre-locked
                                        synchronizers.put(dir, new LockToken());
                                        return;
                                }
                                lock = synchronizers.get(dir);
                        }

                        lock.waitForUnlock();
                }

                @Override
                public void close() {
                        synchronized (synchronizers) {
                                final var lock = synchronizers.get(dir);
                                if (!lock.unlock()) {
                                        synchronizers.remove(dir);
                                }
                        }
                }
        }

        /**
         * Instantiate the output manager.
         *
         * @param baseServerUrl
         *            The base URL of the overall service, used when generating
         *            internal URLs.
         */
        public OutputManagerImpl(final URL baseServerUrl) {
                this.baseServerUrl = baseServerUrl;
        }

        /**
         * Set the number of days after a job has finished to keep results.
         *
         * @param nDaysToKeepResults
         *            The number of days to keep the results
         */
        @Value("${results.purge.days}")
        void setPurgeTimeout(final long nDaysToKeepResults) {
                timeToKeepResults = MILLISECONDS.convert(nDaysToKeepResults, DAYS);
        }

        /** Periodic execution engine. */
        private final ScheduledExecutorService scheduler = newScheduledThreadPool(
                        1);

        /**
         * Arrange for old output to be purged once per day.
         */
        @PostConstruct
        private void initPurgeScheduler() {
                scheduler.scheduleAtFixedRate(this::removeOldFiles, 0, 1, DAYS);
        }

        /**
         * Stop the scheduler running jobs.
         */
        @PreDestroy
        private void stopPurgeScheduler() {
                scheduler.shutdown();
        }

        /**
         * Get the project directory for a given project.
         *
         * @param projectId
         *            The id of the project
         * @return The directory of the project
         */
        private File getProjectDirectory(final String projectId) {
                if (isNull(projectId) || projectId.isEmpty()
                                || projectId.endsWith("/")) {
                        throw new IllegalArgumentException("bad projectId");
                }
                final var name = new File(projectId).getName();
                if (name.equals(".") || name.equals("..") || name.isEmpty()) {
                        throw new IllegalArgumentException("bad projectId");
                }
                return new File(resultsDirectory, name);
        }

        /**
         * Adds outputs to be hosted for a given id, returning a matching list of
         * URLs on which the files are hosted.
         *
         * @param projectId
         *            The id of the project
         * @param id
         *            The id of the job
         * @param baseDirectory
         *            The root directory containing all the files
         * @param outputs
         *            The files to add
         * @return A list of DataItem instances for adding to the job
         * @throws IOException
         *            If anything goes wrong.
         */
        List<DataItem> addOutputs(final String projectId, final int id,
                        final File baseDirectory, final Collection<File> outputs)
                        throws IOException {
                if (isNull(outputs)) {
                        return null;
                }

                final var pId = new File(projectId).getName();
                final int pathStart = baseDirectory.getAbsolutePath().length();
                final var projectDirectory = getProjectDirectory(projectId);
                final var idDirectory = new File(projectDirectory, String.valueOf(id));

                try (var op = new JobLock(idDirectory)) {
                        final var outputData = new ArrayList<DataItem>();
                        for (final var output : outputs) {
                                if (!output.getAbsolutePath()
                                                .startsWith(baseDirectory.getAbsolutePath())) {
                                        throw new IOException("Output file " + output
                                                        + " is outside base directory " + baseDirectory);
                                }

                                var outputPath = output.getAbsolutePath()
                                                .substring(pathStart).replace('\\', '/');
                                if (outputPath.startsWith("/")) {
                                        outputPath = outputPath.substring(1);
                                }

                                final var newOutput = new File(idDirectory, outputPath);
                                newOutput.getParentFile().mkdirs();
                                if (newOutput.exists()) {
                                        if (!newOutput.delete()) {
                                                logger.warn("Could not delete existing file {};"
                                                                + " new file will not be used!", newOutput);
                                        } else {
                                                logger.warn("Overwriting existing file {}", newOutput);
                                        }
                                }
                                if (!newOutput.exists()) {
                                        move(output.toPath(), newOutput.toPath());
                                }
                                final var outputUrl = new URL(baseServerUrl,
                                                "output/" + pId + "/" + id + "/" + outputPath);
                                outputData.add(new DataItem(outputUrl.toExternalForm()));
                                logger.debug("New output {} mapped to {}",
                                                newOutput, outputUrl);
                        }

                        return outputData;
                }
        }

        /**
         * Get a file as a response to a query.
         *
         * @param idDirectory
         *            The directory of the project
         * @param filename
         *            The name of the file to be stored
         * @param download
         *            True if the content type should be set to guarantee that the
         *            file is downloaded, False to attempt to guess the content type
         * @return The response
         */
        private Response getResultFile(final File idDirectory,
                        final String filename, final boolean download) {
                final var resultFile = new File(idDirectory, filename);
                final var purgeFile = getPurgeFile(idDirectory);
                JobManager.checkFileIsInFolder(idDirectory, resultFile);

                try (var op = new JobLock(idDirectory)) {
                        if (purgeFile.exists()) {
                                logger.debug("{} was purged", idDirectory);
                                return status(NOT_FOUND).entity("Results from job "
                                                + idDirectory.getName() + " have been removed")
                                                .build();
                        }

                        if (!resultFile.canRead()) {
                                logger.debug("{} was not found", resultFile);
                                return status(NOT_FOUND).build();
                        }

                        try {
                                if (!download) {
                                        final var contentType =
                                                        probeContentType(resultFile.toPath());
                                        if (nonNull(contentType)) {
                                                logger.debug("File has content type {}", contentType);
                                                return ok(resultFile, contentType).build();
                                        }
                                }
                        } catch (final IOException e) {
                                logger.debug("Content type of {} could not be determined",
                                                resultFile, e);
                        }

                        return ok(resultFile).header("Content-Disposition",
                                        "attachment; filename=" + filename).build();
                }
        }

        /**
         * Get the file that marks a directory as purged.
         *
         * @param directory
         *            The directory to find the file in
         * @return The purge marker file
         */
        private File getPurgeFile(final File directory) {
                return new File(resultsDirectory, PURGED_FILE + directory.getName());
        }

        @Override
        public Response getResultFile(final String projectId, final int id,
                        final String filename, final boolean download) {
                logger.debug("Retrieving {} from {}/{}", filename, projectId, id);
                final var projectDirectory = getProjectDirectory(projectId);
                final var idDirectory = new File(projectDirectory, String.valueOf(id));
                return getResultFile(idDirectory, filename, download);
        }

        @Override
        public Response getResultFile(final int id, final String filename,
                        final boolean download) {
                logger.debug("Retrieving {} from {}", filename, id);
                final var idDirectory = getProjectDirectory(String.valueOf(id));
                return getResultFile(idDirectory, filename, download);
        }

        /**
         * Upload files in recursive subdirectories to UniCore.
         *
         * @param authHeader
         *            The authentication to use
         * @param directory
         *            The directory to start from
         * @param fileManager
         *            The UniCore client
         * @param storageId
         *            The id of the UniCore storage
         * @param filePath
         *            The path in the UniCore storage to upload to
         * @throws IOException
         *             If something goes wrong
         */
        private void recursivelyUploadFiles(final String authHeader,
                        final File directory,
                        final UnicoreFileClient fileManager, final String storageId,
                        final String filePath) throws IOException {
                final var files = directory.listFiles();
                if (isNull(files)) {
                        return;
                }
                for (final var file : files) {
                        if (file.getName().equals(".") || file.getName().equals("..")
                                        || file.getName().isEmpty()) {
                                continue;
                        }
                        final var uploadFileName = filePath + "/" + file.getName();
                        if (file.isDirectory()) {
                                recursivelyUploadFiles(authHeader, file, fileManager, storageId,
                                                uploadFileName);
                                continue;
                        }
                        if (!file.isFile()) {
                                continue;
                        }
                        try (var input = new FileInputStream(file)) {
                                fileManager.upload(authHeader, storageId, uploadFileName,
                                                input);
                        } catch (final WebApplicationException e) {
                                throw new IOException("Error uploading file to " + storageId
                                                + "/" + uploadFileName, e);
                        } catch (final FileNotFoundException e) {
                                // Ignore files which vanish.
                        }
                }
        }

        @Override
        public Response uploadResultsToHPCServer(final String projectId,
                        final int id, final String serverUrl, final String storageId,
                        final String filePath, final String userId, final String token) {
                final var idDirectory =
                                new File(getProjectDirectory(projectId), String.valueOf(id));
                if (!idDirectory.canRead()) {
                        logger.debug("{} was not found", idDirectory);
                        return status(NOT_FOUND).build();
                }

                try {
                        final var authHeader = "Bearer: " + token;
                        final var fileClient = UnicoreFileClient.createClient(serverUrl);
                        try (var op = new JobLock(idDirectory)) {
                                recursivelyUploadFiles(authHeader, idDirectory, fileClient,
                                                storageId, filePath.replaceAll("/+$", ""));
                        }
                } catch (final MalformedURLException e) {
                        logger.error("bad user-supplied URL", e);
                        return status(BAD_REQUEST)
                                        .entity("The URL specified was malformed").build();
                } catch (final Throwable e) {
                        logger.error("failure in upload", e);
                        return serverError()
                                        .entity("General error reading or uploading a file")
                                        .build();
                }

                return ok("ok").build();
        }

        /**
         * Recursively remove a directory.
         *
         * @param directory
         *            The directory to remove
         */
        private void removeDirectory(final File directory) {
                for (final var file : directory.listFiles()) {
                        if (file.isDirectory()) {
                                removeDirectory(file);
                        } else {
                                file.delete();
                        }
                }
                directory.delete();
        }

        /**
         * Remove files that are deemed to have expired.
         */
        private void removeOldFiles() {
                final long startTime = currentTimeMillis();
                for (final var projectDirectory : resultsDirectory.listFiles()) {
                        if (projectDirectory.isDirectory()
                                        && removeOldProjectDirectoryContents(startTime,
                                                        projectDirectory)) {
                                logger.info("No more outputs for project {}",
                                                projectDirectory.getName());
                                projectDirectory.delete();
                        }
                }
        }

        /**
         * Remove project contents that are deemed to have expired.
         *
         * @param startTime
         *            The current time being considered
         * @param projectDirectory
         *            The directory containing the project files
         * @return True if every job in the project has been removed
         */
        private boolean removeOldProjectDirectoryContents(final long startTime,
                        final File projectDirectory) {
                boolean allJobsRemoved = true;
                for (final var jobDirectory : projectDirectory.listFiles()) {
                        logger.debug("Determining whether to remove {} "
                                        + "which is {}ms old of {}", jobDirectory,
                                        startTime - jobDirectory.lastModified(),
                                        timeToKeepResults);
                        if (jobDirectory.isDirectory() && ((startTime
                                        - jobDirectory.lastModified()) > timeToKeepResults)) {
                                logger.info("Removing results for job {}",
                                                jobDirectory.getName());
                                try (var op = new JobLock(jobDirectory)) {
                                        removeDirectory(jobDirectory);
                                }

                                try (var purgedFileWriter =
                                                new PrintWriter(getPurgeFile(jobDirectory))) {
                                        purgedFileWriter.println(currentTimeMillis());
                                } catch (final IOException e) {
                                        logger.error("Error writing purge file", e);
                                }
                        } else {
                                allJobsRemoved = false;
                        }
                }
                return allJobsRemoved;
        }
}

1	/*
2	* Copyright (c) 2014 The University of Manchester
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* https://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16	package uk.ac.manchester.spinnaker.nmpi.jobmanager;
17
18	import static java.lang.System.currentTimeMillis;
19	import static java.nio.file.Files.move;
20	import static java.nio.file.Files.probeContentType;
21	import static java.util.Objects.isNull;
22	import static java.util.Objects.nonNull;
23	import static java.util.concurrent.Executors.newScheduledThreadPool;
24	import static java.util.concurrent.TimeUnit.DAYS;
25	import static java.util.concurrent.TimeUnit.MILLISECONDS;
26	import static jakarta.ws.rs.core.Response.ok;
27	import static jakarta.ws.rs.core.Response.serverError;
28	import static jakarta.ws.rs.core.Response.status;
29	import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
30	import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
31	import static org.slf4j.LoggerFactory.getLogger;
32	import static uk.ac.manchester.spinnaker.utils.ThreadUtils.waitfor;
33
34	import java.io.File;
35	import java.io.FileInputStream;
36	import java.io.FileNotFoundException;
37	import java.io.IOException;
38	import java.io.PrintWriter;
39	import java.net.MalformedURLException;
40	import java.net.URL;
41	import java.util.ArrayList;
42	import java.util.Collection;
43	import java.util.HashMap;
44	import java.util.List;
45	import java.util.Map;
46	import java.util.concurrent.ScheduledExecutorService;
47
48	import jakarta.annotation.PostConstruct;
49	import jakarta.annotation.PreDestroy;
50	import jakarta.ws.rs.WebApplicationException;
51	import jakarta.ws.rs.core.Response;
52
53	import org.slf4j.Logger;
54	import org.springframework.beans.factory.annotation.Value;
55
56	import uk.ac.manchester.spinnaker.nmpi.model.job.nmpi.DataItem;
57	import uk.ac.manchester.spinnaker.nmpi.rest.OutputManager;
58	import uk.ac.manchester.spinnaker.nmpi.rest.UnicoreFileClient;
59
60	/**
61	* Service for managing Job output files.
62	*/
63	//TODO needs security; Role = OutputHandler
64	public class OutputManagerImpl implements OutputManager {
65	/** Indicates that a file has been removed. */
66	private static final String PURGED_FILE = ".purged_";
67
68	/** The directory to store files in. */
69	@Value("${results.directory}")
70	private File resultsDirectory;
71
72	/** The URL of the server. */
73	private final URL baseServerUrl;
74
75	/** The amount of time results should be kept, in milliseconds. */
76	private long timeToKeepResults;
77
78	/** Map of locks for files. */
79	private final Map<File, LockToken> synchronizers = new HashMap<>();	2✔
80
81	/** The logger. */
82	private static final Logger logger = getLogger(OutputManagerImpl.class);	2✔
83
84	/**
85	* A lock token. Initially locked.
86	*/
87	private static final class LockToken {	×
88	/** True if the token is locked. */
89	private boolean locked = true;	×
90
91	/** True if the token is waiting for a lock. */
92	private boolean waiting = false;	×
93
94	/**
95	* Wait until the token is unlocked.
96	*/
97	private synchronized void waitForUnlock() {
98	waiting = true;	×
99
100	// Wait until unlocked
101	while (locked) {	×
102	waitfor(this);	×
103	}
104
105	// Now lock again
106	locked = true;	×
107	waiting = false;	×
108	}	×
109
110	/**
111	* Unlock the token.
112	*
113	* @return True if the token is waiting again.
114	*/
115	private synchronized boolean unlock() {
116	locked = false;	×
117	notifyAll();	×
118	return waiting;	×
119	}
120	}
121
122	/**
123	* A class to lock a job.
124	*/
125	private class JobLock implements AutoCloseable {
126	/** The directory being locked by this token. */
127	private File dir;
128
129	/**
130	* Create a new lock for a directory.
131	*
132	* @param dir
133	* The directory to lock
134	*/
135	JobLock(final File dir) {	×
136	this.dir = dir;	×
137
138	LockToken lock;
139	synchronized (synchronizers) {	×
140	if (!synchronizers.containsKey(dir)) {	×
141	// Constructed pre-locked
142	synchronizers.put(dir, new LockToken());	×
143	return;	×
144	}
145	lock = synchronizers.get(dir);	×
146	}	×
147
148	lock.waitForUnlock();	×
149	}	×
150
151	@Override
152	public void close() {
153	synchronized (synchronizers) {	×
154	final var lock = synchronizers.get(dir);	×
155	if (!lock.unlock()) {	×
156	synchronizers.remove(dir);	×
157	}
158	}	×
159	}	×
160	}
161
162	/**
163	* Instantiate the output manager.
164	*
165	* @param baseServerUrl
166	* The base URL of the overall service, used when generating
167	* internal URLs.
168	*/
169	public OutputManagerImpl(final URL baseServerUrl) {	2✔
170	this.baseServerUrl = baseServerUrl;	2✔
171	}	2✔
172
173	/**
174	* Set the number of days after a job has finished to keep results.
175	*
176	* @param nDaysToKeepResults
177	* The number of days to keep the results
178	*/
179	@Value("${results.purge.days}")
180	void setPurgeTimeout(final long nDaysToKeepResults) {
181	timeToKeepResults = MILLISECONDS.convert(nDaysToKeepResults, DAYS);	2✔
182	}	2✔
183
184	/** Periodic execution engine. */
185	private final ScheduledExecutorService scheduler = newScheduledThreadPool(	2✔
186	1);
187
188	/**
189	* Arrange for old output to be purged once per day.
190	*/
191	@PostConstruct
192	private void initPurgeScheduler() {
193	scheduler.scheduleAtFixedRate(this::removeOldFiles, 0, 1, DAYS);	2✔
194	}	2✔
195
196	/**
197	* Stop the scheduler running jobs.
198	*/
199	@PreDestroy
200	private void stopPurgeScheduler() {
201	scheduler.shutdown();	2✔
202	}	2✔
203
204	/**
205	* Get the project directory for a given project.
206	*
207	* @param projectId
208	* The id of the project
209	* @return The directory of the project
210	*/
211	private File getProjectDirectory(final String projectId) {
212	if (isNull(projectId) \|\| projectId.isEmpty()	×
213	\|\| projectId.endsWith("/")) {	×
214	throw new IllegalArgumentException("bad projectId");	×
215	}
216	final var name = new File(projectId).getName();	×
217	if (name.equals(".") \|\| name.equals("..") \|\| name.isEmpty()) {	×
218	throw new IllegalArgumentException("bad projectId");	×
219	}
220	return new File(resultsDirectory, name);	×
221	}
222
223	/**
224	* Adds outputs to be hosted for a given id, returning a matching list of
225	* URLs on which the files are hosted.
226	*
227	* @param projectId
228	* The id of the project
229	* @param id
230	* The id of the job
231	* @param baseDirectory
232	* The root directory containing all the files
233	* @param outputs
234	* The files to add
235	* @return A list of DataItem instances for adding to the job
236	* @throws IOException
237	* If anything goes wrong.
238	*/
239	List<DataItem> addOutputs(final String projectId, final int id,
240	final File baseDirectory, final Collection<File> outputs)
241	throws IOException {
242	if (isNull(outputs)) {	×
243	return null;	×
244	}
245
246	final var pId = new File(projectId).getName();	×
247	final int pathStart = baseDirectory.getAbsolutePath().length();	×
248	final var projectDirectory = getProjectDirectory(projectId);	×
249	final var idDirectory = new File(projectDirectory, String.valueOf(id));	×
250
251	try (var op = new JobLock(idDirectory)) {	×
252	final var outputData = new ArrayList<DataItem>();	×
253	for (final var output : outputs) {	×
254	if (!output.getAbsolutePath()	×
255	.startsWith(baseDirectory.getAbsolutePath())) {	×
256	throw new IOException("Output file " + output	×
257	+ " is outside base directory " + baseDirectory);
258	}
259
260	var outputPath = output.getAbsolutePath()	×
261	.substring(pathStart).replace('\\', '/');	×
262	if (outputPath.startsWith("/")) {	×
263	outputPath = outputPath.substring(1);	×
264	}
265
266	final var newOutput = new File(idDirectory, outputPath);	×
267	newOutput.getParentFile().mkdirs();	×
268	if (newOutput.exists()) {	×
269	if (!newOutput.delete()) {	×
270	logger.warn("Could not delete existing file {};"	×
271	+ " new file will not be used!", newOutput);
272	} else {
273	logger.warn("Overwriting existing file {}", newOutput);	×
274	}
275	}
276	if (!newOutput.exists()) {	×
277	move(output.toPath(), newOutput.toPath());	×
278	}
279	final var outputUrl = new URL(baseServerUrl,	×
280	"output/" + pId + "/" + id + "/" + outputPath);
281	outputData.add(new DataItem(outputUrl.toExternalForm()));	×
282	logger.debug("New output {} mapped to {}",	×
283	newOutput, outputUrl);
284	}	×
285
286	return outputData;	×
287	}
288	}
289
290	/**
291	* Get a file as a response to a query.
292	*
293	* @param idDirectory
294	* The directory of the project
295	* @param filename
296	* The name of the file to be stored
297	* @param download
298	* True if the content type should be set to guarantee that the
299	* file is downloaded, False to attempt to guess the content type
300	* @return The response
301	*/
302	private Response getResultFile(final File idDirectory,
303	final String filename, final boolean download) {
304	final var resultFile = new File(idDirectory, filename);	×
305	final var purgeFile = getPurgeFile(idDirectory);	×
306	JobManager.checkFileIsInFolder(idDirectory, resultFile);	×
307
308	try (var op = new JobLock(idDirectory)) {	×
309	if (purgeFile.exists()) {	×
310	logger.debug("{} was purged", idDirectory);	×
311	return status(NOT_FOUND).entity("Results from job "	×
312	+ idDirectory.getName() + " have been removed")	×
313	.build();	×
314	}
315
316	if (!resultFile.canRead()) {	×
317	logger.debug("{} was not found", resultFile);	×
318	return status(NOT_FOUND).build();	×
319	}
320
321	try {
322	if (!download) {	×
323	final var contentType =	×
324	probeContentType(resultFile.toPath());	×
325	if (nonNull(contentType)) {	×
326	logger.debug("File has content type {}", contentType);	×
327	return ok(resultFile, contentType).build();	×
328	}
329	}
330	} catch (final IOException e) {	×
331	logger.debug("Content type of {} could not be determined",	×
332	resultFile, e);
333	}	×
334
335	return ok(resultFile).header("Content-Disposition",	×
336	"attachment; filename=" + filename).build();	×
337	}	×
338	}
339
340	/**
341	* Get the file that marks a directory as purged.
342	*
343	* @param directory
344	* The directory to find the file in
345	* @return The purge marker file
346	*/
347	private File getPurgeFile(final File directory) {
348	return new File(resultsDirectory, PURGED_FILE + directory.getName());	×
349	}
350
351	@Override
352	public Response getResultFile(final String projectId, final int id,
353	final String filename, final boolean download) {
354	logger.debug("Retrieving {} from {}/{}", filename, projectId, id);	×
355	final var projectDirectory = getProjectDirectory(projectId);	×
356	final var idDirectory = new File(projectDirectory, String.valueOf(id));	×
357	return getResultFile(idDirectory, filename, download);	×
358	}
359
360	@Override
361	public Response getResultFile(final int id, final String filename,
362	final boolean download) {
363	logger.debug("Retrieving {} from {}", filename, id);	×
364	final var idDirectory = getProjectDirectory(String.valueOf(id));	×
365	return getResultFile(idDirectory, filename, download);	×
366	}
367
368	/**
369	* Upload files in recursive subdirectories to UniCore.
370	*
371	* @param authHeader
372	* The authentication to use
373	* @param directory
374	* The directory to start from
375	* @param fileManager
376	* The UniCore client
377	* @param storageId
378	* The id of the UniCore storage
379	* @param filePath
380	* The path in the UniCore storage to upload to
381	* @throws IOException
382	* If something goes wrong
383	*/
384	private void recursivelyUploadFiles(final String authHeader,
385	final File directory,
386	final UnicoreFileClient fileManager, final String storageId,
387	final String filePath) throws IOException {
388	final var files = directory.listFiles();	×
389	if (isNull(files)) {	×
390	return;	×
391	}
392	for (final var file : files) {	×
393	if (file.getName().equals(".") \|\| file.getName().equals("..")	×
394	\|\| file.getName().isEmpty()) {	×
395	continue;	×
396	}
397	final var uploadFileName = filePath + "/" + file.getName();	×
398	if (file.isDirectory()) {	×
399	recursivelyUploadFiles(authHeader, file, fileManager, storageId,	×
400	uploadFileName);
401	continue;	×
402	}
403	if (!file.isFile()) {	×
404	continue;	×
405	}
406	try (var input = new FileInputStream(file)) {	×
407	fileManager.upload(authHeader, storageId, uploadFileName,	×
408	input);
409	} catch (final WebApplicationException e) {	×
410	throw new IOException("Error uploading file to " + storageId	×
411	+ "/" + uploadFileName, e);
412	} catch (final FileNotFoundException e) {	×
413	// Ignore files which vanish.
414	}	×
415	}
416	}	×
417
418	@Override
419	public Response uploadResultsToHPCServer(final String projectId,
420	final int id, final String serverUrl, final String storageId,
421	final String filePath, final String userId, final String token) {
422	final var idDirectory =	×
423	new File(getProjectDirectory(projectId), String.valueOf(id));	×
424	if (!idDirectory.canRead()) {	×
425	logger.debug("{} was not found", idDirectory);	×
426	return status(NOT_FOUND).build();	×
427	}
428
429	try {
430	final var authHeader = "Bearer: " + token;	×
431	final var fileClient = UnicoreFileClient.createClient(serverUrl);	×
432	try (var op = new JobLock(idDirectory)) {	×
433	recursivelyUploadFiles(authHeader, idDirectory, fileClient,	×
434	storageId, filePath.replaceAll("/+$", ""));	×
435	}
436	} catch (final MalformedURLException e) {	×
437	logger.error("bad user-supplied URL", e);	×
438	return status(BAD_REQUEST)	×
439	.entity("The URL specified was malformed").build();	×
440	} catch (final Throwable e) {	×
441	logger.error("failure in upload", e);	×
442	return serverError()	×
443	.entity("General error reading or uploading a file")	×
444	.build();	×
445	}	×
446
447	return ok("ok").build();	×
448	}
449
450	/**
451	* Recursively remove a directory.
452	*
453	* @param directory
454	* The directory to remove
455	*/
456	private void removeDirectory(final File directory) {
457	for (final var file : directory.listFiles()) {	×
458	if (file.isDirectory()) {	×
459	removeDirectory(file);	×
460	} else {
461	file.delete();	×
462	}
463	}
464	directory.delete();	×
465	}	×
466
467	/**
468	* Remove files that are deemed to have expired.
469	*/
470	private void removeOldFiles() {
471	final long startTime = currentTimeMillis();	2✔
472	for (final var projectDirectory : resultsDirectory.listFiles()) {	2!
473	if (projectDirectory.isDirectory()	2✔
474	&& removeOldProjectDirectoryContents(startTime,	×
475	projectDirectory)) {
476	logger.info("No more outputs for project {}",	×
477	projectDirectory.getName());	×
478	projectDirectory.delete();	×
479	}
480	}
481	}	×
482
483	/**
484	* Remove project contents that are deemed to have expired.
485	*
486	* @param startTime
487	* The current time being considered
488	* @param projectDirectory
489	* The directory containing the project files
490	* @return True if every job in the project has been removed
491	*/
492	private boolean removeOldProjectDirectoryContents(final long startTime,
493	final File projectDirectory) {
494	boolean allJobsRemoved = true;	2✔
495	for (final var jobDirectory : projectDirectory.listFiles()) {	×
496	logger.debug("Determining whether to remove {} "	×
497	+ "which is {}ms old of {}", jobDirectory,
498	startTime - jobDirectory.lastModified(),	×
499	timeToKeepResults);	×
500	if (jobDirectory.isDirectory() && ((startTime	×
501	- jobDirectory.lastModified()) > timeToKeepResults)) {	×
502	logger.info("Removing results for job {}",	×
503	jobDirectory.getName());	×
504	try (var op = new JobLock(jobDirectory)) {	×
505	removeDirectory(jobDirectory);	×
506	}
507
508	try (var purgedFileWriter =	×
509	new PrintWriter(getPurgeFile(jobDirectory))) {	×
510	purgedFileWriter.println(currentTimeMillis());	×
511	} catch (final IOException e) {	×
512	logger.error("Error writing purge file", e);	×
513	}	×
514	} else {
515	allJobsRemoved = false;	×
516	}
517	}
518	return allJobsRemoved;	×
519	}
520	}

SpiNNakerManchester / JavaSpiNNaker / 7087

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous