27460390245

Committed 13 Jun 2026 07:29AM UTC coverage: 94.575% (-0.4%) from 95.005%

Build # 27460390245

Build Type

push

github

Committed by

web-flow

Commit Message

Implement File Deduplication by Hash (#79)

* Implement hash-based file deduplication feature, fix test suite lock issue, and add unit/integration tests

* Update pre-commit configuration to use local Maven-based Java formatting hook

* Add unit tests to cover default methods in UploadStorageService interface

* Add unit tests to fully cover all new code paths in DiskStorageService and Utils

* Expand UtilsTest to cover all methods in Utils.java

* Update AGENTS.md instructions and set CHANGELOG version header to 1.0.0-3.2

* feat: Update AGENTS.md

* Extract getChecksumPath in DiskStorageService, add reusability guidelines to AGENTS.md, and add file manipulation verify check to ITTusFileUploadService

Coverage Stats

570 of 640 branches covered (89.06%)

Branch coverage included in aggregate %.

145 of 150 new or added lines in 8 files covered. (96.67%)

1609 of 1664 relevant lines covered (96.69%)

6.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.63

/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java

package me.desair.tus.server.upload.disk;

import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import me.desair.tus.server.checksum.ChecksumAlgorithm;
import me.desair.tus.server.exception.InvalidUploadOffsetException;
import me.desair.tus.server.exception.TusException;
import me.desair.tus.server.exception.UploadNotFoundException;
import me.desair.tus.server.upload.UploadId;
import me.desair.tus.server.upload.UploadIdFactory;
import me.desair.tus.server.upload.UploadInfo;
import me.desair.tus.server.upload.UploadLockingService;
import me.desair.tus.server.upload.UploadStorageService;
import me.desair.tus.server.upload.UploadType;
import me.desair.tus.server.upload.concatenation.UploadConcatenationService;
import me.desair.tus.server.upload.concatenation.VirtualConcatenationService;
import me.desair.tus.server.util.Utils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** Implementation of {@link UploadStorageService} that implements storage on disk. */
public class DiskStorageService extends AbstractDiskBasedService implements UploadStorageService {

  private static final Logger log = LoggerFactory.getLogger(DiskStorageService.class);

  private static final String UPLOAD_SUB_DIRECTORY = "uploads";
  private static final String INFO_FILE = "info";
  private static final String DATA_FILE = "data";

  private Long maxUploadSize = null;
  private Long uploadExpirationPeriod = null;
  private UploadIdFactory idFactory;
  private UploadConcatenationService uploadConcatenationService;
  private boolean isUploadDeduplicationEnabled = false;

  public DiskStorageService(String storagePath) {
    super(storagePath + File.separator + UPLOAD_SUB_DIRECTORY);
    setUploadConcatenationService(new VirtualConcatenationService(this));
  }

  public DiskStorageService(UploadIdFactory idFactory, String storagePath) {
    this(storagePath);
    Validate.notNull(idFactory, "The IdFactory cannot be null");
    this.idFactory = idFactory;
  }

  @Override
  public void setIdFactory(UploadIdFactory idFactory) {
    Validate.notNull(idFactory, "The IdFactory cannot be null");
    this.idFactory = idFactory;
  }

  @Override
  public void setMaxUploadSize(Long maxUploadSize) {
    this.maxUploadSize = (maxUploadSize != null && maxUploadSize > 0 ? maxUploadSize : 0);
  }

  @Override
  public long getMaxUploadSize() {
    return maxUploadSize == null ? 0 : maxUploadSize;
  }

  @Override
  public void setUploadDeduplicationEnabled(boolean enabled) {
    this.isUploadDeduplicationEnabled = enabled;
  }

  @Override
  public boolean isUploadDeduplicationEnabled() {
    return this.isUploadDeduplicationEnabled;
  }

  @Override
  public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm algorithm)
      throws IOException {
    if (checksum == null || algorithm == null) {
      return null;
    }
    Path checksumFile = getChecksumPath(checksum, algorithm);
    if (Files.exists(checksumFile)) {
      String uploadIdStr =
          new String(Files.readAllBytes(checksumFile), StandardCharsets.UTF_8).trim();
      UploadId id = new UploadId(uploadIdStr);
      UploadInfo info = getUploadInfo(id);
      if (info == null) {
        Files.deleteIfExists(checksumFile);
        return null;
      }
      Path bytesPath = null;
      try {
        bytesPath = getPathInUploadDir(id, DATA_FILE);
      } catch (UploadNotFoundException e) {
        // file doesn't exist
      }
      if (bytesPath == null || !Files.exists(bytesPath)) {
        Files.deleteIfExists(checksumFile);
        return null;
      }
      return info;
    }
    return null;
  }

  @Override
  public UploadInfo getUploadInfo(String uploadUrl, String ownerKey) throws IOException {
    UploadInfo uploadInfo = getUploadInfo(idFactory.readUploadId(uploadUrl));
    if (uploadInfo == null || !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {
      return null;
    } else {
      return uploadInfo;
    }
  }

  @Override
  public UploadInfo getUploadInfo(UploadId id) throws IOException {
    try {
      Path infoPath = getInfoPath(id);
      if (infoPath == null || !Files.exists(infoPath)) {
        return null;
      }
      return Utils.readSerializable(infoPath, UploadInfo.class);
    } catch (UploadNotFoundException e) {
      return null;
    }
  }

  @Override
  public String getUploadUri() {
    return idFactory.getUploadUri();
  }

  @Override
  public UploadInfo create(UploadInfo info, String ownerKey) throws IOException {
    UploadId id = createNewId();

    createUploadDirectory(id);

    try {
      Path bytesPath = getBytesPath(id);

      // Create an empty file to storage the bytes of this upload
      Files.createFile(bytesPath);

      // Set starting values
      info.setId(id);
      info.setOffset(0L);
      info.setOwnerKey(ownerKey);

      update(info);

      return info;
    } catch (UploadNotFoundException e) {
      // Normally this cannot happen
      log.error("Unable to create UploadInfo because of an upload not found exception", e);
      return null;
    }
  }

  @Override
  public void update(UploadInfo uploadInfo) throws IOException, UploadNotFoundException {
    if (uploadInfo != null) {
      if (uploadInfo.getDuplicatesUploadId() != null) {
        // Delete the child's own data file if it exists
        try {
          Path childDataPath = getPathInUploadDir(uploadInfo.getId(), DATA_FILE);
          Files.deleteIfExists(childDataPath);
        } catch (UploadNotFoundException e) {
          // It doesn't exist yet, which is fine
        }

        // Retrieve the parent UploadInfo
        UploadId parentId = uploadInfo.getDuplicatesUploadId();
        UploadInfo parentInfo = getUploadInfo(parentId);
        if (parentInfo != null) {
          Path parentBytesPath = null;
          try {
            parentBytesPath = getPathInUploadDir(parentId, DATA_FILE);
          } catch (UploadNotFoundException e) {
            // Parent file is not found
          }

          if (parentBytesPath != null && Files.exists(parentBytesPath)) {
            // Ensure parent's expiration timestamp is >= child's expiration timestamp
            Long childExpire = uploadInfo.getExpirationTimestamp();
            Long parentExpire = parentInfo.getExpirationTimestamp();
            if (childExpire == null) {
              if (parentExpire != null) {
                parentInfo.setExpirationTimestamp(null);
                Path parentInfoPath = getInfoPath(parentId);
                Utils.writeSerializable(parentInfo, parentInfoPath);
              }
            } else {
              if (parentExpire != null && parentExpire < childExpire) {
                parentInfo.setExpirationTimestamp(childExpire);
                Path parentInfoPath = getInfoPath(parentId);
                Utils.writeSerializable(parentInfo, parentInfoPath);
              }
            }
          }
        }
      } else if (isUploadDeduplicationEnabled()
          && !uploadInfo.isUploadInProgress()
          && uploadInfo.getChecksum() != null
          && uploadInfo.getChecksumAlgorithm() != null) {
        // Index the checksum
        Path checksumFile =
            getChecksumPath(uploadInfo.getChecksum(), uploadInfo.getChecksumAlgorithm());
        Files.createDirectories(checksumFile.getParent());
        Files.write(checksumFile, uploadInfo.getId().toString().getBytes(StandardCharsets.UTF_8));
      }

      Path infoPath = getInfoPath(uploadInfo.getId());
      Utils.writeSerializable(uploadInfo, infoPath);
    }
  }

  @Override
  public UploadInfo append(UploadInfo info, InputStream inputStream)
      throws IOException, TusException {
    if (info != null) {
      Path bytesPath = getBytesPath(info.getId());

      long max = getMaxUploadSize() > 0 ? getMaxUploadSize() : Long.MAX_VALUE;
      long transferred = 0;
      Long offset = info.getOffset();
      long newOffset = offset;

      try (ReadableByteChannel uploadedBytes = Channels.newChannel(inputStream);
          FileChannel file = FileChannel.open(bytesPath, WRITE)) {

        try {
          // Lock will be released when the channel closes
          file.lock();

          // Validate that the given offset is at the end of the file
          if (!offset.equals(file.size())) {
            throw new InvalidUploadOffsetException(
                "The upload offset does not correspond to the written"
                    + " bytes. You can only append to the end of an upload");
          }

          // write all bytes in the channel up to the configured maximum
          transferred = file.transferFrom(uploadedBytes, offset, max - offset);
          file.force(true);
          newOffset = offset + transferred;

        } catch (Exception ex) {
          // An error occurred, try to write as much data as possible
          newOffset = writeAsMuchAsPossible(file);
          throw ex;
        }

      } finally {
        info.setOffset(newOffset);
        update(info);
      }
    }

    return info;
  }

  @Override
  public void removeLastNumberOfBytes(UploadInfo info, long byteCount)
      throws UploadNotFoundException, IOException {

    if (info != null && byteCount > 0) {
      Path bytesPath = getBytesPath(info.getId());

      try (FileChannel file = FileChannel.open(bytesPath, WRITE)) {

        // Lock will be released when the channel closes
        file.lock();

        file.truncate(file.size() - byteCount);
        file.force(true);

        info.setOffset(file.size());
        update(info);
      }
    }
  }

  @Override
  public void terminateUpload(UploadInfo info) throws UploadNotFoundException, IOException {
    if (info != null) {
      if (info.getDuplicatesUploadId() == null
          && info.getChecksum() != null
          && info.getChecksumAlgorithm() != null) {
        Path checksumFile = getChecksumPath(info.getChecksum(), info.getChecksumAlgorithm());
        Files.deleteIfExists(checksumFile);
      }
      Path uploadPath = getPathInStorageDirectory(info.getId());
      FileUtils.deleteDirectory(uploadPath.toFile());
    }
  }

  @Override
  public Long getUploadExpirationPeriod() {
    return uploadExpirationPeriod;
  }

  @Override
  public void setUploadExpirationPeriod(Long uploadExpirationPeriod) {
    this.uploadExpirationPeriod = uploadExpirationPeriod;
  }

  @Override
  public void setUploadConcatenationService(UploadConcatenationService concatenationService) {
    Validate.notNull(concatenationService);
    this.uploadConcatenationService = concatenationService;
  }

  @Override
  public UploadConcatenationService getUploadConcatenationService() {
    return uploadConcatenationService;
  }

  @Override
  public InputStream getUploadedBytes(String uploadUri, String ownerKey)
      throws IOException, UploadNotFoundException {

    UploadId id = idFactory.readUploadId(uploadUri);

    UploadInfo uploadInfo = getUploadInfo(id);
    if (uploadInfo == null || !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {
      throw new UploadNotFoundException(
          "The upload with id " + id + " could not be found for owner " + ownerKey);
    } else {
      return getUploadedBytes(id);
    }
  }

  @Override
  public InputStream getUploadedBytes(UploadId id) throws IOException, UploadNotFoundException {
    InputStream inputStream = null;
    UploadInfo uploadInfo = getUploadInfo(id);
    if (uploadInfo == null) {
      throw new UploadNotFoundException("The upload with id " + id + " was not found.");
    }

    if (uploadInfo.getDuplicatesUploadId() != null) {
      return getUploadedBytes(uploadInfo.getDuplicatesUploadId());
    }

    if (UploadType.CONCATENATED.equals(uploadInfo.getUploadType())
        && uploadConcatenationService != null) {
      inputStream = uploadConcatenationService.getConcatenatedBytes(uploadInfo);

    } else {
      Path bytesPath = getBytesPath(id);
      // If bytesPath is not null, we know this is a valid Upload URI
      if (bytesPath != null) {
        if (!Files.exists(bytesPath)) {
          throw new UploadNotFoundException(
              "The upload bytes for id " + id + " could not be found.");
        }
        inputStream = Channels.newInputStream(FileChannel.open(bytesPath, READ));
      }
    }

    return inputStream;
  }

  @Override
  public void copyUploadTo(UploadInfo info, OutputStream outputStream)
      throws UploadNotFoundException, IOException {

    List<UploadInfo> uploads = getUploads(info);

    try (WritableByteChannel outputChannel = Channels.newChannel(outputStream)) {

      for (UploadInfo upload : uploads) {
        if (upload == null) {
          log.warn("We cannot copy the bytes of an upload that does not exist");

        } else if (upload.isUploadInProgress()) {
          log.warn(
              "We cannot copy the bytes of upload {} because it is still in progress",
              upload.getId());

        } else {
          UploadId readId =
              upload.getDuplicatesUploadId() != null
                  ? upload.getDuplicatesUploadId()
                  : upload.getId();
          Path bytesPath = getBytesPath(readId);
          if (!Files.exists(bytesPath)) {
            throw new UploadNotFoundException(
                "The upload bytes for id " + readId + " could not be found.");
          }
          try (FileChannel file = FileChannel.open(bytesPath, READ)) {
            // Efficiently copy the bytes to the output stream
            file.transferTo(0, upload.getLength(), outputChannel);
          }
        }
      }
    }
  }

  @Override
  public void cleanupExpiredUploads(UploadLockingService uploadLockingService) throws IOException {
    try (DirectoryStream<Path> expiredUploadsStream =
        Files.newDirectoryStream(
            getStoragePath(), new ExpiredUploadFilter(this, uploadLockingService))) {

      for (Path path : expiredUploadsStream) {
        FileUtils.deleteDirectory(path.toFile());
      }
    }
  }

  private List<UploadInfo> getUploads(UploadInfo info) throws IOException, UploadNotFoundException {
    List<UploadInfo> uploads;

    if (info != null
        && UploadType.CONCATENATED.equals(info.getUploadType())
        && uploadConcatenationService != null) {
      uploadConcatenationService.merge(info);
      uploads = uploadConcatenationService.getPartialUploads(info);
    } else {
      uploads = Collections.singletonList(info);
    }
    return uploads;
  }

  private Path getBytesPath(UploadId id) throws UploadNotFoundException {
    return getPathInUploadDir(id, DATA_FILE);
  }

  private Path getInfoPath(UploadId id) throws UploadNotFoundException {
    return getPathInUploadDir(id, INFO_FILE);
  }

  private Path getChecksumPath(String checksum, ChecksumAlgorithm algorithm) {
    return getStoragePath()
        .getParent()
        .resolve("checksums")
        .resolve(algorithm.toString())
        .resolve(checksum);
  }

  private Path createUploadDirectory(UploadId id) throws IOException {
    return Files.createDirectories(getPathInStorageDirectory(id));
  }

  private Path getPathInUploadDir(UploadId id, String fileName) throws UploadNotFoundException {
    // Get the upload directory
    Path uploadDir = getPathInStorageDirectory(id);
    if (uploadDir != null && Files.exists(uploadDir)) {
      return uploadDir.resolve(fileName);
    } else {
      throw new UploadNotFoundException("The upload for id " + id + " was not found.");
    }
  }

  private synchronized UploadId createNewId() throws IOException {
    UploadId id;
    do {
      id = idFactory.createId();
      // For extra safety, double check that this ID is not in use yet
    } while (getUploadInfo(id) != null);
    return id;
  }

  private long writeAsMuchAsPossible(FileChannel file) throws IOException {
    long offset = 0;
    if (file != null) {
      file.force(true);
      offset = file.size();
    }
    return offset;
  }
}

1	package me.desair.tus.server.upload.disk;
2
3	import static java.nio.file.StandardOpenOption.READ;
4	import static java.nio.file.StandardOpenOption.WRITE;
5
6	import java.io.File;
7	import java.io.IOException;
8	import java.io.InputStream;
9	import java.io.OutputStream;
10	import java.nio.channels.Channels;
11	import java.nio.channels.FileChannel;
12	import java.nio.channels.ReadableByteChannel;
13	import java.nio.channels.WritableByteChannel;
14	import java.nio.charset.StandardCharsets;
15	import java.nio.file.DirectoryStream;
16	import java.nio.file.Files;
17	import java.nio.file.Path;
18	import java.util.Collections;
19	import java.util.List;
20	import java.util.Objects;
21	import me.desair.tus.server.checksum.ChecksumAlgorithm;
22	import me.desair.tus.server.exception.InvalidUploadOffsetException;
23	import me.desair.tus.server.exception.TusException;
24	import me.desair.tus.server.exception.UploadNotFoundException;
25	import me.desair.tus.server.upload.UploadId;
26	import me.desair.tus.server.upload.UploadIdFactory;
27	import me.desair.tus.server.upload.UploadInfo;
28	import me.desair.tus.server.upload.UploadLockingService;
29	import me.desair.tus.server.upload.UploadStorageService;
30	import me.desair.tus.server.upload.UploadType;
31	import me.desair.tus.server.upload.concatenation.UploadConcatenationService;
32	import me.desair.tus.server.upload.concatenation.VirtualConcatenationService;
33	import me.desair.tus.server.util.Utils;
34	import org.apache.commons.io.FileUtils;
35	import org.apache.commons.lang3.Validate;
36	import org.slf4j.Logger;
37	import org.slf4j.LoggerFactory;
38
39	/** Implementation of {@link UploadStorageService} that implements storage on disk. */
40	public class DiskStorageService extends AbstractDiskBasedService implements UploadStorageService {
41
42	private static final Logger log = LoggerFactory.getLogger(DiskStorageService.class);	8✔
43
44	private static final String UPLOAD_SUB_DIRECTORY = "uploads";
45	private static final String INFO_FILE = "info";
46	private static final String DATA_FILE = "data";
47
48	private Long maxUploadSize = null;	6✔
49	private Long uploadExpirationPeriod = null;	6✔
50	private UploadIdFactory idFactory;
51	private UploadConcatenationService uploadConcatenationService;
52	private boolean isUploadDeduplicationEnabled = false;	6✔
53
54	public DiskStorageService(String storagePath) {
55	super(storagePath + File.separator + UPLOAD_SUB_DIRECTORY);	10✔
56	setUploadConcatenationService(new VirtualConcatenationService(this));	12✔
57	}	2✔
58
59	public DiskStorageService(UploadIdFactory idFactory, String storagePath) {
60	this(storagePath);	6✔
61	Validate.notNull(idFactory, "The IdFactory cannot be null");	12✔
62	this.idFactory = idFactory;	6✔
63	}	2✔
64
65	@Override
66	public void setIdFactory(UploadIdFactory idFactory) {
67	Validate.notNull(idFactory, "The IdFactory cannot be null");	6✔
68	this.idFactory = idFactory;	3✔
69	}	1✔
70
71	@Override
72	public void setMaxUploadSize(Long maxUploadSize) {
73	this.maxUploadSize = (maxUploadSize != null && maxUploadSize > 0 ? maxUploadSize : 0);	28✔
74	}	2✔
75
76	@Override
77	public long getMaxUploadSize() {
78	return maxUploadSize == null ? 0 : maxUploadSize;	18✔
79	}
80
81	@Override
82	public void setUploadDeduplicationEnabled(boolean enabled) {
83	this.isUploadDeduplicationEnabled = enabled;	6✔
84	}	2✔
85
86	@Override
87	public boolean isUploadDeduplicationEnabled() {
88	return this.isUploadDeduplicationEnabled;	6✔
89	}
90
91	@Override
92	public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm algorithm)
93	throws IOException {
94	if (checksum == null \|\| algorithm == null) {	8✔
95	return null;	2✔
96	}
97	Path checksumFile = getChecksumPath(checksum, algorithm);	10✔
98	if (Files.exists(checksumFile)) {	10✔
99	String uploadIdStr =	6✔
100	new String(Files.readAllBytes(checksumFile), StandardCharsets.UTF_8).trim();	10✔
101	UploadId id = new UploadId(uploadIdStr);	10✔
102	UploadInfo info = getUploadInfo(id);	8✔
103	if (info == null) {	4✔
104	Files.deleteIfExists(checksumFile);	3✔
105	return null;	2✔
106	}
107	Path bytesPath = null;	4✔
108	try {
109	bytesPath = getPathInUploadDir(id, DATA_FILE);	10✔
NEW 110	} catch (UploadNotFoundException e) {	×
111	// file doesn't exist
112	}	2✔
113	if (bytesPath == null \|\| !Files.exists(bytesPath)) {	14!
114	Files.deleteIfExists(checksumFile);	3✔
115	return null;	2✔
116	}
117	return info;	4✔
118	}
119	return null;	4✔
120	}
121
122	@Override
123	public UploadInfo getUploadInfo(String uploadUrl, String ownerKey) throws IOException {
124	UploadInfo uploadInfo = getUploadInfo(idFactory.readUploadId(uploadUrl));	14✔
125	if (uploadInfo == null \|\| !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {	14✔
126	return null;	4✔
127	} else {
128	return uploadInfo;	4✔
129	}
130	}
131
132	@Override
133	public UploadInfo getUploadInfo(UploadId id) throws IOException {
134	try {
135	Path infoPath = getInfoPath(id);	8✔
136	if (infoPath == null \|\| !Files.exists(infoPath)) {	14!
137	return null;	2✔
138	}
139	return Utils.readSerializable(infoPath, UploadInfo.class);	10✔
140	} catch (UploadNotFoundException e) {	2✔
141	return null;	4✔
142	}
143	}
144
145	@Override
146	public String getUploadUri() {
147	return idFactory.getUploadUri();	8✔
148	}
149
150	@Override
151	public UploadInfo create(UploadInfo info, String ownerKey) throws IOException {
152	UploadId id = createNewId();	6✔
153
154	createUploadDirectory(id);	8✔
155
156	try {
157	Path bytesPath = getBytesPath(id);	8✔
158
159	// Create an empty file to storage the bytes of this upload
160	Files.createFile(bytesPath);	10✔
161
162	// Set starting values
163	info.setId(id);	6✔
164	info.setOffset(0L);	8✔
165	info.setOwnerKey(ownerKey);	6✔
166
167	update(info);	6✔
168
169	return info;	4✔
170	} catch (UploadNotFoundException e) {	×
171	// Normally this cannot happen
172	log.error("Unable to create UploadInfo because of an upload not found exception", e);	×
173	return null;	×
174	}
175	}
176
177	@Override
178	public void update(UploadInfo uploadInfo) throws IOException, UploadNotFoundException {
179	if (uploadInfo != null) {	4!
180	if (uploadInfo.getDuplicatesUploadId() != null) {	6✔
181	// Delete the child's own data file if it exists
182	try {
183	Path childDataPath = getPathInUploadDir(uploadInfo.getId(), DATA_FILE);	12✔
184	Files.deleteIfExists(childDataPath);	6✔
NEW 185	} catch (UploadNotFoundException e) {	×
186	// It doesn't exist yet, which is fine
187	}	2✔
188
189	// Retrieve the parent UploadInfo
190	UploadId parentId = uploadInfo.getDuplicatesUploadId();	6✔
191	UploadInfo parentInfo = getUploadInfo(parentId);	8✔
192	if (parentInfo != null) {	4!
193	Path parentBytesPath = null;	4✔
194	try {
195	parentBytesPath = getPathInUploadDir(parentId, DATA_FILE);	10✔
NEW 196	} catch (UploadNotFoundException e) {	×
197	// Parent file is not found
198	}	2✔
199
200	if (parentBytesPath != null && Files.exists(parentBytesPath)) {	14!
201	// Ensure parent's expiration timestamp is >= child's expiration timestamp
202	Long childExpire = uploadInfo.getExpirationTimestamp();	6✔
203	Long parentExpire = parentInfo.getExpirationTimestamp();	6✔
204	if (childExpire == null) {	4✔
205	if (parentExpire != null) {	2✔
206	parentInfo.setExpirationTimestamp(null);	3✔
207	Path parentInfoPath = getInfoPath(parentId);	4✔
208	Utils.writeSerializable(parentInfo, parentInfoPath);	3✔
209	}	1✔
210	} else {
211	if (parentExpire != null && parentExpire < childExpire) {	16!
212	parentInfo.setExpirationTimestamp(childExpire);	6✔
213	Path parentInfoPath = getInfoPath(parentId);	8✔
214	Utils.writeSerializable(parentInfo, parentInfoPath);	6✔
215	}
216	}
217	}
218	}
219	} else if (isUploadDeduplicationEnabled()	10✔
220	&& !uploadInfo.isUploadInProgress()	6✔
221	&& uploadInfo.getChecksum() != null	6✔
222	&& uploadInfo.getChecksumAlgorithm() != null) {	4!
223	// Index the checksum
224	Path checksumFile =	4✔
225	getChecksumPath(uploadInfo.getChecksum(), uploadInfo.getChecksumAlgorithm());	10✔
226	Files.createDirectories(checksumFile.getParent());	12✔
227	Files.write(checksumFile, uploadInfo.getId().toString().getBytes(StandardCharsets.UTF_8));	20✔
228	}
229
230	Path infoPath = getInfoPath(uploadInfo.getId());	10✔
231	Utils.writeSerializable(uploadInfo, infoPath);	6✔
232	}
233	}	2✔
234
235	@Override
236	public UploadInfo append(UploadInfo info, InputStream inputStream)
237	throws IOException, TusException {
238	if (info != null) {	4!
239	Path bytesPath = getBytesPath(info.getId());	10✔
240
241	long max = getMaxUploadSize() > 0 ? getMaxUploadSize() : Long.MAX_VALUE;	20✔
242	long transferred = 0;	4✔
243	Long offset = info.getOffset();	6✔
244	long newOffset = offset;	6✔
245
246	try (ReadableByteChannel uploadedBytes = Channels.newChannel(inputStream);	6✔
247	FileChannel file = FileChannel.open(bytesPath, WRITE)) {	18✔
248
249	try {
250	// Lock will be released when the channel closes
251	file.lock();	6✔
252
253	// Validate that the given offset is at the end of the file
254	if (!offset.equals(file.size())) {	12✔
255	throw new InvalidUploadOffsetException(	5✔
256	"The upload offset does not correspond to the written"
257	+ " bytes. You can only append to the end of an upload");
258	}
259
260	// write all bytes in the channel up to the configured maximum
261	transferred = file.transferFrom(uploadedBytes, offset, max - offset);	20✔
262	file.force(true);	6✔
263	newOffset = offset + transferred;	10✔
264
265	} catch (Exception ex) {	1✔
266	// An error occurred, try to write as much data as possible
267	newOffset = writeAsMuchAsPossible(file);	4✔
268	throw ex;	2✔
269	}	2✔
270
271	} finally {
272	info.setOffset(newOffset);	8✔
273	update(info);	6✔
274	}
275	}
276
277	return info;	4✔
278	}
279
280	@Override
281	public void removeLastNumberOfBytes(UploadInfo info, long byteCount)
282	throws UploadNotFoundException, IOException {
283
284	if (info != null && byteCount > 0) {	12✔
285	Path bytesPath = getBytesPath(info.getId());	10✔
286
287	try (FileChannel file = FileChannel.open(bytesPath, WRITE)) {	18✔
288
289	// Lock will be released when the channel closes
290	file.lock();	6✔
291
292	file.truncate(file.size() - byteCount);	14✔
293	file.force(true);	6✔
294
295	info.setOffset(file.size());	10✔
296	update(info);	6✔
297	}
298	}
299	}	2✔
300
301	@Override
302	public void terminateUpload(UploadInfo info) throws UploadNotFoundException, IOException {
303	if (info != null) {	4✔
304	if (info.getDuplicatesUploadId() == null	8!
305	&& info.getChecksum() != null	6✔
306	&& info.getChecksumAlgorithm() != null) {	4!
307	Path checksumFile = getChecksumPath(info.getChecksum(), info.getChecksumAlgorithm());	14✔
308	Files.deleteIfExists(checksumFile);	6✔
309	}
310	Path uploadPath = getPathInStorageDirectory(info.getId());	10✔
311	FileUtils.deleteDirectory(uploadPath.toFile());	6✔
312	}
313	}	2✔
314
315	@Override
316	public Long getUploadExpirationPeriod() {
317	return uploadExpirationPeriod;	3✔
318	}
319
320	@Override
321	public void setUploadExpirationPeriod(Long uploadExpirationPeriod) {
322	this.uploadExpirationPeriod = uploadExpirationPeriod;	3✔
323	}	1✔
324
325	@Override
326	public void setUploadConcatenationService(UploadConcatenationService concatenationService) {
327	Validate.notNull(concatenationService);	6✔
328	this.uploadConcatenationService = concatenationService;	6✔
329	}	2✔
330
331	@Override
332	public UploadConcatenationService getUploadConcatenationService() {
333	return uploadConcatenationService;	3✔
334	}
335
336	@Override
337	public InputStream getUploadedBytes(String uploadUri, String ownerKey)
338	throws IOException, UploadNotFoundException {
339
340	UploadId id = idFactory.readUploadId(uploadUri);	10✔
341
342	UploadInfo uploadInfo = getUploadInfo(id);	8✔
343	if (uploadInfo == null \|\| !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {	14!
344	throw new UploadNotFoundException(	8✔
345	"The upload with id " + id + " could not be found for owner " + ownerKey);
346	} else {
347	return getUploadedBytes(id);	8✔
348	}
349	}
350
351	@Override
352	public InputStream getUploadedBytes(UploadId id) throws IOException, UploadNotFoundException {
353	InputStream inputStream = null;	4✔
354	UploadInfo uploadInfo = getUploadInfo(id);	8✔
355	if (uploadInfo == null) {	4✔
356	throw new UploadNotFoundException("The upload with id " + id + " was not found.");	7✔
357	}
358
359	if (uploadInfo.getDuplicatesUploadId() != null) {	6✔
360	return getUploadedBytes(uploadInfo.getDuplicatesUploadId());	5✔
361	}
362
363	if (UploadType.CONCATENATED.equals(uploadInfo.getUploadType())	13!
364	&& uploadConcatenationService != null) {
365	inputStream = uploadConcatenationService.getConcatenatedBytes(uploadInfo);	6✔
366
367	} else {
368	Path bytesPath = getBytesPath(id);	8✔
369	// If bytesPath is not null, we know this is a valid Upload URI
370	if (bytesPath != null) {	4!
371	if (!Files.exists(bytesPath)) {	10!
NEW 372	throw new UploadNotFoundException(	×
373	"The upload bytes for id " + id + " could not be found.");
374	}
375	inputStream = Channels.newInputStream(FileChannel.open(bytesPath, READ));	20✔
376	}
377	}
378
379	return inputStream;	4✔
380	}
381
382	@Override
383	public void copyUploadTo(UploadInfo info, OutputStream outputStream)
384	throws UploadNotFoundException, IOException {
385
386	List<UploadInfo> uploads = getUploads(info);	8✔
387
388	try (WritableByteChannel outputChannel = Channels.newChannel(outputStream)) {	6✔
389
390	for (UploadInfo upload : uploads) {	20✔
391	if (upload == null) {	4!
392	log.warn("We cannot copy the bytes of an upload that does not exist");	×
393
394	} else if (upload.isUploadInProgress()) {	6!
395	log.warn(	×
396	"We cannot copy the bytes of upload {} because it is still in progress",
397	upload.getId());	×
398
399	} else {
400	UploadId readId =
401	upload.getDuplicatesUploadId() != null	6✔
402	? upload.getDuplicatesUploadId()	6✔
403	: upload.getId();	6✔
404	Path bytesPath = getBytesPath(readId);	8✔
405	if (!Files.exists(bytesPath)) {	10!
NEW 406	throw new UploadNotFoundException(	×
407	"The upload bytes for id " + readId + " could not be found.");
408	}
409	try (FileChannel file = FileChannel.open(bytesPath, READ)) {	18✔
410	// Efficiently copy the bytes to the output stream
411	file.transferTo(0, upload.getLength(), outputChannel);	16✔
412	}
413	}
414	}	2✔
415	}
416	}	2✔
417
418	@Override
419	public void cleanupExpiredUploads(UploadLockingService uploadLockingService) throws IOException {
420	try (DirectoryStream<Path> expiredUploadsStream =	2✔
421	Files.newDirectoryStream(	4✔
422	getStoragePath(), new ExpiredUploadFilter(this, uploadLockingService))) {	12✔
423
424	for (Path path : expiredUploadsStream) {	20✔
425	FileUtils.deleteDirectory(path.toFile());	6✔
426	}	2✔
427	}
428	}	2✔
429
430	private List<UploadInfo> getUploads(UploadInfo info) throws IOException, UploadNotFoundException {
431	List<UploadInfo> uploads;
432
433	if (info != null	8!
434	&& UploadType.CONCATENATED.equals(info.getUploadType())	9!
435	&& uploadConcatenationService != null) {
436	uploadConcatenationService.merge(info);	4✔
437	uploads = uploadConcatenationService.getPartialUploads(info);	6✔
438	} else {
439	uploads = Collections.singletonList(info);	6✔
440	}
441	return uploads;	4✔
442	}
443
444	private Path getBytesPath(UploadId id) throws UploadNotFoundException {
445	return getPathInUploadDir(id, DATA_FILE);	10✔
446	}
447
448	private Path getInfoPath(UploadId id) throws UploadNotFoundException {
449	return getPathInUploadDir(id, INFO_FILE);	10✔
450	}
451
452	private Path getChecksumPath(String checksum, ChecksumAlgorithm algorithm) {
453	return getStoragePath()	6✔
454	.getParent()	4✔
455	.resolve("checksums")	4✔
456	.resolve(algorithm.toString())	6✔
457	.resolve(checksum);	2✔
458	}
459
460	private Path createUploadDirectory(UploadId id) throws IOException {
461	return Files.createDirectories(getPathInStorageDirectory(id));	14✔
462	}
463
464	private Path getPathInUploadDir(UploadId id, String fileName) throws UploadNotFoundException {
465	// Get the upload directory
466	Path uploadDir = getPathInStorageDirectory(id);	8✔
467	if (uploadDir != null && Files.exists(uploadDir)) {	14✔
468	return uploadDir.resolve(fileName);	8✔
469	} else {
470	throw new UploadNotFoundException("The upload for id " + id + " was not found.");	14✔
471	}
472	}
473
474	private synchronized UploadId createNewId() throws IOException {
475	UploadId id;
476	do {
477	id = idFactory.createId();	8✔
478	// For extra safety, double check that this ID is not in use yet
479	} while (getUploadInfo(id) != null);	8!
480	return id;	4✔
481	}
482
483	private long writeAsMuchAsPossible(FileChannel file) throws IOException {
484	long offset = 0;	2✔
485	if (file != null) {	2!
486	file.force(true);	3✔
487	offset = file.size();	3✔
488	}
489	return offset;	2✔
490	}
491	}

tomdesair / tus-java-server / 27460390245

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous