27440598708

Committed 12 Jun 2026 08:16PM UTC coverage: 94.596% (-0.4%) from 95.005%

Build # 27440598708

Build Type

Pull #79

github

Committed by

web-flow

Commit Message

Merge cb9f12824 into a7a0299f3

Pull Request Pull Request #79: Implement File Deduplication by Hash

Coverage Stats

570 of 640 branches covered (89.06%)

Branch coverage included in aggregate %.

154 of 159 new or added lines in 8 files covered. (96.86%)

7 existing lines in 1 file now uncovered.

1618 of 1673 relevant lines covered (96.71%)

6.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.91

/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java

package me.desair.tus.server.upload.disk;

import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import me.desair.tus.server.checksum.ChecksumAlgorithm;
import me.desair.tus.server.exception.InvalidUploadOffsetException;
import me.desair.tus.server.exception.TusException;
import me.desair.tus.server.exception.UploadNotFoundException;
import me.desair.tus.server.upload.UploadId;
import me.desair.tus.server.upload.UploadIdFactory;
import me.desair.tus.server.upload.UploadInfo;
import me.desair.tus.server.upload.UploadLockingService;
import me.desair.tus.server.upload.UploadStorageService;
import me.desair.tus.server.upload.UploadType;
import me.desair.tus.server.upload.concatenation.UploadConcatenationService;
import me.desair.tus.server.upload.concatenation.VirtualConcatenationService;
import me.desair.tus.server.util.Utils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** Implementation of {@link UploadStorageService} that implements storage on disk. */
public class DiskStorageService extends AbstractDiskBasedService implements UploadStorageService {

  private static final Logger log = LoggerFactory.getLogger(DiskStorageService.class);

  private static final String UPLOAD_SUB_DIRECTORY = "uploads";
  private static final String INFO_FILE = "info";
  private static final String DATA_FILE = "data";

  private Long maxUploadSize = null;
  private Long uploadExpirationPeriod = null;
  private UploadIdFactory idFactory;
  private UploadConcatenationService uploadConcatenationService;
  private boolean isUploadDeduplicationEnabled = false;

  public DiskStorageService(String storagePath) {
    super(storagePath + File.separator + UPLOAD_SUB_DIRECTORY);
    setUploadConcatenationService(new VirtualConcatenationService(this));
  }

  public DiskStorageService(UploadIdFactory idFactory, String storagePath) {
    this(storagePath);
    Validate.notNull(idFactory, "The IdFactory cannot be null");
    this.idFactory = idFactory;
  }

  @Override
  public void setIdFactory(UploadIdFactory idFactory) {
    Validate.notNull(idFactory, "The IdFactory cannot be null");
    this.idFactory = idFactory;
  }

  @Override
  public void setMaxUploadSize(Long maxUploadSize) {
    this.maxUploadSize = (maxUploadSize != null && maxUploadSize > 0 ? maxUploadSize : 0);
  }

  @Override
  public long getMaxUploadSize() {
    return maxUploadSize == null ? 0 : maxUploadSize;
  }

  @Override
  public void setUploadDeduplicationEnabled(boolean enabled) {
    this.isUploadDeduplicationEnabled = enabled;
  }

  @Override
  public boolean isUploadDeduplicationEnabled() {
    return this.isUploadDeduplicationEnabled;
  }

  @Override
  public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm algorithm)
      throws IOException {
    if (checksum == null || algorithm == null) {
      return null;
    }
    Path checksumFile =
        getStoragePath()
            .getParent()
            .resolve("checksums")
            .resolve(algorithm.toString())
            .resolve(checksum);
    if (Files.exists(checksumFile)) {
      String uploadIdStr =
          new String(Files.readAllBytes(checksumFile), StandardCharsets.UTF_8).trim();
      UploadId id = new UploadId(uploadIdStr);
      UploadInfo info = getUploadInfo(id);
      if (info == null) {
        Files.deleteIfExists(checksumFile);
        return null;
      }
      Path bytesPath = null;
      try {
        bytesPath = getPathInUploadDir(id, DATA_FILE);
      } catch (UploadNotFoundException e) {
        // file doesn't exist
      }
      if (bytesPath == null || !Files.exists(bytesPath)) {
        Files.deleteIfExists(checksumFile);
        return null;
      }
      return info;
    }
    return null;
  }

  @Override
  public UploadInfo getUploadInfo(String uploadUrl, String ownerKey) throws IOException {
    UploadInfo uploadInfo = getUploadInfo(idFactory.readUploadId(uploadUrl));
    if (uploadInfo == null || !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {
      return null;
    } else {
      return uploadInfo;
    }
  }

  @Override
  public UploadInfo getUploadInfo(UploadId id) throws IOException {
    try {
      Path infoPath = getInfoPath(id);
      if (infoPath == null || !Files.exists(infoPath)) {
        return null;
      }
      return Utils.readSerializable(infoPath, UploadInfo.class);
    } catch (UploadNotFoundException e) {
      return null;
    }
  }

  @Override
  public String getUploadUri() {
    return idFactory.getUploadUri();
  }

  @Override
  public UploadInfo create(UploadInfo info, String ownerKey) throws IOException {
    UploadId id = createNewId();

    createUploadDirectory(id);

    try {
      Path bytesPath = getBytesPath(id);

      // Create an empty file to storage the bytes of this upload
      Files.createFile(bytesPath);

      // Set starting values
      info.setId(id);
      info.setOffset(0L);
      info.setOwnerKey(ownerKey);

      update(info);

      return info;
    } catch (UploadNotFoundException e) {
      // Normally this cannot happen
      log.error("Unable to create UploadInfo because of an upload not found exception", e);
      return null;
    }
  }

  @Override
  public void update(UploadInfo uploadInfo) throws IOException, UploadNotFoundException {
    if (uploadInfo != null) {
      if (uploadInfo.getDuplicatesUploadId() != null) {
        // Delete the child's own data file if it exists
        try {
          Path childDataPath = getPathInUploadDir(uploadInfo.getId(), DATA_FILE);
          Files.deleteIfExists(childDataPath);
        } catch (UploadNotFoundException e) {
          // It doesn't exist yet, which is fine
        }

        // Retrieve the parent UploadInfo
        UploadId parentId = uploadInfo.getDuplicatesUploadId();
        UploadInfo parentInfo = getUploadInfo(parentId);
        if (parentInfo != null) {
          Path parentBytesPath = null;
          try {
            parentBytesPath = getPathInUploadDir(parentId, DATA_FILE);
          } catch (UploadNotFoundException e) {
            // Parent file is not found
          }

          if (parentBytesPath != null && Files.exists(parentBytesPath)) {
            // Ensure parent's expiration timestamp is >= child's expiration timestamp
            Long childExpire = uploadInfo.getExpirationTimestamp();
            Long parentExpire = parentInfo.getExpirationTimestamp();
            if (childExpire == null) {
              if (parentExpire != null) {
                parentInfo.setExpirationTimestamp(null);
                Path parentInfoPath = getInfoPath(parentId);
                Utils.writeSerializable(parentInfo, parentInfoPath);
              }
            } else {
              if (parentExpire != null && parentExpire < childExpire) {
                parentInfo.setExpirationTimestamp(childExpire);
                Path parentInfoPath = getInfoPath(parentId);
                Utils.writeSerializable(parentInfo, parentInfoPath);
              }
            }
          }
        }
      } else if (isUploadDeduplicationEnabled()
          && !uploadInfo.isUploadInProgress()
          && uploadInfo.getChecksum() != null
          && uploadInfo.getChecksumAlgorithm() != null) {
        // Index the checksum
        Path checksumFile =
            getStoragePath()
                .getParent()
                .resolve("checksums")
                .resolve(uploadInfo.getChecksumAlgorithm().toString())
                .resolve(uploadInfo.getChecksum());
        Files.createDirectories(checksumFile.getParent());
        Files.write(checksumFile, uploadInfo.getId().toString().getBytes(StandardCharsets.UTF_8));
      }

      Path infoPath = getInfoPath(uploadInfo.getId());
      Utils.writeSerializable(uploadInfo, infoPath);
    }
  }

  @Override
  public UploadInfo append(UploadInfo info, InputStream inputStream)
      throws IOException, TusException {
    if (info != null) {
      Path bytesPath = getBytesPath(info.getId());

      long max = getMaxUploadSize() > 0 ? getMaxUploadSize() : Long.MAX_VALUE;
      long transferred = 0;
      Long offset = info.getOffset();
      long newOffset = offset;

      try (ReadableByteChannel uploadedBytes = Channels.newChannel(inputStream);
          FileChannel file = FileChannel.open(bytesPath, WRITE)) {

        try {
          // Lock will be released when the channel closes
          file.lock();

          // Validate that the given offset is at the end of the file
          if (!offset.equals(file.size())) {
            throw new InvalidUploadOffsetException(
                "The upload offset does not correspond to the written"
                    + " bytes. You can only append to the end of an upload");
          }

          // write all bytes in the channel up to the configured maximum
          transferred = file.transferFrom(uploadedBytes, offset, max - offset);
          file.force(true);
          newOffset = offset + transferred;

        } catch (Exception ex) {
          // An error occurred, try to write as much data as possible
          newOffset = writeAsMuchAsPossible(file);
          throw ex;
        }

      } finally {
        info.setOffset(newOffset);
        update(info);
      }
    }

    return info;
  }

  @Override
  public void removeLastNumberOfBytes(UploadInfo info, long byteCount)
      throws UploadNotFoundException, IOException {

    if (info != null && byteCount > 0) {
      Path bytesPath = getBytesPath(info.getId());

      try (FileChannel file = FileChannel.open(bytesPath, WRITE)) {

        // Lock will be released when the channel closes
        file.lock();

        file.truncate(file.size() - byteCount);
        file.force(true);

        info.setOffset(file.size());
        update(info);
      }
    }
  }

  @Override
  public void terminateUpload(UploadInfo info) throws UploadNotFoundException, IOException {
    if (info != null) {
      if (info.getDuplicatesUploadId() == null
          && info.getChecksum() != null
          && info.getChecksumAlgorithm() != null) {
        Path checksumFile =
            getStoragePath()
                .getParent()
                .resolve("checksums")
                .resolve(info.getChecksumAlgorithm().toString())
                .resolve(info.getChecksum());
        Files.deleteIfExists(checksumFile);
      }
      Path uploadPath = getPathInStorageDirectory(info.getId());
      FileUtils.deleteDirectory(uploadPath.toFile());
    }
  }

  @Override
  public Long getUploadExpirationPeriod() {
    return uploadExpirationPeriod;
  }

  @Override
  public void setUploadExpirationPeriod(Long uploadExpirationPeriod) {
    this.uploadExpirationPeriod = uploadExpirationPeriod;
  }

  @Override
  public void setUploadConcatenationService(UploadConcatenationService concatenationService) {
    Validate.notNull(concatenationService);
    this.uploadConcatenationService = concatenationService;
  }

  @Override
  public UploadConcatenationService getUploadConcatenationService() {
    return uploadConcatenationService;
  }

  @Override
  public InputStream getUploadedBytes(String uploadUri, String ownerKey)
      throws IOException, UploadNotFoundException {

    UploadId id = idFactory.readUploadId(uploadUri);

    UploadInfo uploadInfo = getUploadInfo(id);
    if (uploadInfo == null || !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {
      throw new UploadNotFoundException(
          "The upload with id " + id + " could not be found for owner " + ownerKey);
    } else {
      return getUploadedBytes(id);
    }
  }

  @Override
  public InputStream getUploadedBytes(UploadId id) throws IOException, UploadNotFoundException {
    InputStream inputStream = null;
    UploadInfo uploadInfo = getUploadInfo(id);
    if (uploadInfo == null) {
      throw new UploadNotFoundException("The upload with id " + id + " was not found.");
    }

    if (uploadInfo.getDuplicatesUploadId() != null) {
      return getUploadedBytes(uploadInfo.getDuplicatesUploadId());
    }

    if (UploadType.CONCATENATED.equals(uploadInfo.getUploadType())
        && uploadConcatenationService != null) {
      inputStream = uploadConcatenationService.getConcatenatedBytes(uploadInfo);

    } else {
      Path bytesPath = getBytesPath(id);
      // If bytesPath is not null, we know this is a valid Upload URI
      if (bytesPath != null) {
        if (!Files.exists(bytesPath)) {
          throw new UploadNotFoundException(
              "The upload bytes for id " + id + " could not be found.");
        }
        inputStream = Channels.newInputStream(FileChannel.open(bytesPath, READ));
      }
    }

    return inputStream;
  }

  @Override
  public void copyUploadTo(UploadInfo info, OutputStream outputStream)
      throws UploadNotFoundException, IOException {

    List<UploadInfo> uploads = getUploads(info);

    try (WritableByteChannel outputChannel = Channels.newChannel(outputStream)) {

      for (UploadInfo upload : uploads) {
        if (upload == null) {
          log.warn("We cannot copy the bytes of an upload that does not exist");

        } else if (upload.isUploadInProgress()) {
          log.warn(
              "We cannot copy the bytes of upload {} because it is still in progress",
              upload.getId());

        } else {
          UploadId readId =
              upload.getDuplicatesUploadId() != null
                  ? upload.getDuplicatesUploadId()
                  : upload.getId();
          Path bytesPath = getBytesPath(readId);
          if (!Files.exists(bytesPath)) {
            throw new UploadNotFoundException(
                "The upload bytes for id " + readId + " could not be found.");
          }
          try (FileChannel file = FileChannel.open(bytesPath, READ)) {
            // Efficiently copy the bytes to the output stream
            file.transferTo(0, upload.getLength(), outputChannel);
          }
        }
      }
    }
  }

  @Override
  public void cleanupExpiredUploads(UploadLockingService uploadLockingService) throws IOException {
    try (DirectoryStream<Path> expiredUploadsStream =
        Files.newDirectoryStream(
            getStoragePath(), new ExpiredUploadFilter(this, uploadLockingService))) {

      for (Path path : expiredUploadsStream) {
        FileUtils.deleteDirectory(path.toFile());
      }
    }
  }

  private List<UploadInfo> getUploads(UploadInfo info) throws IOException, UploadNotFoundException {
    List<UploadInfo> uploads;

    if (info != null
        && UploadType.CONCATENATED.equals(info.getUploadType())
        && uploadConcatenationService != null) {
      uploadConcatenationService.merge(info);
      uploads = uploadConcatenationService.getPartialUploads(info);
    } else {
      uploads = Collections.singletonList(info);
    }
    return uploads;
  }

  private Path getBytesPath(UploadId id) throws UploadNotFoundException {
    return getPathInUploadDir(id, DATA_FILE);
  }

  private Path getInfoPath(UploadId id) throws UploadNotFoundException {
    return getPathInUploadDir(id, INFO_FILE);
  }

  private Path createUploadDirectory(UploadId id) throws IOException {
    return Files.createDirectories(getPathInStorageDirectory(id));
  }

  private Path getPathInUploadDir(UploadId id, String fileName) throws UploadNotFoundException {
    // Get the upload directory
    Path uploadDir = getPathInStorageDirectory(id);
    if (uploadDir != null && Files.exists(uploadDir)) {
      return uploadDir.resolve(fileName);
    } else {
      throw new UploadNotFoundException("The upload for id " + id + " was not found.");
    }
  }

  private synchronized UploadId createNewId() throws IOException {
    UploadId id;
    do {
      id = idFactory.createId();
      // For extra safety, double check that this ID is not in use yet
    } while (getUploadInfo(id) != null);
    return id;
  }

  private long writeAsMuchAsPossible(FileChannel file) throws IOException {
    long offset = 0;
    if (file != null) {
      file.force(true);
      offset = file.size();
    }
    return offset;
  }
}

1	package me.desair.tus.server.upload.disk;
2
3	import static java.nio.file.StandardOpenOption.READ;
4	import static java.nio.file.StandardOpenOption.WRITE;
5
6	import java.io.File;
7	import java.io.IOException;
8	import java.io.InputStream;
9	import java.io.OutputStream;
10	import java.nio.channels.Channels;
11	import java.nio.channels.FileChannel;
12	import java.nio.channels.ReadableByteChannel;
13	import java.nio.channels.WritableByteChannel;
14	import java.nio.charset.StandardCharsets;
15	import java.nio.file.DirectoryStream;
16	import java.nio.file.Files;
17	import java.nio.file.Path;
18	import java.util.Collections;
19	import java.util.List;
20	import java.util.Objects;
21	import me.desair.tus.server.checksum.ChecksumAlgorithm;
22	import me.desair.tus.server.exception.InvalidUploadOffsetException;
23	import me.desair.tus.server.exception.TusException;
24	import me.desair.tus.server.exception.UploadNotFoundException;
25	import me.desair.tus.server.upload.UploadId;
26	import me.desair.tus.server.upload.UploadIdFactory;
27	import me.desair.tus.server.upload.UploadInfo;
28	import me.desair.tus.server.upload.UploadLockingService;
29	import me.desair.tus.server.upload.UploadStorageService;
30	import me.desair.tus.server.upload.UploadType;
31	import me.desair.tus.server.upload.concatenation.UploadConcatenationService;
32	import me.desair.tus.server.upload.concatenation.VirtualConcatenationService;
33	import me.desair.tus.server.util.Utils;
34	import org.apache.commons.io.FileUtils;
35	import org.apache.commons.lang3.Validate;
36	import org.slf4j.Logger;
37	import org.slf4j.LoggerFactory;
38
39	/** Implementation of {@link UploadStorageService} that implements storage on disk. */
40	public class DiskStorageService extends AbstractDiskBasedService implements UploadStorageService {
41
42	private static final Logger log = LoggerFactory.getLogger(DiskStorageService.class);	8✔
43
44	private static final String UPLOAD_SUB_DIRECTORY = "uploads";
45	private static final String INFO_FILE = "info";
46	private static final String DATA_FILE = "data";
47
48	private Long maxUploadSize = null;	6✔
49	private Long uploadExpirationPeriod = null;	6✔
50	private UploadIdFactory idFactory;
51	private UploadConcatenationService uploadConcatenationService;
52	private boolean isUploadDeduplicationEnabled = false;	6✔
53
54	public DiskStorageService(String storagePath) {
55	super(storagePath + File.separator + UPLOAD_SUB_DIRECTORY);	10✔
56	setUploadConcatenationService(new VirtualConcatenationService(this));	12✔
57	}	2✔
58
59	public DiskStorageService(UploadIdFactory idFactory, String storagePath) {
60	this(storagePath);	6✔
61	Validate.notNull(idFactory, "The IdFactory cannot be null");	12✔
62	this.idFactory = idFactory;	6✔
63	}	2✔
64
65	@Override
66	public void setIdFactory(UploadIdFactory idFactory) {
67	Validate.notNull(idFactory, "The IdFactory cannot be null");	6✔
68	this.idFactory = idFactory;	3✔
69	}	1✔
70
71	@Override
72	public void setMaxUploadSize(Long maxUploadSize) {
73	this.maxUploadSize = (maxUploadSize != null && maxUploadSize > 0 ? maxUploadSize : 0);	28✔
74	}	2✔
75
76	@Override
77	public long getMaxUploadSize() {
78	return maxUploadSize == null ? 0 : maxUploadSize;	18✔
79	}
80
81	@Override
82	public void setUploadDeduplicationEnabled(boolean enabled) {
83	this.isUploadDeduplicationEnabled = enabled;	6✔
84	}	2✔
85
86	@Override
87	public boolean isUploadDeduplicationEnabled() {
88	return this.isUploadDeduplicationEnabled;	6✔
89	}
90
91	@Override
92	public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm algorithm)
93	throws IOException {
94	if (checksum == null \|\| algorithm == null) {	8✔
95	return null;	2✔
96	}
97	Path checksumFile =	2✔
98	getStoragePath()	2✔
99	.getParent()	4✔
100	.resolve("checksums")	4✔
101	.resolve(algorithm.toString())	6✔
102	.resolve(checksum);	4✔
103	if (Files.exists(checksumFile)) {	10✔
104	String uploadIdStr =	6✔
105	new String(Files.readAllBytes(checksumFile), StandardCharsets.UTF_8).trim();	10✔
106	UploadId id = new UploadId(uploadIdStr);	10✔
107	UploadInfo info = getUploadInfo(id);	8✔
108	if (info == null) {	4✔
109	Files.deleteIfExists(checksumFile);	3✔
110	return null;	2✔
111	}
112	Path bytesPath = null;	4✔
113	try {
114	bytesPath = getPathInUploadDir(id, DATA_FILE);	10✔
NEW 115	} catch (UploadNotFoundException e) {	×
116	// file doesn't exist
117	}	2✔
118	if (bytesPath == null \|\| !Files.exists(bytesPath)) {	14!
119	Files.deleteIfExists(checksumFile);	3✔
120	return null;	2✔
121	}
122	return info;	4✔
123	}
124	return null;	4✔
125	}
126
127	@Override
128	public UploadInfo getUploadInfo(String uploadUrl, String ownerKey) throws IOException {
129	UploadInfo uploadInfo = getUploadInfo(idFactory.readUploadId(uploadUrl));	14✔
130	if (uploadInfo == null \|\| !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {	14✔
131	return null;	4✔
132	} else {
133	return uploadInfo;	4✔
134	}
135	}
136
137	@Override
138	public UploadInfo getUploadInfo(UploadId id) throws IOException {
139	try {
140	Path infoPath = getInfoPath(id);	8✔
141	if (infoPath == null \|\| !Files.exists(infoPath)) {	14!
142	return null;	2✔
143	}
144	return Utils.readSerializable(infoPath, UploadInfo.class);	10✔
145	} catch (UploadNotFoundException e) {	2✔
146	return null;	4✔
147	}
148	}
149
150	@Override
151	public String getUploadUri() {
152	return idFactory.getUploadUri();	8✔
153	}
154
155	@Override
156	public UploadInfo create(UploadInfo info, String ownerKey) throws IOException {
157	UploadId id = createNewId();	6✔
158
159	createUploadDirectory(id);	8✔
160
161	try {
162	Path bytesPath = getBytesPath(id);	8✔
163
164	// Create an empty file to storage the bytes of this upload
165	Files.createFile(bytesPath);	10✔
166
167	// Set starting values
168	info.setId(id);	6✔
169	info.setOffset(0L);	8✔
170	info.setOwnerKey(ownerKey);	6✔
171
172	update(info);	6✔
173
174	return info;	4✔
UNCOV 175	} catch (UploadNotFoundException e) {	×
176	// Normally this cannot happen
UNCOV 177	log.error("Unable to create UploadInfo because of an upload not found exception", e);	×
UNCOV 178	return null;	×
179	}
180	}
181
182	@Override
183	public void update(UploadInfo uploadInfo) throws IOException, UploadNotFoundException {
184	if (uploadInfo != null) {	4!
185	if (uploadInfo.getDuplicatesUploadId() != null) {	6✔
186	// Delete the child's own data file if it exists
187	try {
188	Path childDataPath = getPathInUploadDir(uploadInfo.getId(), DATA_FILE);	12✔
189	Files.deleteIfExists(childDataPath);	6✔
NEW 190	} catch (UploadNotFoundException e) {	×
191	// It doesn't exist yet, which is fine
192	}	2✔
193
194	// Retrieve the parent UploadInfo
195	UploadId parentId = uploadInfo.getDuplicatesUploadId();	6✔
196	UploadInfo parentInfo = getUploadInfo(parentId);	8✔
197	if (parentInfo != null) {	4!
198	Path parentBytesPath = null;	4✔
199	try {
200	parentBytesPath = getPathInUploadDir(parentId, DATA_FILE);	10✔
NEW 201	} catch (UploadNotFoundException e) {	×
202	// Parent file is not found
203	}	2✔
204
205	if (parentBytesPath != null && Files.exists(parentBytesPath)) {	14!
206	// Ensure parent's expiration timestamp is >= child's expiration timestamp
207	Long childExpire = uploadInfo.getExpirationTimestamp();	6✔
208	Long parentExpire = parentInfo.getExpirationTimestamp();	6✔
209	if (childExpire == null) {	4✔
210	if (parentExpire != null) {	2✔
211	parentInfo.setExpirationTimestamp(null);	3✔
212	Path parentInfoPath = getInfoPath(parentId);	4✔
213	Utils.writeSerializable(parentInfo, parentInfoPath);	3✔
214	}	1✔
215	} else {
216	if (parentExpire != null && parentExpire < childExpire) {	16!
217	parentInfo.setExpirationTimestamp(childExpire);	6✔
218	Path parentInfoPath = getInfoPath(parentId);	8✔
219	Utils.writeSerializable(parentInfo, parentInfoPath);	6✔
220	}
221	}
222	}
223	}
224	} else if (isUploadDeduplicationEnabled()	10✔
225	&& !uploadInfo.isUploadInProgress()	6✔
226	&& uploadInfo.getChecksum() != null	6✔
227	&& uploadInfo.getChecksumAlgorithm() != null) {	4!
228	// Index the checksum
229	Path checksumFile =	2✔
230	getStoragePath()	2✔
231	.getParent()	4✔
232	.resolve("checksums")	4✔
233	.resolve(uploadInfo.getChecksumAlgorithm().toString())	8✔
234	.resolve(uploadInfo.getChecksum());	6✔
235	Files.createDirectories(checksumFile.getParent());	12✔
236	Files.write(checksumFile, uploadInfo.getId().toString().getBytes(StandardCharsets.UTF_8));	20✔
237	}
238
239	Path infoPath = getInfoPath(uploadInfo.getId());	10✔
240	Utils.writeSerializable(uploadInfo, infoPath);	6✔
241	}
242	}	2✔
243
244	@Override
245	public UploadInfo append(UploadInfo info, InputStream inputStream)
246	throws IOException, TusException {
247	if (info != null) {	4!
248	Path bytesPath = getBytesPath(info.getId());	10✔
249
250	long max = getMaxUploadSize() > 0 ? getMaxUploadSize() : Long.MAX_VALUE;	20✔
251	long transferred = 0;	4✔
252	Long offset = info.getOffset();	6✔
253	long newOffset = offset;	6✔
254
255	try (ReadableByteChannel uploadedBytes = Channels.newChannel(inputStream);	6✔
256	FileChannel file = FileChannel.open(bytesPath, WRITE)) {	18✔
257
258	try {
259	// Lock will be released when the channel closes
260	file.lock();	6✔
261
262	// Validate that the given offset is at the end of the file
263	if (!offset.equals(file.size())) {	12✔
264	throw new InvalidUploadOffsetException(	5✔
265	"The upload offset does not correspond to the written"
266	+ " bytes. You can only append to the end of an upload");
267	}
268
269	// write all bytes in the channel up to the configured maximum
270	transferred = file.transferFrom(uploadedBytes, offset, max - offset);	20✔
271	file.force(true);	6✔
272	newOffset = offset + transferred;	10✔
273
274	} catch (Exception ex) {	1✔
275	// An error occurred, try to write as much data as possible
276	newOffset = writeAsMuchAsPossible(file);	4✔
277	throw ex;	2✔
278	}	2✔
279
280	} finally {
281	info.setOffset(newOffset);	8✔
282	update(info);	6✔
283	}
284	}
285
286	return info;	4✔
287	}
288
289	@Override
290	public void removeLastNumberOfBytes(UploadInfo info, long byteCount)
291	throws UploadNotFoundException, IOException {
292
293	if (info != null && byteCount > 0) {	12✔
294	Path bytesPath = getBytesPath(info.getId());	10✔
295
296	try (FileChannel file = FileChannel.open(bytesPath, WRITE)) {	18✔
297
298	// Lock will be released when the channel closes
299	file.lock();	6✔
300
301	file.truncate(file.size() - byteCount);	14✔
302	file.force(true);	6✔
303
304	info.setOffset(file.size());	10✔
305	update(info);	6✔
306	}
307	}
308	}	2✔
309
310	@Override
311	public void terminateUpload(UploadInfo info) throws UploadNotFoundException, IOException {
312	if (info != null) {	4✔
313	if (info.getDuplicatesUploadId() == null	8!
314	&& info.getChecksum() != null	6✔
315	&& info.getChecksumAlgorithm() != null) {	4!
316	Path checksumFile =	2✔
317	getStoragePath()	2✔
318	.getParent()	4✔
319	.resolve("checksums")	4✔
320	.resolve(info.getChecksumAlgorithm().toString())	8✔
321	.resolve(info.getChecksum());	6✔
322	Files.deleteIfExists(checksumFile);	6✔
323	}
324	Path uploadPath = getPathInStorageDirectory(info.getId());	10✔
325	FileUtils.deleteDirectory(uploadPath.toFile());	6✔
326	}
327	}	2✔
328
329	@Override
330	public Long getUploadExpirationPeriod() {
331	return uploadExpirationPeriod;	3✔
332	}
333
334	@Override
335	public void setUploadExpirationPeriod(Long uploadExpirationPeriod) {
336	this.uploadExpirationPeriod = uploadExpirationPeriod;	3✔
337	}	1✔
338
339	@Override
340	public void setUploadConcatenationService(UploadConcatenationService concatenationService) {
341	Validate.notNull(concatenationService);	6✔
342	this.uploadConcatenationService = concatenationService;	6✔
343	}	2✔
344
345	@Override
346	public UploadConcatenationService getUploadConcatenationService() {
347	return uploadConcatenationService;	3✔
348	}
349
350	@Override
351	public InputStream getUploadedBytes(String uploadUri, String ownerKey)
352	throws IOException, UploadNotFoundException {
353
354	UploadId id = idFactory.readUploadId(uploadUri);	10✔
355
356	UploadInfo uploadInfo = getUploadInfo(id);	8✔
357	if (uploadInfo == null \|\| !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {	14!
358	throw new UploadNotFoundException(	8✔
359	"The upload with id " + id + " could not be found for owner " + ownerKey);
360	} else {
361	return getUploadedBytes(id);	8✔
362	}
363	}
364
365	@Override
366	public InputStream getUploadedBytes(UploadId id) throws IOException, UploadNotFoundException {
367	InputStream inputStream = null;	4✔
368	UploadInfo uploadInfo = getUploadInfo(id);	8✔
369	if (uploadInfo == null) {	4✔
370	throw new UploadNotFoundException("The upload with id " + id + " was not found.");	7✔
371	}
372
373	if (uploadInfo.getDuplicatesUploadId() != null) {	6✔
374	return getUploadedBytes(uploadInfo.getDuplicatesUploadId());	5✔
375	}
376
377	if (UploadType.CONCATENATED.equals(uploadInfo.getUploadType())	13!
378	&& uploadConcatenationService != null) {
379	inputStream = uploadConcatenationService.getConcatenatedBytes(uploadInfo);	6✔
380
381	} else {
382	Path bytesPath = getBytesPath(id);	8✔
383	// If bytesPath is not null, we know this is a valid Upload URI
384	if (bytesPath != null) {	4!
385	if (!Files.exists(bytesPath)) {	10!
NEW UNCOV 386	throw new UploadNotFoundException(	×
387	"The upload bytes for id " + id + " could not be found.");
388	}
389	inputStream = Channels.newInputStream(FileChannel.open(bytesPath, READ));	20✔
390	}
391	}
392
393	return inputStream;	4✔
394	}
395
396	@Override
397	public void copyUploadTo(UploadInfo info, OutputStream outputStream)
398	throws UploadNotFoundException, IOException {
399
400	List<UploadInfo> uploads = getUploads(info);	8✔
401
402	try (WritableByteChannel outputChannel = Channels.newChannel(outputStream)) {	6✔
403
404	for (UploadInfo upload : uploads) {	20✔
405	if (upload == null) {	4!
406	log.warn("We cannot copy the bytes of an upload that does not exist");	×
407
408	} else if (upload.isUploadInProgress()) {	6!
UNCOV 409	log.warn(	×
410	"We cannot copy the bytes of upload {} because it is still in progress",
UNCOV 411	upload.getId());	×
412
413	} else {
414	UploadId readId =
415	upload.getDuplicatesUploadId() != null	6✔
416	? upload.getDuplicatesUploadId()	6✔
417	: upload.getId();	6✔
418	Path bytesPath = getBytesPath(readId);	8✔
419	if (!Files.exists(bytesPath)) {	10!
NEW UNCOV 420	throw new UploadNotFoundException(	×
421	"The upload bytes for id " + readId + " could not be found.");
422	}
423	try (FileChannel file = FileChannel.open(bytesPath, READ)) {	18✔
424	// Efficiently copy the bytes to the output stream
425	file.transferTo(0, upload.getLength(), outputChannel);	16✔
426	}
427	}
428	}	2✔
429	}
430	}	2✔
431
432	@Override
433	public void cleanupExpiredUploads(UploadLockingService uploadLockingService) throws IOException {
434	try (DirectoryStream<Path> expiredUploadsStream =	2✔
435	Files.newDirectoryStream(	4✔
436	getStoragePath(), new ExpiredUploadFilter(this, uploadLockingService))) {	12✔
437
438	for (Path path : expiredUploadsStream) {	20✔
439	FileUtils.deleteDirectory(path.toFile());	6✔
440	}	2✔
441	}
442	}	2✔
443
444	private List<UploadInfo> getUploads(UploadInfo info) throws IOException, UploadNotFoundException {
445	List<UploadInfo> uploads;
446
447	if (info != null	8!
448	&& UploadType.CONCATENATED.equals(info.getUploadType())	9!
449	&& uploadConcatenationService != null) {
450	uploadConcatenationService.merge(info);	4✔
451	uploads = uploadConcatenationService.getPartialUploads(info);	6✔
452	} else {
453	uploads = Collections.singletonList(info);	6✔
454	}
455	return uploads;	4✔
456	}
457
458	private Path getBytesPath(UploadId id) throws UploadNotFoundException {
459	return getPathInUploadDir(id, DATA_FILE);	10✔
460	}
461
462	private Path getInfoPath(UploadId id) throws UploadNotFoundException {
463	return getPathInUploadDir(id, INFO_FILE);	10✔
464	}
465
466	private Path createUploadDirectory(UploadId id) throws IOException {
467	return Files.createDirectories(getPathInStorageDirectory(id));	14✔
468	}
469
470	private Path getPathInUploadDir(UploadId id, String fileName) throws UploadNotFoundException {
471	// Get the upload directory
472	Path uploadDir = getPathInStorageDirectory(id);	8✔
473	if (uploadDir != null && Files.exists(uploadDir)) {	14✔
474	return uploadDir.resolve(fileName);	8✔
475	} else {
476	throw new UploadNotFoundException("The upload for id " + id + " was not found.");	14✔
477	}
478	}
479
480	private synchronized UploadId createNewId() throws IOException {
481	UploadId id;
482	do {
483	id = idFactory.createId();	8✔
484	// For extra safety, double check that this ID is not in use yet
485	} while (getUploadInfo(id) != null);	8!
486	return id;	4✔
487	}
488
489	private long writeAsMuchAsPossible(FileChannel file) throws IOException {
490	long offset = 0;	2✔
491	if (file != null) {	2!
492	file.force(true);	3✔
493	offset = file.size();	3✔
494	}
495	return offset;	2✔
496	}
497	}

tomdesair / tus-java-server / 27440598708

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous