Build has been canceled!

28898324220

Committed 07 Jul 2026 09:00PM UTC coverage: 95.729% (+0.9%) from 94.813%

Build # 28898324220

Build Type

Pull #105

github

Committed by

web-flow

Commit Message

Merge 287f920bd into 4b2f45149

Pull Request Pull Request #105: WIP feat: Implement IETF Resumable Uploads for HTTP (RUFH) Protocol

Coverage Stats

1103 of 1206 branches covered (91.46%)

Branch coverage included in aggregate %.

766 of 777 new or added lines in 42 files covered. (98.58%)

2573 of 2634 relevant lines covered (97.68%)

6.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.8

/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java

package me.desair.tus.server.upload.disk;

import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import me.desair.tus.server.checksum.ChecksumAlgorithm;
import me.desair.tus.server.exception.InvalidUploadOffsetException;
import me.desair.tus.server.exception.TusException;
import me.desair.tus.server.exception.UploadNotFoundException;
import me.desair.tus.server.upload.UploadId;
import me.desair.tus.server.upload.UploadIdFactory;
import me.desair.tus.server.upload.UploadInfo;
import me.desair.tus.server.upload.UploadLockingService;
import me.desair.tus.server.upload.UploadStorageService;
import me.desair.tus.server.upload.UploadType;
import me.desair.tus.server.upload.concatenation.UploadConcatenationService;
import me.desair.tus.server.upload.concatenation.VirtualConcatenationService;
import me.desair.tus.server.util.Utils;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** Implementation of {@link UploadStorageService} that implements storage on disk. */
public class DiskStorageService extends AbstractDiskBasedService implements UploadStorageService {

  private static final Logger log = LoggerFactory.getLogger(DiskStorageService.class);

  private static final String UPLOAD_SUB_DIRECTORY = "uploads";
  private static final String INFO_FILE = "info";
  private static final String DATA_FILE = "data";

  private Long maxUploadSize = null;
  private Long maxAppendSize = null;
  private Long uploadExpirationPeriod = null;
  private UploadIdFactory idFactory;
  private UploadConcatenationService uploadConcatenationService;
  private boolean isUploadDeduplicationEnabled = false;

  public DiskStorageService(String storagePath) {
    super(storagePath + File.separator + UPLOAD_SUB_DIRECTORY);
    setUploadConcatenationService(new VirtualConcatenationService(this));
  }

  public DiskStorageService(UploadIdFactory idFactory, String storagePath) {
    this(storagePath);
    Objects.requireNonNull(idFactory, "The IdFactory cannot be null");
    this.idFactory = idFactory;
  }

  @Override
  public void setIdFactory(UploadIdFactory idFactory) {
    Objects.requireNonNull(idFactory, "The IdFactory cannot be null");
    this.idFactory = idFactory;
  }

  @Override
  public void setMaxUploadSize(Long maxUploadSize) {
    this.maxUploadSize = (maxUploadSize != null && maxUploadSize > 0 ? maxUploadSize : 0);
  }

  @Override
  public long getMaxUploadSize() {
    return maxUploadSize == null ? 0 : maxUploadSize;
  }

  @Override
  public void setMaxAppendSize(Long maxAppendSize) {
    this.maxAppendSize = (maxAppendSize != null && maxAppendSize > 0 ? maxAppendSize : null);
  }

  @Override
  public Long getMaxAppendSize() {
    if (maxAppendSize != null && maxAppendSize > 0) {
      return maxAppendSize;
    }
    long maxUpload = getMaxUploadSize();
    return maxUpload > 0 ? maxUpload : null;
  }

  @Override
  public void setUploadDeduplicationEnabled(boolean enabled) {
    this.isUploadDeduplicationEnabled = enabled;
  }

  @Override
  public boolean isUploadDeduplicationEnabled() {
    return this.isUploadDeduplicationEnabled;
  }

  @Override
  public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm algorithm)
      throws IOException {
    if (checksum == null || algorithm == null) {
      return null;
    }
    Path checksumFile = getChecksumPath(checksum, algorithm);
    if (Files.exists(checksumFile)) {
      String uploadIdStr =
          new String(Files.readAllBytes(checksumFile), StandardCharsets.UTF_8).trim();
      UploadId id = new UploadId(uploadIdStr);
      UploadInfo info = getUploadInfo(id);
      if (info == null) {
        Files.deleteIfExists(checksumFile);
        return null;
      }
      Path bytesPath = null;
      try {
        bytesPath = getPathInUploadDir(id, DATA_FILE);
      } catch (UploadNotFoundException e) {
        // file doesn't exist
      }
      if (bytesPath == null || !Files.exists(bytesPath)) {
        Files.deleteIfExists(checksumFile);
        return null;
      }
      return info;
    }
    return null;
  }

  @Override
  public UploadInfo getUploadInfo(String uploadUrl, String ownerKey) throws IOException {
    UploadInfo uploadInfo = getUploadInfo(idFactory.readUploadId(uploadUrl));
    if (uploadInfo == null || !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {
      return null;
    } else {
      return uploadInfo;
    }
  }

  @Override
  public UploadInfo getUploadInfo(UploadId id) throws IOException {
    try {
      Path infoPath = getInfoPath(id);
      if (infoPath == null || !Files.exists(infoPath)) {
        return null;
      }
      return Utils.readSerializable(infoPath, UploadInfo.class);
    } catch (UploadNotFoundException e) {
      return null;
    }
  }

  @Override
  public String getUploadUri() {
    return idFactory.getUploadUri();
  }

  @Override
  public UploadInfo create(UploadInfo info, String ownerKey) throws IOException {
    UploadId id = createNewId();

    createUploadDirectory(id);

    try {
      Path bytesPath = getBytesPath(id);

      // Create an empty file to storage the bytes of this upload
      Files.createFile(bytesPath);

      // Set starting values
      info.setId(id);
      info.setOffset(0L);
      info.setOwnerKey(ownerKey);

      update(info);

      return info;
    } catch (UploadNotFoundException e) {
      // Normally this cannot happen
      log.error("Unable to create UploadInfo because of an upload not found exception", e);
      return null;
    }
  }

  @Override
  public void update(UploadInfo uploadInfo) throws IOException, UploadNotFoundException {
    if (uploadInfo != null) {
      if (uploadInfo.getDuplicatesUploadId() != null) {
        // Delete the child's own data file if it exists
        try {
          Path childDataPath = getPathInUploadDir(uploadInfo.getId(), DATA_FILE);
          Files.deleteIfExists(childDataPath);
        } catch (UploadNotFoundException e) {
          // It doesn't exist yet, which is fine
        }

        // Retrieve the parent UploadInfo
        UploadId parentId = uploadInfo.getDuplicatesUploadId();
        UploadInfo parentInfo = getUploadInfo(parentId);
        if (parentInfo != null) {
          Path parentBytesPath = null;
          try {
            parentBytesPath = getPathInUploadDir(parentId, DATA_FILE);
          } catch (UploadNotFoundException e) {
            // Parent file is not found
          }

          if (parentBytesPath != null && Files.exists(parentBytesPath)) {
            // Ensure parent's expiration timestamp is >= child's expiration timestamp
            Long childExpire = uploadInfo.getExpirationTimestamp();
            Long parentExpire = parentInfo.getExpirationTimestamp();
            if (childExpire == null) {
              if (parentExpire != null) {
                parentInfo.setExpirationTimestamp(null);
                Path parentInfoPath = getInfoPath(parentId);
                Utils.writeSerializable(parentInfo, parentInfoPath);
              }
            } else {
              if (parentExpire != null && parentExpire < childExpire) {
                parentInfo.setExpirationTimestamp(childExpire);
                Path parentInfoPath = getInfoPath(parentId);
                Utils.writeSerializable(parentInfo, parentInfoPath);
              }
            }
          }
        }
      } else if (isUploadDeduplicationEnabled()
          && !uploadInfo.isUploadInProgress()
          && uploadInfo.getChecksum() != null
          && uploadInfo.getChecksumAlgorithm() != null) {
        // Index the checksum
        Path checksumFile =
            getChecksumPath(uploadInfo.getChecksum(), uploadInfo.getChecksumAlgorithm());
        Files.createDirectories(checksumFile.getParent());
        Files.write(checksumFile, uploadInfo.getId().toString().getBytes(StandardCharsets.UTF_8));
      }

      Path infoPath = getInfoPath(uploadInfo.getId());
      Utils.writeSerializable(uploadInfo, infoPath);
    }
  }

  @Override
  public UploadInfo append(UploadInfo info, InputStream inputStream)
      throws IOException, TusException {
    if (info != null) {
      Path bytesPath = getBytesPath(info.getId());

      long max = getMaxUploadSize() > 0 ? getMaxUploadSize() : Long.MAX_VALUE;
      long transferred = 0;
      Long offset = info.getOffset();
      long newOffset = offset;

      try (ReadableByteChannel uploadedBytes = Channels.newChannel(inputStream);
          FileChannel file = FileChannel.open(bytesPath, WRITE)) {

        try {
          // Lock will be released when the channel closes
          file.lock();

          // Validate that the given offset is at the end of the file
          if (!offset.equals(file.size())) {
            throw new InvalidUploadOffsetException(
                "The upload offset does not correspond to the written"
                    + " bytes. You can only append to the end of an upload");
          }

          // write all bytes in the channel up to the configured maximum
          transferred = file.transferFrom(uploadedBytes, offset, max - offset);
          file.force(true);
          newOffset = offset + transferred;

        } catch (Exception ex) {
          // An error occurred, try to write as much data as possible
          newOffset = writeAsMuchAsPossible(file);
          throw ex;
        }

      } finally {
        info.setOffset(newOffset);
        update(info);
      }
    }

    return info;
  }

  @Override
  public void removeLastNumberOfBytes(UploadInfo info, long byteCount)
      throws UploadNotFoundException, IOException {

    if (info != null && byteCount > 0) {
      Path bytesPath = getBytesPath(info.getId());

      try (FileChannel file = FileChannel.open(bytesPath, WRITE)) {

        // Lock will be released when the channel closes
        file.lock();

        file.truncate(file.size() - byteCount);
        file.force(true);

        info.setOffset(file.size());
        update(info);
      }
    }
  }

  @Override
  public void terminateUpload(UploadInfo info) throws UploadNotFoundException, IOException {
    if (info != null) {
      if (info.getDuplicatesUploadId() == null
          && info.getChecksum() != null
          && info.getChecksumAlgorithm() != null) {
        Path checksumFile = getChecksumPath(info.getChecksum(), info.getChecksumAlgorithm());
        Files.deleteIfExists(checksumFile);
      }
      Path uploadPath = getPathInStorageDirectory(info.getId());
      FileUtils.deleteDirectory(uploadPath.toFile());
    }
  }

  @Override
  public Long getUploadExpirationPeriod() {
    return uploadExpirationPeriod;
  }

  @Override
  public void setUploadExpirationPeriod(Long uploadExpirationPeriod) {
    this.uploadExpirationPeriod = uploadExpirationPeriod;
  }

  @Override
  public void setUploadConcatenationService(UploadConcatenationService concatenationService) {
    Objects.requireNonNull(concatenationService);
    this.uploadConcatenationService = concatenationService;
  }

  @Override
  public UploadConcatenationService getUploadConcatenationService() {
    return uploadConcatenationService;
  }

  @Override
  public InputStream getUploadedBytes(String uploadUri, String ownerKey)
      throws IOException, UploadNotFoundException {

    UploadId id = idFactory.readUploadId(uploadUri);

    UploadInfo uploadInfo = getUploadInfo(id);
    if (uploadInfo == null || !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {
      throw new UploadNotFoundException(
          "The upload with id " + id + " could not be found for owner " + ownerKey);
    } else {
      return getUploadedBytes(id);
    }
  }

  @Override
  public InputStream getUploadedBytes(UploadId id) throws IOException, UploadNotFoundException {
    InputStream inputStream = null;
    UploadInfo uploadInfo = getUploadInfo(id);
    if (uploadInfo == null) {
      throw new UploadNotFoundException("The upload with id " + id + " was not found.");
    }

    if (uploadInfo.getDuplicatesUploadId() != null) {
      return getUploadedBytes(uploadInfo.getDuplicatesUploadId());
    }

    if (UploadType.CONCATENATED.equals(uploadInfo.getUploadType())
        && uploadConcatenationService != null) {
      inputStream = uploadConcatenationService.getConcatenatedBytes(uploadInfo);

    } else {
      Path bytesPath = getBytesPath(id);
      // If bytesPath is not null, we know this is a valid Upload URI
      if (bytesPath != null) {
        if (!Files.exists(bytesPath)) {
          throw new UploadNotFoundException(
              "The upload bytes for id " + id + " could not be found.");
        }
        inputStream = Channels.newInputStream(FileChannel.open(bytesPath, READ));
      }
    }

    return inputStream;
  }

  @Override
  public void copyUploadTo(UploadInfo info, OutputStream outputStream)
      throws UploadNotFoundException, IOException {

    List<UploadInfo> uploads = getUploads(info);

    try (WritableByteChannel outputChannel = Channels.newChannel(outputStream)) {

      for (UploadInfo upload : uploads) {
        if (upload == null) {
          log.warn("We cannot copy the bytes of an upload that does not exist");

        } else if (upload.isUploadInProgress()) {
          log.warn(
              "We cannot copy the bytes of upload {} because it is still in progress",
              upload.getId());

        } else {
          UploadId readId =
              upload.getDuplicatesUploadId() != null
                  ? upload.getDuplicatesUploadId()
                  : upload.getId();
          Path bytesPath = getBytesPath(readId);
          if (!Files.exists(bytesPath)) {
            throw new UploadNotFoundException(
                "The upload bytes for id " + readId + " could not be found.");
          }
          try (FileChannel file = FileChannel.open(bytesPath, READ)) {
            // Efficiently copy the bytes to the output stream
            file.transferTo(0, upload.getLength(), outputChannel);
          }
        }
      }
    }
  }

  @Override
  public void cleanupExpiredUploads(UploadLockingService uploadLockingService) throws IOException {
    try (DirectoryStream<Path> expiredUploadsStream =
        Files.newDirectoryStream(
            getStoragePath(), new ExpiredUploadFilter(this, uploadLockingService))) {

      for (Path path : expiredUploadsStream) {
        FileUtils.deleteDirectory(path.toFile());
      }
    }
  }

  private List<UploadInfo> getUploads(UploadInfo info) throws IOException, UploadNotFoundException {
    List<UploadInfo> uploads;

    if (info != null
        && UploadType.CONCATENATED.equals(info.getUploadType())
        && uploadConcatenationService != null) {
      uploadConcatenationService.merge(info);
      uploads = uploadConcatenationService.getPartialUploads(info);
    } else {
      uploads = Collections.singletonList(info);
    }
    return uploads;
  }

  private Path getBytesPath(UploadId id) throws UploadNotFoundException {
    return getPathInUploadDir(id, DATA_FILE);
  }

  private Path getInfoPath(UploadId id) throws UploadNotFoundException {
    return getPathInUploadDir(id, INFO_FILE);
  }

  private Path getChecksumPath(String checksum, ChecksumAlgorithm algorithm) throws IOException {
    String filename = checksum;
    if (filename != null) {
      if (!filename.matches("^[0-9a-fA-F]+$")) {
        try {
          byte[] bytes = Base64.decodeBase64(filename);
          if (bytes != null && bytes.length > 0) {
            StringBuilder sb = new StringBuilder();
            for (byte b : bytes) {
              sb.append(String.format("%02x", b));
            }
            filename = sb.toString();
          }
        } catch (Exception e) {
          // Ignore, keep original filename
        }
      }
    }

    if (!isSafePathComponent(filename)) {
      throw new IOException("The checksum contains an unsafe value.");
    }
    if (!isSafePathComponent(algorithm.toString())) {
      throw new IOException("The checksum algorithm contains an unsafe value.");
    }
    return getStoragePath()
        .getParent()
        .resolve("checksums")
        .resolve(algorithm.toString())
        .resolve(filename);
  }

  private boolean isSafePathComponent(String component) {
    return component != null
        && !component.trim().isEmpty()
        && !component.contains("/")
        && !component.contains("\\")
        && !component.contains("..");
  }

  private Path createUploadDirectory(UploadId id) throws IOException {
    return Files.createDirectories(getPathInStorageDirectory(id));
  }

  private Path getPathInUploadDir(UploadId id, String fileName) throws UploadNotFoundException {
    // Get the upload directory
    Path uploadDir = getPathInStorageDirectory(id);
    if (uploadDir != null && Files.exists(uploadDir)) {
      if (!isSafePathComponent(fileName)) {
        throw new IllegalArgumentException("The file name contains an unsafe value.");
      }
      return uploadDir.resolve(fileName);
    } else {
      throw new UploadNotFoundException("The upload for id " + id + " was not found.");
    }
  }

  private synchronized UploadId createNewId() throws IOException {
    UploadId id;
    do {
      id = idFactory.createId();
      // For extra safety, double check that this ID is not in use yet
    } while (getUploadInfo(id) != null);
    return id;
  }

  private long writeAsMuchAsPossible(FileChannel file) throws IOException {
    long offset = 0;
    if (file != null) {
      file.force(true);
      offset = file.size();
    }
    return offset;
  }
}

1	package me.desair.tus.server.upload.disk;
2
3	import static java.nio.file.StandardOpenOption.READ;
4	import static java.nio.file.StandardOpenOption.WRITE;
5
6	import java.io.File;
7	import java.io.IOException;
8	import java.io.InputStream;
9	import java.io.OutputStream;
10	import java.nio.channels.Channels;
11	import java.nio.channels.FileChannel;
12	import java.nio.channels.ReadableByteChannel;
13	import java.nio.channels.WritableByteChannel;
14	import java.nio.charset.StandardCharsets;
15	import java.nio.file.DirectoryStream;
16	import java.nio.file.Files;
17	import java.nio.file.Path;
18	import java.util.Collections;
19	import java.util.List;
20	import java.util.Objects;
21	import me.desair.tus.server.checksum.ChecksumAlgorithm;
22	import me.desair.tus.server.exception.InvalidUploadOffsetException;
23	import me.desair.tus.server.exception.TusException;
24	import me.desair.tus.server.exception.UploadNotFoundException;
25	import me.desair.tus.server.upload.UploadId;
26	import me.desair.tus.server.upload.UploadIdFactory;
27	import me.desair.tus.server.upload.UploadInfo;
28	import me.desair.tus.server.upload.UploadLockingService;
29	import me.desair.tus.server.upload.UploadStorageService;
30	import me.desair.tus.server.upload.UploadType;
31	import me.desair.tus.server.upload.concatenation.UploadConcatenationService;
32	import me.desair.tus.server.upload.concatenation.VirtualConcatenationService;
33	import me.desair.tus.server.util.Utils;
34	import org.apache.commons.codec.binary.Base64;
35	import org.apache.commons.io.FileUtils;
36	import org.slf4j.Logger;
37	import org.slf4j.LoggerFactory;
38
39	/** Implementation of {@link UploadStorageService} that implements storage on disk. */
40	public class DiskStorageService extends AbstractDiskBasedService implements UploadStorageService {
41
42	private static final Logger log = LoggerFactory.getLogger(DiskStorageService.class);	8✔
43
44	private static final String UPLOAD_SUB_DIRECTORY = "uploads";
45	private static final String INFO_FILE = "info";
46	private static final String DATA_FILE = "data";
47
48	private Long maxUploadSize = null;	6✔
49	private Long maxAppendSize = null;	6✔
50	private Long uploadExpirationPeriod = null;	6✔
51	private UploadIdFactory idFactory;
52	private UploadConcatenationService uploadConcatenationService;
53	private boolean isUploadDeduplicationEnabled = false;	6✔
54
55	public DiskStorageService(String storagePath) {
56	super(storagePath + File.separator + UPLOAD_SUB_DIRECTORY);	10✔
57	setUploadConcatenationService(new VirtualConcatenationService(this));	12✔
58	}	2✔
59
60	public DiskStorageService(UploadIdFactory idFactory, String storagePath) {
61	this(storagePath);	6✔
62	Objects.requireNonNull(idFactory, "The IdFactory cannot be null");	8✔
63	this.idFactory = idFactory;	6✔
64	}	2✔
65
66	@Override
67	public void setIdFactory(UploadIdFactory idFactory) {
68	Objects.requireNonNull(idFactory, "The IdFactory cannot be null");	8✔
69	this.idFactory = idFactory;	6✔
70	}	2✔
71
72	@Override
73	public void setMaxUploadSize(Long maxUploadSize) {
74	this.maxUploadSize = (maxUploadSize != null && maxUploadSize > 0 ? maxUploadSize : 0);	28✔
75	}	2✔
76
77	@Override
78	public long getMaxUploadSize() {
79	return maxUploadSize == null ? 0 : maxUploadSize;	18✔
80	}
81
82	@Override
83	public void setMaxAppendSize(Long maxAppendSize) {
84	this.maxAppendSize = (maxAppendSize != null && maxAppendSize > 0 ? maxAppendSize : null);	17✔
85	}	2✔
86
87	@Override
88	public Long getMaxAppendSize() {
89	if (maxAppendSize != null && maxAppendSize > 0) {	12✔
90	return maxAppendSize;	3✔
91	}
92	long maxUpload = getMaxUploadSize();	6✔
93	return maxUpload > 0 ? maxUpload : null;	18✔
94	}
95
96	@Override
97	public void setUploadDeduplicationEnabled(boolean enabled) {
98	this.isUploadDeduplicationEnabled = enabled;	6✔
99	}	2✔
100
101	@Override
102	public boolean isUploadDeduplicationEnabled() {
103	return this.isUploadDeduplicationEnabled;	6✔
104	}
105
106	@Override
107	public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm algorithm)
108	throws IOException {
109	if (checksum == null \|\| algorithm == null) {	8✔
110	return null;	2✔
111	}
112	Path checksumFile = getChecksumPath(checksum, algorithm);	10✔
113	if (Files.exists(checksumFile)) {	10✔
114	String uploadIdStr =	6✔
115	new String(Files.readAllBytes(checksumFile), StandardCharsets.UTF_8).trim();	10✔
116	UploadId id = new UploadId(uploadIdStr);	10✔
117	UploadInfo info = getUploadInfo(id);	8✔
118	if (info == null) {	4✔
119	Files.deleteIfExists(checksumFile);	3✔
120	return null;	2✔
121	}
122	Path bytesPath = null;	4✔
123	try {
124	bytesPath = getPathInUploadDir(id, DATA_FILE);	10✔
125	} catch (UploadNotFoundException e) {	×
126	// file doesn't exist
127	}	2✔
128	if (bytesPath == null \|\| !Files.exists(bytesPath)) {	14!
129	Files.deleteIfExists(checksumFile);	3✔
130	return null;	2✔
131	}
132	return info;	4✔
133	}
134	return null;	4✔
135	}
136
137	@Override
138	public UploadInfo getUploadInfo(String uploadUrl, String ownerKey) throws IOException {
139	UploadInfo uploadInfo = getUploadInfo(idFactory.readUploadId(uploadUrl));	14✔
140	if (uploadInfo == null \|\| !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {	14✔
141	return null;	4✔
142	} else {
143	return uploadInfo;	4✔
144	}
145	}
146
147	@Override
148	public UploadInfo getUploadInfo(UploadId id) throws IOException {
149	try {
150	Path infoPath = getInfoPath(id);	8✔
151	if (infoPath == null \|\| !Files.exists(infoPath)) {	14!
152	return null;	2✔
153	}
154	return Utils.readSerializable(infoPath, UploadInfo.class);	10✔
155	} catch (UploadNotFoundException e) {	2✔
156	return null;	4✔
157	}
158	}
159
160	@Override
161	public String getUploadUri() {
162	return idFactory.getUploadUri();	8✔
163	}
164
165	@Override
166	public UploadInfo create(UploadInfo info, String ownerKey) throws IOException {
167	UploadId id = createNewId();	6✔
168
169	createUploadDirectory(id);	8✔
170
171	try {
172	Path bytesPath = getBytesPath(id);	8✔
173
174	// Create an empty file to storage the bytes of this upload
175	Files.createFile(bytesPath);	10✔
176
177	// Set starting values
178	info.setId(id);	6✔
179	info.setOffset(0L);	8✔
180	info.setOwnerKey(ownerKey);	6✔
181
182	update(info);	6✔
183
184	return info;	4✔
185	} catch (UploadNotFoundException e) {	×
186	// Normally this cannot happen
187	log.error("Unable to create UploadInfo because of an upload not found exception", e);	×
188	return null;	×
189	}
190	}
191
192	@Override
193	public void update(UploadInfo uploadInfo) throws IOException, UploadNotFoundException {
194	if (uploadInfo != null) {	4!
195	if (uploadInfo.getDuplicatesUploadId() != null) {	6✔
196	// Delete the child's own data file if it exists
197	try {
198	Path childDataPath = getPathInUploadDir(uploadInfo.getId(), DATA_FILE);	12✔
199	Files.deleteIfExists(childDataPath);	6✔
200	} catch (UploadNotFoundException e) {	×
201	// It doesn't exist yet, which is fine
202	}	2✔
203
204	// Retrieve the parent UploadInfo
205	UploadId parentId = uploadInfo.getDuplicatesUploadId();	6✔
206	UploadInfo parentInfo = getUploadInfo(parentId);	8✔
207	if (parentInfo != null) {	4!
208	Path parentBytesPath = null;	4✔
209	try {
210	parentBytesPath = getPathInUploadDir(parentId, DATA_FILE);	10✔
211	} catch (UploadNotFoundException e) {	×
212	// Parent file is not found
213	}	2✔
214
215	if (parentBytesPath != null && Files.exists(parentBytesPath)) {	14!
216	// Ensure parent's expiration timestamp is >= child's expiration timestamp
217	Long childExpire = uploadInfo.getExpirationTimestamp();	6✔
218	Long parentExpire = parentInfo.getExpirationTimestamp();	6✔
219	if (childExpire == null) {	4✔
220	if (parentExpire != null) {	2✔
221	parentInfo.setExpirationTimestamp(null);	3✔
222	Path parentInfoPath = getInfoPath(parentId);	4✔
223	Utils.writeSerializable(parentInfo, parentInfoPath);	3✔
224	}	1✔
225	} else {
226	if (parentExpire != null && parentExpire < childExpire) {	16!
227	parentInfo.setExpirationTimestamp(childExpire);	6✔
228	Path parentInfoPath = getInfoPath(parentId);	8✔
229	Utils.writeSerializable(parentInfo, parentInfoPath);	6✔
230	}
231	}
232	}
233	}
234	} else if (isUploadDeduplicationEnabled()	10✔
235	&& !uploadInfo.isUploadInProgress()	6✔
236	&& uploadInfo.getChecksum() != null	6✔
237	&& uploadInfo.getChecksumAlgorithm() != null) {	4!
238	// Index the checksum
239	Path checksumFile =	4✔
240	getChecksumPath(uploadInfo.getChecksum(), uploadInfo.getChecksumAlgorithm());	10✔
241	Files.createDirectories(checksumFile.getParent());	12✔
242	Files.write(checksumFile, uploadInfo.getId().toString().getBytes(StandardCharsets.UTF_8));	20✔
243	}
244
245	Path infoPath = getInfoPath(uploadInfo.getId());	10✔
246	Utils.writeSerializable(uploadInfo, infoPath);	6✔
247	}
248	}	2✔
249
250	@Override
251	public UploadInfo append(UploadInfo info, InputStream inputStream)
252	throws IOException, TusException {
253	if (info != null) {	4!
254	Path bytesPath = getBytesPath(info.getId());	10✔
255
256	long max = getMaxUploadSize() > 0 ? getMaxUploadSize() : Long.MAX_VALUE;	20✔
257	long transferred = 0;	4✔
258	Long offset = info.getOffset();	6✔
259	long newOffset = offset;	6✔
260
261	try (ReadableByteChannel uploadedBytes = Channels.newChannel(inputStream);	6✔
262	FileChannel file = FileChannel.open(bytesPath, WRITE)) {	18✔
263
264	try {
265	// Lock will be released when the channel closes
266	file.lock();	6✔
267
268	// Validate that the given offset is at the end of the file
269	if (!offset.equals(file.size())) {	12✔
270	throw new InvalidUploadOffsetException(	5✔
271	"The upload offset does not correspond to the written"
272	+ " bytes. You can only append to the end of an upload");
273	}
274
275	// write all bytes in the channel up to the configured maximum
276	transferred = file.transferFrom(uploadedBytes, offset, max - offset);	20✔
277	file.force(true);	6✔
278	newOffset = offset + transferred;	10✔
279
280	} catch (Exception ex) {	2✔
281	// An error occurred, try to write as much data as possible
282	newOffset = writeAsMuchAsPossible(file);	8✔
283	throw ex;	4✔
284	}	2✔
285
286	} finally {
287	info.setOffset(newOffset);	8✔
288	update(info);	6✔
289	}
290	}
291
292	return info;	4✔
293	}
294
295	@Override
296	public void removeLastNumberOfBytes(UploadInfo info, long byteCount)
297	throws UploadNotFoundException, IOException {
298
299	if (info != null && byteCount > 0) {	12✔
300	Path bytesPath = getBytesPath(info.getId());	10✔
301
302	try (FileChannel file = FileChannel.open(bytesPath, WRITE)) {	18✔
303
304	// Lock will be released when the channel closes
305	file.lock();	6✔
306
307	file.truncate(file.size() - byteCount);	14✔
308	file.force(true);	6✔
309
310	info.setOffset(file.size());	10✔
311	update(info);	6✔
312	}
313	}
314	}	2✔
315
316	@Override
317	public void terminateUpload(UploadInfo info) throws UploadNotFoundException, IOException {
318	if (info != null) {	4✔
319	if (info.getDuplicatesUploadId() == null	8!
320	&& info.getChecksum() != null	6✔
321	&& info.getChecksumAlgorithm() != null) {	4!
322	Path checksumFile = getChecksumPath(info.getChecksum(), info.getChecksumAlgorithm());	14✔
323	Files.deleteIfExists(checksumFile);	6✔
324	}
325	Path uploadPath = getPathInStorageDirectory(info.getId());	10✔
326	FileUtils.deleteDirectory(uploadPath.toFile());	6✔
327	}
328	}	2✔
329
330	@Override
331	public Long getUploadExpirationPeriod() {
332	return uploadExpirationPeriod;	6✔
333	}
334
335	@Override
336	public void setUploadExpirationPeriod(Long uploadExpirationPeriod) {
337	this.uploadExpirationPeriod = uploadExpirationPeriod;	6✔
338	}	2✔
339
340	@Override
341	public void setUploadConcatenationService(UploadConcatenationService concatenationService) {
342	Objects.requireNonNull(concatenationService);	6✔
343	this.uploadConcatenationService = concatenationService;	6✔
344	}	2✔
345
346	@Override
347	public UploadConcatenationService getUploadConcatenationService() {
348	return uploadConcatenationService;	3✔
349	}
350
351	@Override
352	public InputStream getUploadedBytes(String uploadUri, String ownerKey)
353	throws IOException, UploadNotFoundException {
354
355	UploadId id = idFactory.readUploadId(uploadUri);	10✔
356
357	UploadInfo uploadInfo = getUploadInfo(id);	8✔
358	if (uploadInfo == null \|\| !Objects.equals(uploadInfo.getOwnerKey(), ownerKey)) {	14!
359	throw new UploadNotFoundException(	8✔
360	"The upload with id " + id + " could not be found for owner " + ownerKey);
361	} else {
362	return getUploadedBytes(id);	8✔
363	}
364	}
365
366	@Override
367	public InputStream getUploadedBytes(UploadId id) throws IOException, UploadNotFoundException {
368	InputStream inputStream = null;	4✔
369	UploadInfo uploadInfo = getUploadInfo(id);	8✔
370	if (uploadInfo == null) {	4✔
371	throw new UploadNotFoundException("The upload with id " + id + " was not found.");	7✔
372	}
373
374	if (uploadInfo.getDuplicatesUploadId() != null) {	6✔
375	return getUploadedBytes(uploadInfo.getDuplicatesUploadId());	5✔
376	}
377
378	if (UploadType.CONCATENATED.equals(uploadInfo.getUploadType())	13!
379	&& uploadConcatenationService != null) {
380	inputStream = uploadConcatenationService.getConcatenatedBytes(uploadInfo);	6✔
381
382	} else {
383	Path bytesPath = getBytesPath(id);	8✔
384	// If bytesPath is not null, we know this is a valid Upload URI
385	if (bytesPath != null) {	4!
386	if (!Files.exists(bytesPath)) {	10!
387	throw new UploadNotFoundException(	×
388	"The upload bytes for id " + id + " could not be found.");
389	}
390	inputStream = Channels.newInputStream(FileChannel.open(bytesPath, READ));	20✔
391	}
392	}
393
394	return inputStream;	4✔
395	}
396
397	@Override
398	public void copyUploadTo(UploadInfo info, OutputStream outputStream)
399	throws UploadNotFoundException, IOException {
400
401	List<UploadInfo> uploads = getUploads(info);	8✔
402
403	try (WritableByteChannel outputChannel = Channels.newChannel(outputStream)) {	6✔
404
405	for (UploadInfo upload : uploads) {	20✔
406	if (upload == null) {	4!
407	log.warn("We cannot copy the bytes of an upload that does not exist");	×
408
409	} else if (upload.isUploadInProgress()) {	6!
410	log.warn(	×
411	"We cannot copy the bytes of upload {} because it is still in progress",
412	upload.getId());	×
413
414	} else {
415	UploadId readId =
416	upload.getDuplicatesUploadId() != null	6✔
417	? upload.getDuplicatesUploadId()	6✔
418	: upload.getId();	6✔
419	Path bytesPath = getBytesPath(readId);	8✔
420	if (!Files.exists(bytesPath)) {	10!
421	throw new UploadNotFoundException(	×
422	"The upload bytes for id " + readId + " could not be found.");
423	}
424	try (FileChannel file = FileChannel.open(bytesPath, READ)) {	18✔
425	// Efficiently copy the bytes to the output stream
426	file.transferTo(0, upload.getLength(), outputChannel);	16✔
427	}
428	}
429	}	2✔
430	}
431	}	2✔
432
433	@Override
434	public void cleanupExpiredUploads(UploadLockingService uploadLockingService) throws IOException {
435	try (DirectoryStream<Path> expiredUploadsStream =	2✔
436	Files.newDirectoryStream(	4✔
437	getStoragePath(), new ExpiredUploadFilter(this, uploadLockingService))) {	12✔
438
439	for (Path path : expiredUploadsStream) {	20✔
440	FileUtils.deleteDirectory(path.toFile());	6✔
441	}	2✔
442	}
443	}	2✔
444
445	private List<UploadInfo> getUploads(UploadInfo info) throws IOException, UploadNotFoundException {
446	List<UploadInfo> uploads;
447
448	if (info != null	8!
449	&& UploadType.CONCATENATED.equals(info.getUploadType())	9!
450	&& uploadConcatenationService != null) {
451	uploadConcatenationService.merge(info);	4✔
452	uploads = uploadConcatenationService.getPartialUploads(info);	6✔
453	} else {
454	uploads = Collections.singletonList(info);	6✔
455	}
456	return uploads;	4✔
457	}
458
459	private Path getBytesPath(UploadId id) throws UploadNotFoundException {
460	return getPathInUploadDir(id, DATA_FILE);	10✔
461	}
462
463	private Path getInfoPath(UploadId id) throws UploadNotFoundException {
464	return getPathInUploadDir(id, INFO_FILE);	10✔
465	}
466
467	private Path getChecksumPath(String checksum, ChecksumAlgorithm algorithm) throws IOException {
468	String filename = checksum;	4✔
469	if (filename != null) {	4!
470	if (!filename.matches("^[0-9a-fA-F]+$")) {	8!
471	try {
472	byte[] bytes = Base64.decodeBase64(filename);	6✔
473	if (bytes != null && bytes.length > 0) {	10!
474	StringBuilder sb = new StringBuilder();	8✔
475	for (byte b : bytes) {	32✔
476	sb.append(String.format("%02x", b));	24✔
477	}
478	filename = sb.toString();	6✔
479	}
NEW 480	} catch (Exception e) {	×
481	// Ignore, keep original filename
482	}	2✔
483	}
484	}
485
486	if (!isSafePathComponent(filename)) {	8✔
487	throw new IOException("The checksum contains an unsafe value.");	5✔
488	}
489	if (!isSafePathComponent(algorithm.toString())) {	10!
490	throw new IOException("The checksum algorithm contains an unsafe value.");	×
491	}
492	return getStoragePath()	6✔
493	.getParent()	4✔
494	.resolve("checksums")	4✔
495	.resolve(algorithm.toString())	6✔
496	.resolve(filename);	2✔
497	}
498
499	private boolean isSafePathComponent(String component) {
500	return component != null	8!
501	&& !component.trim().isEmpty()	10✔
502	&& !component.contains("/")	8✔
503	&& !component.contains("\\")	8!
504	&& !component.contains("..");	9✔
505	}
506
507	private Path createUploadDirectory(UploadId id) throws IOException {
508	return Files.createDirectories(getPathInStorageDirectory(id));	14✔
509	}
510
511	private Path getPathInUploadDir(UploadId id, String fileName) throws UploadNotFoundException {
512	// Get the upload directory
513	Path uploadDir = getPathInStorageDirectory(id);	8✔
514	if (uploadDir != null && Files.exists(uploadDir)) {	14✔
515	if (!isSafePathComponent(fileName)) {	8!
516	throw new IllegalArgumentException("The file name contains an unsafe value.");	×
517	}
518	return uploadDir.resolve(fileName);	8✔
519	} else {
520	throw new UploadNotFoundException("The upload for id " + id + " was not found.");	14✔
521	}
522	}
523
524	private synchronized UploadId createNewId() throws IOException {
525	UploadId id;
526	do {
527	id = idFactory.createId();	8✔
528	// For extra safety, double check that this ID is not in use yet
529	} while (getUploadInfo(id) != null);	8!
530	return id;	4✔
531	}
532
533	private long writeAsMuchAsPossible(FileChannel file) throws IOException {
534	long offset = 0;	4✔
535	if (file != null) {	4!
536	file.force(true);	6✔
537	offset = file.size();	6✔
538	}
539	return offset;	4✔
540	}
541	}

tomdesair / tus-java-server / 28898324220

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous