Skip to content

Optimize S3 server-side compose capability check using encoded part sizes #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tomdesair merged 1 commit into from
Oct 25, 2026
Merged

Optimize S3 server-side compose capability check using encoded part sizes #149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 5 additions & 24 deletions src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,29 +195,8 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep
boolean completed = checkAllCompleted(expirationPeriod, partialUploads);

if (totalLength != null && totalLength > 0 && completed) {
// AWS S3 composeObject / multipart upload constraints:
// 1. Single part: must be >= minPartSize (5MB) and have valid storageUploadId to compose.
// 2. Multiple parts (>= 2): all parts EXCEPT the final part must be >= minPartSize (5MB).
// The final part is allowed to be < 5MB while still using zero-copy server-side
// composition.
boolean canUseServerSideCopy = !partialUploads.isEmpty();
if (partialUploads.size() == 1) {
UploadInfo p = partialUploads.get(0);
canUseServerSideCopy =
p.getLength() != null && p.getLength() >= minPartSize && p.getStorageUploadId() != null;
} else {
for (int i = 0; i < partialUploads.size(); i++) {
UploadInfo p = partialUploads.get(i);
if (p.getLength() == null || p.getStorageUploadId() == null) {
canUseServerSideCopy = false;
break;
}
if (i < partialUploads.size() - 1 && p.getLength() < minPartSize) {
canUseServerSideCopy = false;
break;
}
}
}
boolean canUseServerSideCopy =
s3ComposeHelper.canComposePartialUploads(partialUploads, minPartSize);

String targetObjectKey = buildObjectKey(uploadInfo.getId());

Expand Down Expand Up @@ -288,8 +267,10 @@ private void mergeUsingServerSideCopy(
s3ComposeHelper.compose(bucket, targetKey, partKeys);
} catch (Exception e) {
// If server-side composition fails for any reason (e.g. S3 permissions, regional policy,
// or unhandled multipart copy restriction), fall back to streaming re-upload as a safety net.
// or unhandled multipart copy restriction), mark compose as unsupported on this helper
// and fall back to streaming re-upload as a safety net.
// This ensures concatenated uploads always succeed even if server-side copy is denied.
s3ComposeHelper.setComposeSupported(false);
log.warn(
"Server-side S3 composition failed for target key {}; falling back to streaming re-upload: {}",
targetKey,
Expand Down
103 changes: 101 additions & 2 deletions src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import me.desair.tus.server.upload.UploadInfo;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
Expand Down Expand Up @@ -62,6 +65,8 @@ public class S3ServerSideComposeHelper implements Closeable {
private final Provider provider;
private final OkHttpClient httpClient;
private final String explicitRegion;
private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock();
private boolean composeSupported = true;

/**
* Constructs an instance with explicit connection parameters without requiring a {@link
Expand Down Expand Up @@ -89,15 +94,15 @@ public S3ServerSideComposeHelper(
this(minioClient, null, null, null, null);
}

private S3ServerSideComposeHelper(
S3ServerSideComposeHelper(
MinioClient minioClient, String endpoint, String region, String accessKey, String secretKey) {
this.minioClient = minioClient;
this.explicitRegion = (region != null && !region.isEmpty()) ? region : "local";

Http.BaseUrl base = null;
Provider prov = null;
OkHttpClient client = null;
MinioAsyncClient async = null;
MinioClient syncClient = minioClient;

if (endpoint != null && !endpoint.isEmpty()) {
try {
Expand All @@ -114,6 +119,9 @@ private S3ServerSideComposeHelper(
asyncBuilder.region(region);
}
async = asyncBuilder.build();
if (syncClient == null) {
syncClient = S3Utils.buildMinioClient(endpoint, region, accessKey, secretKey);
}
} catch (Exception e) {
log.warn(
"Failed to initialize native S3 connection from endpoint {}: {}",
Expand All @@ -122,6 +130,7 @@ private S3ServerSideComposeHelper(
}
}

this.minioClient = syncClient;
this.baseUrl = base;
this.provider = prov;
this.httpClient = client;
Expand All @@ -137,6 +146,96 @@ public boolean isAvailable() {
return asyncClient != null && baseUrl != null && httpClient != null;
}

/**
* Returns whether S3 server-side composition is supported by the storage endpoint.
*
* @return {@code true} if server-side compose is supported; {@code false} if disabled or failed
*/
public boolean isComposeSupported() {
thisObjectLock.readLock().lock();
try {
return composeSupported;
} finally {
thisObjectLock.readLock().unlock();
}
}

/**
* Configures whether S3 server-side composition is supported by the storage endpoint.
*
* @param composeSupported {@code true} to enable; {@code false} to disable
*/
public void setComposeSupported(boolean composeSupported) {
thisObjectLock.writeLock().lock();
try {
this.composeSupported = composeSupported;
} finally {
thisObjectLock.writeLock().unlock();
}
}

/**
* Checks whether the given part keys meet S3's minimum part size requirement for server-side
* composition (all parts except the final part must be at least {@code minPartSize} bytes).
*
* <p>Part sizes are resolved in O(1) time from the {@code _<size>} suffix encoded in each part
* key name, falling back to {@code statObject} only for legacy keys without a size suffix.
*
* @param bucket The S3 bucket name
* @param partKeys The ordered list of part keys to compose
* @param minPartSize Minimum required byte size for non-final parts (typically 5 MiB)
* @return {@code true} if server-side composition can be used; {@code false} otherwise
*/
public boolean canCompose(String bucket, List<String> partKeys, long minPartSize) {
if (!isComposeSupported() || partKeys == null || partKeys.isEmpty()) {
return false;
}
// AWS S3 requires all parts except the last to be >= minPartSize (5 MB)
for (int i = 0; i < partKeys.size() - 1; i++) {
if (S3Utils.getPartSize(minioClient, bucket, partKeys.get(i)) < minPartSize) {
return false;
}
}
return true;
}

/**
* Checks whether the given partial uploads meet S3's constraints for server-side composition.
*
* <p>AWS S3 composeObject / multipart upload constraints:
*
* <ul>
* <li>Single part: must be &ge; minPartSize (5MB) and have a valid storageUploadId.
* <li>Multiple parts (&ge; 2): all parts EXCEPT the final part must be &ge; minPartSize (5MB)
* and all parts must have valid storageUploadId. The final part is allowed to be &lt; 5MB.
* </ul>
*
* @param partialUploads The partial uploads to check
* @param minPartSize Minimum required byte size for non-final parts (typically 5 MiB)
* @return {@code true} if server-side composition can be used; {@code false} otherwise
*/
public boolean canComposePartialUploads(List<UploadInfo> partialUploads, long minPartSize) {
if (!isComposeSupported() || partialUploads == null || partialUploads.isEmpty()) {
return false;
}
if (partialUploads.size() == 1) {
UploadInfo p = partialUploads.get(0);
return p.getLength() != null
&& p.getLength() >= minPartSize
&& p.getStorageUploadId() != null;
}
for (int i = 0; i < partialUploads.size(); i++) {
UploadInfo p = partialUploads.get(i);
if (p.getLength() == null || p.getStorageUploadId() == null) {
return false;
}
if (i < partialUploads.size() - 1 && p.getLength() < minPartSize) {
return false;
}
}
return true;
}

/**
* Composes the given part keys into the target object key entirely server-side on S3.
*
Expand Down
133 changes: 52 additions & 81 deletions src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import me.desair.tus.server.checksum.ChecksumKey;
Expand Down Expand Up @@ -110,8 +108,6 @@ public class S3StorageService extends AbstractUploadStorageService {
private final S3ChecksumIndex checksumIndex;
private DeduplicationService deduplicationService;

private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock();
private boolean s3ComposeObjectSupported = true;
private S3ServerSideComposeHelper s3ComposeHelper;

/**
Expand Down Expand Up @@ -751,25 +747,21 @@ public boolean isS3ConditionalWritesSupported() {
}

boolean isS3ComposeObjectSupported() {
thisObjectLock.readLock().lock();
try {
return s3ComposeObjectSupported;
} finally {
thisObjectLock.readLock().unlock();
}
return s3ComposeHelper != null && s3ComposeHelper.isComposeSupported();
}

void setS3ComposeObjectSupported(boolean s3ComposeObjectSupported) {
thisObjectLock.writeLock().lock();
try {
this.s3ComposeObjectSupported = s3ComposeObjectSupported;
} finally {
thisObjectLock.writeLock().unlock();
if (this.s3ComposeHelper != null) {
this.s3ComposeHelper.setComposeSupported(s3ComposeObjectSupported);
}
}

void setS3ServerSideComposeHelper(S3ServerSideComposeHelper s3ComposeHelper) {
this.s3ComposeHelper = s3ComposeHelper;
if (this.concatenationService instanceof S3ConcatenationService) {
((S3ConcatenationService) this.concatenationService)
.setS3ServerSideComposeHelper(s3ComposeHelper);
}
}

S3ServerSideComposeHelper getS3ServerSideComposeHelper() {
Expand Down Expand Up @@ -874,49 +866,51 @@ private PreparedStream prepareStreamWithExistingIncompletePart(
? new ArrayList<>(info.getUploadPartKeys())
: new ArrayList<>();

long existingPartsTotalSize = calculateTotalPartsSize(existingPartKeys);
long existingPartsTotalSize = 0L;
long lastPartSize = 0L;
for (int i = 0; i < existingPartKeys.size(); i++) {
long partSize = getPartSize(existingPartKeys.get(i));
existingPartsTotalSize += partSize;
if (i == existingPartKeys.size() - 1) {
lastPartSize = partSize;
}
}

// 2. Roll back any existing sub-5MB numbered parts into the stream buffer so they can grow
if (!existingPartKeys.isEmpty()) {
if (!existingPartKeys.isEmpty() && lastPartSize > 0 && lastPartSize < minPartSize) {
String lastPartKey = existingPartKeys.get(existingPartKeys.size() - 1);
long lastPartSize = getPartSize(lastPartKey);
if (lastPartSize > 0 && lastPartSize < minPartSize) {
log.info(
"Rolling back sub-5MB part {} (size: {}) into stream buffer",
lastPartKey,
lastPartSize);
// Stream into a secure temp file instead of buffering up to 5 MB directly in JVM heap
File tempPrepFile =
Files.createTempFile(temporaryDirectory, "tus-s3-prep-", ".tmp").toFile();
InputStream prependedStream = null;
try {
try (InputStream lastPartStream =
S3Utils.getObjectInputStream(minioClient, bucket, lastPartKey);
FileOutputStream fos = new FileOutputStream(tempPrepFile)) {
IOUtils.copyLarge(lastPartStream, fos);
}
long tempPrepLength = tempPrepFile.length();
prependedBytes += tempPrepLength;
prependedStream =
new FileInputStream(tempPrepFile) {
@Override
public void close() throws IOException {
try {
super.close();
} finally {
FileUtils.deleteQuietly(tempPrepFile);
}
}
};
combinedStream = new SequenceInputStream(prependedStream, combinedStream);
staleTailKeys.add(lastPartKey);
existingPartsTotalSize -= tempPrepLength;
existingPartKeys.remove(existingPartKeys.size() - 1);
} catch (Exception e) {
IOUtils.closeQuietly(prependedStream);
FileUtils.deleteQuietly(tempPrepFile);
log.debug("Error inspecting last part key {}: {}", lastPartKey, e.getMessage());
log.info(
"Rolling back sub-5MB part {} (size: {}) into stream buffer", lastPartKey, lastPartSize);
// Stream into a secure temp file instead of buffering up to 5 MB directly in JVM heap
File tempPrepFile = Files.createTempFile(temporaryDirectory, "tus-s3-prep-", ".tmp").toFile();
InputStream prependedStream = null;
try {
try (InputStream lastPartStream =
S3Utils.getObjectInputStream(minioClient, bucket, lastPartKey);
FileOutputStream fos = new FileOutputStream(tempPrepFile)) {
IOUtils.copyLarge(lastPartStream, fos);
}
long tempPrepLength = tempPrepFile.length();
prependedBytes += tempPrepLength;
prependedStream =
new FileInputStream(tempPrepFile) {
@Override
public void close() throws IOException {
try {
super.close();
} finally {
FileUtils.deleteQuietly(tempPrepFile);
}
}
};
combinedStream = new SequenceInputStream(prependedStream, combinedStream);
staleTailKeys.add(lastPartKey);
existingPartsTotalSize -= tempPrepLength;
existingPartKeys.remove(existingPartKeys.size() - 1);
} catch (Exception e) {
IOUtils.closeQuietly(prependedStream);
FileUtils.deleteQuietly(tempPrepFile);
log.debug("Error inspecting last part key {}: {}", lastPartKey, e.getMessage());
}
}

Expand Down Expand Up @@ -946,19 +940,7 @@ private long calculateTotalPartsSize(List<String> partKeys) {
* extracted in O(1) time without any network calls; otherwise falls back to statObject.
*/
private long getPartSize(String partKey) {
if (partKey == null) {
return 0L;
}
int underscoreIdx = partKey.lastIndexOf('_');
if (underscoreIdx >= 0 && underscoreIdx < partKey.length() - 1) {
try {
return Long.parseLong(partKey.substring(underscoreIdx + 1));
} catch (NumberFormatException ignored) {
// Fall back to statObject if suffix is not numeric
}
}
long size = S3Utils.getObjectSize(minioClient, bucket, partKey);
return size >= 0 ? size : 0L;
return S3Utils.getPartSize(minioClient, bucket, partKey);
}

/**
Expand Down Expand Up @@ -1238,20 +1220,9 @@ private void finalizeCompletedUploadIfFinished(
}

if (!partKeys.isEmpty()) {
boolean canUseServerSideCompose = true;
if (partKeys.size() > 1) {
// AWS S3 requires all parts except the last to be >= 5 MB (5,242,880 bytes) for
// composeObject. We check sizes using getPartSize() in O(1) time without statObject.
for (int i = 0; i < partKeys.size() - 1; i++) {
long size = getPartSize(partKeys.get(i));
if (size < minPartSize) {
canUseServerSideCompose = false;
break;
}
}
}
boolean canUseServerSideCompose = s3ComposeHelper.canCompose(bucket, partKeys, minPartSize);

if (canUseServerSideCompose && isS3ComposeObjectSupported()) {
if (canUseServerSideCompose) {
try {
// Perform S3 server-side object composition via native S3 multipart copy
s3ComposeHelper.compose(bucket, objectKey, partKeys);
Expand All @@ -1272,7 +1243,7 @@ private void finalizeCompletedUploadIfFinished(
mergeUsingStreamingReupload(objectKey, partKeys, newOffset);
}
} else {
if (!canUseServerSideCompose) {
if (isS3ComposeObjectSupported()) {
log.info(
"Detected sub-5MB non-final parts for ID {}. Using streaming part composition"
+ " fallback.",
Expand Down
Loading
Loading