fix: enforce append size validation across storage backends and support chunked creation

scripts/rufh_conformity_test.py

@@ -111,7 +111,7 @@ def parse_headers(lines):
     return res_headers
 
 
-def http_request(method, url, headers=None, body=None, test_name=""):
+def http_request(method, url, headers=None, body=None, test_name="", chunked=False):
     """
     Socket-based HTTP client helper that transparently handles HTTP 104 interim response frames.
     Tracks 104 interim response detection in INTERIM_RESPONSES_DETECTED.
@@ -130,19 +130,34 @@ def http_request(method, url, headers=None, body=None, test_name=""):
         if "Connection" not in req_headers:
             req_headers["Connection"] = "close"
 
+        is_chunked = chunked or (req_headers.get("Transfer-Encoding", "").lower() == "chunked")
+        if is_chunked:
+            req_headers["Transfer-Encoding"] = "chunked"
+            if "Content-Length" in req_headers:
+                del req_headers["Content-Length"]
+
         req_lines = [f"{method} {path} HTTP/1.1", f"Host: {host}:{port}"]
         for k, v in req_headers.items():
             req_lines.append(f"{k}: {v}")
-        if body is not None and "Content-Length" not in req_headers:
+        if body is not None and not is_chunked and "Content-Length" not in req_headers:
             body_len = len(body) if isinstance(body, bytes) else len(body.encode("utf-8"))
             req_lines.append(f"Content-Length: {body_len}")
         req_lines.append("")
         req_lines.append("")
         req_data = "\r\n".join(req_lines).encode("latin1")
-        if body:
+        if is_chunked:
+            if body:
+                body_bytes = body if isinstance(body, bytes) else body.encode("utf-8")
+                req_data += f"{len(body_bytes):X}\r\n".encode("latin1") + body_bytes + b"\r\n0\r\n\r\n"
+            else:
+                req_data += b"0\r\n\r\n"
+        elif body:
             req_data += body if isinstance(body, bytes) else body.encode("utf-8")
 
-        s.sendall(req_data)
+        try:
+            s.sendall(req_data)
+        except (BrokenPipeError, ConnectionResetError):
+            pass
 
         # Read response data
         resp_bytes = b""
@@ -873,6 +888,109 @@ def test_append_exceeding_max_append_size(self, target_url, request):
         assert status == 413, "Append exceeding max-append-size MUST be rejected with 413"
         assert UPLOAD_LIMIT in rej_headers, "413 rejection SHOULD include Upload-Limit"
 
+    def test_chunked_append_and_creation_enforce_max_append_size(self, target_url, request):
+        """
+        §4.1.4, §4.7: Chunked Creation and Append Exceeding max-append-size Enforced by Storage Backend.
+        Quote: "The server MAY enforce limits on the size of individual data append requests."
+        Expected behavior: When Content-Length is omitted (Transfer-Encoding: chunked), storage streaming
+        enforces max-append-size on both creation POST and append PATCH, returning 413, preserving offset,
+        and accepting chunked appends within limit.
+        """
+        _, resp_headers, _, _ = http_request("OPTIONS", target_url, test_name=request.node.name)
+        limit_hdr = resp_headers.get(UPLOAD_LIMIT, "")
+        max_append_size = None
+        min_append_size = None
+        for part in limit_hdr.split(","):
+            part = part.strip()
+            if "max-append-size=" in part:
+                max_append_size = int(part.split("=")[1].strip())
+            elif "min-append-size=" in part:
+                min_append_size = int(part.split("=")[1].strip())
+        if max_append_size is None or max_append_size > 10 * 1024 * 1024:
+            pytest.skip("Server max-append-size is not advertised or too large to test over socket")
+
+        # Step 1: Chunked creation POST exceeding max-append-size
+        headers = {
+            UPLOAD_COMPLETE: FALSE,
+        }
+        oversized_body = b"X" * (max_append_size + 1)
+        status, rej_headers, _, _ = http_request("POST", target_url, headers=headers, body=oversized_body, chunked=True, test_name=request.node.name)
+        assert status == 413, f"Chunked creation exceeding max-append-size MUST be rejected with 413, got {status}"
+
+        # Step 2: Create a clean partial upload without body and without declared length
+        upload_uri = create_partial_upload(target_url, test_name=request.node.name, upload_length=None)
+
+        # Step 3: Chunked append PATCH exceeding max-append-size
+        headers = {
+            UPLOAD_OFFSET: "0",
+            UPLOAD_COMPLETE: FALSE,
+            CONTENT_TYPE: APPLICATION_PARTIAL_UPLOAD,
+        }
+        status, rej_headers, _, _ = http_request("PATCH", upload_uri, headers=headers, body=oversized_body, chunked=True, test_name=request.node.name)
+        assert status == 413, f"Chunked append exceeding max-append-size MUST be rejected with 413, got {status}"
+
+        # Step 4: Verify offset remains 0 after rejection
+        status, head_headers, _, _ = http_request("HEAD", upload_uri, test_name=request.node.name)
+        assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload offset MUST remain 0 after rejected chunked append"
+
+        # Step 5: Chunked append PATCH within limit succeeds
+        valid_size = min_append_size if (min_append_size and min_append_size <= max_append_size) else min(max_append_size, 16)
+        valid_body = b"Y" * valid_size
+        status, _, _, _ = http_request("PATCH", upload_uri, headers=headers, body=valid_body, chunked=True, test_name=request.node.name)
+        assert status in (200, 204), f"Chunked append within max-append-size MUST succeed, got {status}"
+        status, head_headers, _, _ = http_request("HEAD", upload_uri, test_name=request.node.name)
+        assert head_headers.get(UPLOAD_OFFSET) == str(valid_size), f"Upload offset MUST be {valid_size} after valid chunked append"
+
+    def test_chunked_append_and_creation_enforce_min_append_size(self, target_url, request):
+        """
+        §4.1.4: Chunked Creation and Append Below min-append-size Enforced by Storage Backend.
+        Quote: "min-append-size: An Integer indicating the minimum number of bytes that MUST be appended
+        with each upload append request. This limit does not apply to upload creation requests with no content,
+        or to requests completing the upload by including the Upload-Complete: ?1 header field."
+        Expected behavior: When Content-Length is omitted (Transfer-Encoding: chunked), storage streaming
+        enforces min-append-size on creation POST and append PATCH (when Upload-Complete: ?0), returning 400/413,
+        preserving offset, and accepting chunked appends meeting min-append-size.
+        """
+        _, resp_headers, _, _ = http_request("OPTIONS", target_url, test_name=request.node.name)
+        limit_hdr = resp_headers.get(UPLOAD_LIMIT, "")
+        min_append_size = None
+        for part in limit_hdr.split(","):
+            if "min-append-size=" in part:
+                min_append_size = int(part.split("=")[1].strip())
+        if min_append_size is None or min_append_size <= 1:
+            pytest.skip("Server does not advertise min-append-size > 1")
+
+        # Step 1: Chunked creation POST below min-append-size with content
+        headers = {
+            UPLOAD_COMPLETE: FALSE,
+        }
+        small_body = b"x" * (min_append_size - 1)
+        status, rej_headers, _, _ = http_request("POST", target_url, headers=headers, body=small_body, chunked=True, test_name=request.node.name)
+        assert status in (400, 413), f"Chunked creation below min-append-size MUST be rejected with 400/413, got {status}"
+
+        # Step 2: Create a clean partial upload without body and without declared length
+        upload_uri = create_partial_upload(target_url, test_name=request.node.name, upload_length=None)
+
+        # Step 3: Chunked append PATCH below min-append-size without completion
+        headers = {
+            UPLOAD_OFFSET: "0",
+            UPLOAD_COMPLETE: FALSE,
+            CONTENT_TYPE: APPLICATION_PARTIAL_UPLOAD,
+        }
+        status, rej_headers, _, _ = http_request("PATCH", upload_uri, headers=headers, body=small_body, chunked=True, test_name=request.node.name)
+        assert status in (400, 413), f"Chunked append below min-append-size MUST be rejected with 400/413, got {status}"
+
+        # Step 4: Verify offset remains 0 after rejection
+        status, head_headers, _, _ = http_request("HEAD", upload_uri, test_name=request.node.name)
+        assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload offset MUST remain 0 after rejected chunked append"
+
+        # Step 5: Chunked append PATCH meeting min-append-size succeeds
+        valid_body = b"y" * min_append_size
+        status, _, _, _ = http_request("PATCH", upload_uri, headers=headers, body=valid_body, chunked=True, test_name=request.node.name)
+        assert status in (200, 204), f"Chunked append meeting min-append-size MUST succeed, got {status}"
+        status, head_headers, _, _ = http_request("HEAD", upload_uri, test_name=request.node.name)
+        assert head_headers.get(UPLOAD_OFFSET) == str(min_append_size), f"Upload offset MUST be {min_append_size} after valid chunked append"
+
 
 class TestUploadResourceDeactivation:
     """Tests for Upload Resource Deactivation (§4.5, §4.4.2)."""

src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java

@@ -13,8 +13,6 @@
 import me.desair.tus.server.util.InterruptibleInputStream;
 import me.desair.tus.server.util.TusServletRequest;
 import me.desair.tus.server.util.TusServletResponse;
-import me.desair.tus.server.util.Utils;
-import org.apache.commons.lang3.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,12 +38,9 @@ public HttpProblemDetails process(
       TusException exception)
       throws IOException, TusException {
 
-    Long contentLength = Utils.getLongHeader(servletRequest, HttpHeader.CONTENT_LENGTH);
-    // Allow chunked transfer encoding where Content-Length is omitted or negative
-    boolean hasContent =
-        (contentLength != null && contentLength > 0)
-            || Strings.CI.contains(
-                servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING), "chunked");
+    // Check if the creation request carries upload data to append in the initial request body.
+    // Supports requests with Content-Length as well as chunked encoding or HTTP/2 streaming.
+    boolean hasContent = servletRequest != null && servletRequest.hasContent();
     if (hasContent) {
       String location = servletResponse.getHeader(HttpHeader.LOCATION);
       if (location != null) {

src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java

@@ -16,7 +16,6 @@
 import me.desair.tus.server.util.TusServletRequest;
 import me.desair.tus.server.util.TusServletResponse;
 import me.desair.tus.server.util.Utils;
-import org.apache.commons.lang3.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -100,49 +99,48 @@ public HttpProblemDetails process(
 
     // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content,
     // or to requests completing the upload by including the Upload-Complete: ?1 header field."
-    // An empty creation request carries no body (contentLength <= 0 without chunked encoding).
-    // In servlet requests without a body, getContentLengthLong() returns -1. We must verify
-    // that actual payload content is present before invoking storageService.append(); otherwise,
-    // sending a 0-byte stream to backends with minAppendSize configured (e.g. S3 or Azure Blob)
-    // would trigger MinAppendSizeNotMetException on an empty creation request.
-    boolean hasContent =
-        contentLength > 0
-            || (contentLength < 0
-                && servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING) != null
-                && Strings.CI.contains(
-                    servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING), "chunked"));
-
-    InputStream is = servletRequest.getContentInputStream();
+    // An empty creation request carries no body. In requests without Content-Length (e.g.
+    // HTTP/1.1 chunked encoding, HTTP/2 streaming, or upstream proxies stripping hop-by-hop
+    // headers per RFC 9112 §6.3), getContentLengthLong() returns -1.
+    // servletRequest.hasContent() peeks 1 byte via PushbackInputStream to accurately detect
+    // whether actual payload bytes are present before invoking storageService.append();
+    // otherwise, passing an empty stream to backends with minAppendSize would erroneously reject an
+    // empty creation request.
+    boolean hasContent = servletRequest != null && servletRequest.hasContent();
+
     InterruptibleInputStream interruptibleStream = null;
-    if (is != null && hasContent) {
-      interruptibleStream = new InterruptibleInputStream(is);
-      uploadLockingService.registerInputStream(uploadUri, interruptibleStream);
-      is = interruptibleStream;
-      try {
-        UploadInfo appended = uploadStorageService.append(uploadInfo, is);
-        if (appended != null) {
-          uploadInfo = appended;
-        }
-      } catch (IOException e) {
-        // When an upload stream is interrupted by the locking service watchdog or a concurrent
-        // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage
-        // backend (Disk, S3, Azure) commits all bytes received up to the interruption and updates
-        // the offset in storage. We reload the updated UploadInfo and acknowledge the partial
-        // upload rather than propagating an unhandled error to the servlet container.
-        if (interruptibleStream != null && interruptibleStream.isInterrupted()) {
-          // Refresh UploadInfo first so the log statement and response reflect the true committed
-          // byte offset persisted by the storage backend up to the interruption point.
-          UploadInfo refreshed = uploadStorageService.getUploadInfo(uploadUri, ownerKey);
-          if (refreshed != null) {
-            uploadInfo = refreshed;
+    if (hasContent) {
+      InputStream is = servletRequest.getContentInputStream();
+      if (is != null) {
+        interruptibleStream = new InterruptibleInputStream(is);
+        uploadLockingService.registerInputStream(uploadUri, interruptibleStream);
+        is = interruptibleStream;
+        try {
+          UploadInfo appended = uploadStorageService.append(uploadInfo, is);
+          if (appended != null) {
+            uploadInfo = appended;
+          }
+        } catch (IOException e) {
+          // When an upload stream is interrupted by the locking service watchdog or a concurrent
+          // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage
+          // backend (Disk, S3, Azure) commits all bytes received up to the interruption and updates
+          // the offset in storage. We reload the updated UploadInfo and acknowledge the partial
+          // upload rather than propagating an unhandled error to the servlet container.
+          if (interruptibleStream != null && interruptibleStream.isInterrupted()) {
+            // Refresh UploadInfo first so the log statement and response reflect the true committed
+            // byte offset persisted by the storage backend up to the interruption point.
+            UploadInfo refreshed = uploadStorageService.getUploadInfo(uploadUri, ownerKey);
+            if (refreshed != null) {
+              uploadInfo = refreshed;
+            }
+            log.info(
+                "RUFH creation request with body for URI {} was interrupted by locking service"
+                    + " contention; saved partial upload up to offset {}",
+                uploadUri,
+                uploadInfo != null ? uploadInfo.getOffset() : "unknown");
+          } else {
+            throw e;
           }
-          log.info(
-              "RUFH creation request with body for URI {} was interrupted by locking service"
-                  + " contention; saved partial upload up to offset {}",
-              uploadUri,
-              uploadInfo != null ? uploadInfo.getOffset() : "unknown");
-        } else {
-          throw e;
         }
       }
     }

src/main/java/me/desair/tus/server/rufh/validation/RufhAppendValidator.java

@@ -154,11 +154,14 @@ public void validate(
     // Section 4.1.4: min-append-size validation with exemption for Upload-Complete: ?1
     // "This limit does not apply to upload creation requests with no content, or to requests
     // completing the upload by including the Upload-Complete: ?1 header field."
+    // Only validate minAppendSize by Content-Length header when contentLength >= 0. For chunked
+    // requests (Transfer-Encoding: chunked, where contentLength is -1), payload size cannot be
+    // determined from headers alone and is enforced by the storage service append() method.
     Long minAppendSize = uploadStorageService.getMinAppendSize();
     Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader);
     boolean isCompleteExempt = Boolean.TRUE.equals(uploadComplete);
 
-    if (minAppendSize != null && minAppendSize > 0 && !isCompleteExempt) {
+    if (minAppendSize != null && minAppendSize > 0 && !isCompleteExempt && contentLength >= 0) {
       if (contentLength < minAppendSize) {
         throw new MinAppendSizeNotMetException(
             "The request payload size ("

src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java

@@ -247,8 +247,16 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream)
         File chunkFile = null;
         boolean handedOff = false;
         try {
+          long maxBytesToRead = optimalBlockSize;
+          if (effectiveMaxAppendSize != null) {
+            // Read at most 1 byte beyond the remaining allowed append size so oversized streams
+            // are interrupted immediately without buffering large amounts of excess data to disk.
+            long remainingAllowed = Math.max(0L, effectiveMaxAppendSize - totalAppended);
+            maxBytesToRead = Math.min(optimalBlockSize, remainingAllowed + 1L);
+          }
+
           chunkFile = Files.createTempFile(tempBufferDir, "tus-azure-chunk-", ".tmp").toFile();
-          ReadChunkResult chunkResult = readChunk(inputStream, chunkFile, optimalBlockSize);
+          ReadChunkResult chunkResult = readChunk(inputStream, chunkFile, maxBytesToRead);
           long chunkSize = chunkResult.bytesRead;
           if (chunkResult.exception != null) {
             streamException = chunkResult.exception;
@@ -258,18 +266,10 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream)
             break;
           }
 
-          if (effectiveMaxAppendSize != null
-              && (totalAppended + chunkSize) > effectiveMaxAppendSize) {
-            // If maxAppendSize is exceeded, stop reading from the stream but commit
-            // previously staged blocks so no uploaded data or offset is corrupted.
-            pendingTusException =
-                new MaxAppendSizeExceededException(
-                    "Append payload size "
-                        + (totalAppended + chunkSize)
-                        + " exceeded limit of "
-                        + effectiveMaxAppendSize);
-            break;
-          }
+          // Validate append size against effectiveMaxAppendSize. If exceeded, throws
+          // MaxAppendSizeExceededException, which is caught below as pendingTusException to stop
+          // reading while still committing any previously staged blocks without data loss.
+          validateMaxAppendSize(totalAppended + chunkSize, effectiveMaxAppendSize);
 
           int plannedIndex = blockIds.size() + plannedBlockIds.size();
           // Validate chunk against remaining block budget