Fix #720: fail to write very long Smile strings

release-notes/CREDITS-2.x

@@ -451,3 +451,9 @@ Shanchao Li (@tonghuaroot)
 * Reported #693: (avro) Incomplete number length validation in Avro
   decoder (for `BigDecimal`)
  (2.21.4)
+
+Benjamin Muschko (@bmuschko)
+
+* Reported #720: (smile) `SmileGenerator` int overflow in `maxLen` for very long
+  Strings causes `ArrayIndexOutOfBoundsException`
+ (2.21.6)

release-notes/VERSION-2.x

@@ -22,6 +22,9 @@ Active maintainers:
   buffer reload
 #715: (protobuf) `ProtobufGenerator._reportWrongWireType()` always reports `string`,
   ignoring actual type
+#720: (smile) `SmileGenerator` int overflow in `maxLen` for very long Strings
+  (>= 715,827,882 chars) causes `ArrayIndexOutOfBoundsException`
+ (reported by Benjamin M)
 
 2.21.5 (06-Jul-2026)
 

smile/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileGenerator.java

@@ -811,7 +811,7 @@ private final void _writeNonShortFieldName(final String name, final int len) thr
         _writeByte(TOKEN_KEY_LONG_STRING);
         // can we still make a temp copy?
         // but will encoded version fit in buffer?
-        int maxLen = len + len + len;
+        long maxLen = (long) len + len + len;
         if (maxLen <= _outputBuffer.length) { // yes indeed
             if ((_outputTail + maxLen) >= _outputEnd) {
                 _flushBuffer();
@@ -1046,7 +1046,7 @@ private final void _writeSharedStringValueReference(int ix) throws IOException
     private final void _writeNonSharedString(final String text, final int len) throws IOException
     {
         // Expansion can be 3x for Unicode; and then there's type byte and end marker, so:
-        int maxLen = len + len + len + 2;
+        long maxLen = (long) len + len + len + 2;
         // Next: does it always fit within output buffer?
         if (maxLen > _outputBuffer.length) { // nope
             // can't rewrite type buffer, so can't speculate it might be all-ASCII
@@ -1107,7 +1107,7 @@ public void writeString(char[] text, int offset, int len) throws IOException
             _outputBuffer[origOffset] = typeToken;
         } else { // "long" String, never shared
             // but might still fit within buffer?
-            int maxLen = len + len + len + 2;
+            long maxLen = (long) len + len + len + 2;
             if (maxLen <= _outputBuffer.length) { // yes indeed
                 if ((_outputTail + maxLen) >= _outputEnd) {
                     _flushBuffer();
@@ -1215,7 +1215,7 @@ public void writeRawUTF8String(byte[] text, int offset, int len)
             }
         } else { // "long" String
             // but might still fit within buffer?
-            int maxLen = len + len + len + 2;
+            long maxLen = (long) len + len + len + 2;
             if (maxLen <= _outputBuffer.length) { // yes indeed
                 if ((_outputTail + maxLen) >= _outputEnd) {
                     _flushBuffer();

smile/src/test/java/com/fasterxml/jackson/dataformat/smile/gen/SmileGeneratorHugeStringTest.java

@@ -0,0 +1,119 @@
+package com.fasterxml.jackson.dataformat.smile.gen;
+
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.dataformat.smile.BaseTestForSmile;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class SmileGeneratorHugeStringTest extends BaseTestForSmile
+{
+    // Length chosen so that BOTH `3*len` (long field-name guard) and `3*len + 2`
+    // (String/char[]/raw-UTF8 guards) overflow a signed int to a negative value:
+    //   3 * 715_827_883      = 2_147_483_649 -> overflows (Integer.MAX_VALUE is 2_147_483_647)
+    //   3 * 715_827_883 + 2  = 2_147_483_651 -> overflows
+    private final static int OVERFLOW_LEN = 715_827_883;
+
+    // Require a comfortable margin of heap before even attempting allocation. Peak on JDK 8
+    // (no compact strings) is char[len] (~1.4 GB) + a copied String char[] (~1.4 GB) ~= 2.9 GB,
+    // so gate well above that to skip cleanly rather than OOM on mid-sized heaps.
+    private final static long REQUIRED_HEAP = 4L * 1024 * 1024 * 1024; // 4 GB
+
+    // Discards output but counts bytes, so we do not also retain ~700 MB of encoded data
+    private static final class CountingOutputStream extends OutputStream {
+        long count;
+
+        @Override
+        public void write(int b) { count++; }
+
+        @Override
+        public void write(byte[] b, int off, int len) { count += len; }
+    }
+
+    private static char[] newAsciiChars() {
+        char[] chars = new char[OVERFLOW_LEN];
+        Arrays.fill(chars, 'a'); // all-ASCII to hit the fast in-buffer encoding loop
+        return chars;
+    }
+
+    // (1) String value path: writeString(String) -> _writeNonSharedString
+    @Test
+    public void testHugeStringValueDoesNotOverflowBuffer() throws Exception
+    {
+        assumeTrue(Runtime.getRuntime().maxMemory() >= REQUIRED_HEAP,
+                "Requires large heap (>= 4 GB) to allocate a ~700M char String");
+
+        char[] chars = newAsciiChars();
+        String big = new String(chars);
+        chars = null; // release the source array before encoding
+
+        CountingOutputStream out = new CountingOutputStream();
+        try (JsonGenerator gen = smileGenerator(out, true)) {
+            gen.writeString(big);
+        }
+        // type byte + payload + end marker: must have written the whole thing
+        assertTrue(out.count > OVERFLOW_LEN,
+                "Expected > " + OVERFLOW_LEN + " bytes, got " + out.count);
+    }
+
+    // (2) char[] value path: writeString(char[], off, len)
+    @Test
+    public void testHugeCharArrayValueDoesNotOverflowBuffer() throws Exception
+    {
+        assumeTrue(Runtime.getRuntime().maxMemory() >= REQUIRED_HEAP,
+                "Requires large heap (>= 4 GB) to allocate a ~700M char[]");
+
+        char[] chars = newAsciiChars();
+        CountingOutputStream out = new CountingOutputStream();
+        try (JsonGenerator gen = smileGenerator(out, true)) {
+            gen.writeString(chars, 0, chars.length);
+        }
+        assertTrue(out.count > OVERFLOW_LEN,
+                "Expected > " + OVERFLOW_LEN + " bytes, got " + out.count);
+    }
+
+    // (3) long field-name path: writeFieldName(String) -> _writeNonShortFieldName
+    @Test
+    public void testHugeFieldNameDoesNotOverflowBuffer() throws Exception
+    {
+        assumeTrue(Runtime.getRuntime().maxMemory() >= REQUIRED_HEAP,
+                "Requires large heap (>= 4 GB) to allocate a ~700M char field name");
+
+        char[] chars = newAsciiChars();
+        String bigName = new String(chars);
+        chars = null; // release the source array before encoding
+
+        CountingOutputStream out = new CountingOutputStream();
+        try (JsonGenerator gen = smileGenerator(out, true)) {
+            gen.writeStartObject();
+            gen.writeFieldName(bigName);
+            gen.writeNull();
+            gen.writeEndObject();
+        }
+        assertTrue(out.count > OVERFLOW_LEN,
+                "Expected > " + OVERFLOW_LEN + " bytes, got " + out.count);
+    }
+
+    // (4) raw UTF-8 value path: writeRawUTF8String(byte[], off, len)
+    @Test
+    public void testHugeRawUTF8StringDoesNotOverflowBuffer() throws Exception
+    {
+        assumeTrue(Runtime.getRuntime().maxMemory() >= REQUIRED_HEAP,
+                "Requires large heap (>= 4 GB) to allocate a ~700M byte[]");
+
+        byte[] bytes = new byte[OVERFLOW_LEN];
+        Arrays.fill(bytes, (byte) 'a'); // all-ASCII so byteLen == len
+
+        CountingOutputStream out = new CountingOutputStream();
+        try (JsonGenerator gen = smileGenerator(out, true)) {
+            gen.writeRawUTF8String(bytes, 0, bytes.length);
+        }
+        assertTrue(out.count > OVERFLOW_LEN,
+                "Expected > " + OVERFLOW_LEN + " bytes, got " + out.count);
+    }
+}