From 7556ad0b9a573d412716c9f1e9e3278112ac42f2 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 06:36:04 +0800 Subject: [PATCH 1/4] [common] Drop grouping quotes in array and row string cast rules StringToArrayCastRule.splitArrayElements and StringToRowCastRule.splitRowFields toggle quote state on a double quote but then still append the character, so the quote characters stay embedded in the parsed token: an ARRAY default value like ["a,b", c] silently stores the element "a,b" with literal quotes, and non-string element types fail parsing instead. The map rule already drops its grouping quotes. Skip the append when the quote toggles state, in both rules: quotes group a token across separators and are not part of the value. Assisted-by: GLM-5.3 --- .../paimon/casting/StringToArrayCastRule.java | 3 ++ .../paimon/casting/StringToRowCastRule.java | 3 ++ .../paimon/casting/CastExecutorTest.java | 32 +++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java index 2e48e0b82582..c1dacc12b557 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java @@ -138,7 +138,10 @@ private List splitArrayElements(String content) { } else if (c == '\\') { escaped = true; } else if (c == '"') { + // quotes group a token across separators; the quote characters + // themselves are dropped rather than embedded in the token inQuotes = !inQuotes; + continue; } else if (!inQuotes) { if (StringUtils.isOpenBracket(c)) { bracketStack.push(c); diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java index 5a2379d273a2..643ebf711323 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java @@ -169,7 +169,10 @@ private List splitRowFields(String content) { } else if (c == '\\') { escaped = true; } else if (c == '"') { + // quotes group a token across separators; the quote characters + // themselves are dropped rather than embedded in the token inQuotes = !inQuotes; + continue; } else if (!inQuotes) { if (StringUtils.isOpenBracket(c)) { bracketStack.push(c); diff --git a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java index 31a776c87da3..b34be09f5b6a 100644 --- a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java @@ -975,6 +975,38 @@ public void testRowToString() { BinaryString.fromString("{1, {2025-01-06, {1 -> [1, null, 2]}, null}}")); } + @Test + public void testStringToArrayPreservesQuotedSeparator() { + ArrayType arrayType = new ArrayType(DataTypes.STRING()); + compareCastResult( + CastExecutors.resolve(VarCharType.STRING_TYPE, arrayType), + BinaryString.fromString("[\"a,b\", c]"), + new GenericArray( + new Object[] { + BinaryString.fromString("a,b"), BinaryString.fromString("c") + })); + + // an empty quoted token is dropped like in the map rule: empty-string elements + // are not expressible in this mini-language + compareCastResult( + CastExecutors.resolve(VarCharType.STRING_TYPE, arrayType), + BinaryString.fromString("[\"\", a]"), + new GenericArray(new Object[] {BinaryString.fromString("a")})); + } + + @Test + public void testStringToRowPreservesQuotedSeparator() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD(0, "f0", DataTypes.STRING()), + DataTypes.FIELD(1, "f1", DataTypes.INT())); + GenericRow expected = GenericRow.of(BinaryString.fromString("a,b"), 2); + compareCastResult( + CastExecutors.resolve(VarCharType.STRING_TYPE, rowType), + BinaryString.fromString("{\"a,b\", 2}"), + expected); + } + @Test public void testSplitMapEntriesWithQuotes() { String content = "1, \"abc\""; From 836ef4d1f992010cba10d32d22b8782942a7fb2d Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 13:39:02 +0800 Subject: [PATCH 2/4] fix: make quoting mean literal in array and row cast rules Stripping the quote characters was only half of it. The splitter forgot that a token had been quoted, so quoting could not express the two values it exists for: an empty quoted token vanished (changing an array's element count and breaking a row's field count) and a quoted "null" became SQL NULL instead of the four-character string. Escapes had the same shape of bug in the other direction: the backslash survived into the value, so nothing could be escaped. Both rules now share one splitter that reports whether a token was quoted. A quoted token keeps its inner whitespace and is never read as the null literal; an unquoted empty token is still absent, so a trailing or doubled separator does not invent a value. The map rule is untouched: it splits entries rather than values, so quote state at the entry level does not tell it whether a key or value was quoted, and fixing it needs quote-aware key/value splitting. Co-Authored-By: Claude Code --- .../paimon/casting/StringToArrayCastRule.java | 52 +------- .../paimon/casting/StringToRowCastRule.java | 60 ++------- .../apache/paimon/casting/TokenSplitter.java | 124 ++++++++++++++++++ .../paimon/casting/CastExecutorTest.java | 61 +++++++-- 4 files changed, 188 insertions(+), 109 deletions(-) create mode 100644 paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java index c1dacc12b557..93f5657c2877 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java @@ -26,11 +26,9 @@ import org.apache.paimon.types.DataTypeFamily; import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.VarCharType; -import org.apache.paimon.utils.StringUtils; import java.util.ArrayList; import java.util.List; -import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -114,55 +112,15 @@ private String extractArrayContent(String str) { private List parseArrayElements( String content, CastExecutor elementCastExecutor) { List elements = new ArrayList<>(); - for (String token : splitArrayElements(content)) { - String trimmedToken = token.trim(); + for (TokenSplitter.Token token : TokenSplitter.split(content)) { + String value = token.value(); + // only an unquoted null is the null element; "null" is the four-character string Object element = - "null".equals(trimmedToken) + !token.quoted() && "null".equals(value) ? null - : elementCastExecutor.cast(BinaryString.fromString(trimmedToken)); + : elementCastExecutor.cast(BinaryString.fromString(value)); elements.add(element); } return elements; } - - private List splitArrayElements(String content) { - List elements = new ArrayList<>(); - StringBuilder current = new StringBuilder(); - Stack bracketStack = new Stack<>(); - boolean inQuotes = false; - boolean escaped = false; - - for (char c : content.toCharArray()) { - if (escaped) { - escaped = false; - } else if (c == '\\') { - escaped = true; - } else if (c == '"') { - // quotes group a token across separators; the quote characters - // themselves are dropped rather than embedded in the token - inQuotes = !inQuotes; - continue; - } else if (!inQuotes) { - if (StringUtils.isOpenBracket(c)) { - bracketStack.push(c); - } else if (StringUtils.isCloseBracket(c) && !bracketStack.isEmpty()) { - bracketStack.pop(); - } else if (c == ',' && bracketStack.isEmpty()) { - addCurrentElement(elements, current); - continue; - } - } - current.append(c); - } - - addCurrentElement(elements, current); - return elements; - } - - private void addCurrentElement(List elements, StringBuilder current) { - if (current.length() > 0) { - elements.add(current.toString()); - current.setLength(0); - } - } } diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java index 643ebf711323..e7f830e6a914 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java @@ -26,11 +26,8 @@ import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.RowType; import org.apache.paimon.types.VarCharType; -import org.apache.paimon.utils.StringUtils; -import java.util.ArrayList; import java.util.List; -import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -93,7 +90,7 @@ private InternalRow parseRow( if (content.isEmpty()) { return createNullRow(fieldCount); } - List fieldValues = splitRowFields(content); + List fieldValues = TokenSplitter.split(content); if (fieldValues.size() != fieldCount) { throw new RuntimeException( "Row field count mismatch. Expected: " @@ -137,63 +134,22 @@ private GenericRow createNullRow(int fieldCount) { } private GenericRow createRowFromFields( - List fieldValues, + List fieldValues, CastExecutor[] fieldCastExecutors, int fieldCount) { GenericRow row = new GenericRow(fieldCount); for (int i = 0; i < fieldCount; i++) { - String fieldValue = fieldValues.get(i).trim(); - Object value = parseFieldValue(fieldValue, fieldCastExecutors[i]); - row.setField(i, value); + row.setField(i, parseFieldValue(fieldValues.get(i), fieldCastExecutors[i])); } return row; } private Object parseFieldValue( - String fieldValue, CastExecutor castExecutor) { - return "null".equals(fieldValue) + TokenSplitter.Token token, CastExecutor castExecutor) { + String value = token.value(); + // only an unquoted null is the null field; "null" is the four-character string + return !token.quoted() && "null".equals(value) ? null - : castExecutor.cast(BinaryString.fromString(fieldValue)); - } - - private List splitRowFields(String content) { - List fields = new ArrayList<>(); - StringBuilder current = new StringBuilder(); - Stack bracketStack = new Stack<>(); - boolean inQuotes = false; - boolean escaped = false; - - for (char c : content.toCharArray()) { - if (escaped) { - escaped = false; - } else if (c == '\\') { - escaped = true; - } else if (c == '"') { - // quotes group a token across separators; the quote characters - // themselves are dropped rather than embedded in the token - inQuotes = !inQuotes; - continue; - } else if (!inQuotes) { - if (StringUtils.isOpenBracket(c)) { - bracketStack.push(c); - } else if (StringUtils.isCloseBracket(c) && !bracketStack.isEmpty()) { - bracketStack.pop(); - } else if (c == ',' && bracketStack.isEmpty()) { - addCurrentField(fields, current); - continue; - } - } - current.append(c); - } - - addCurrentField(fields, current); - return fields; - } - - private void addCurrentField(List fields, StringBuilder current) { - if (current.length() > 0) { - fields.add(current.toString()); - current.setLength(0); - } + : castExecutor.cast(BinaryString.fromString(value)); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java new file mode 100644 index 000000000000..67aaddd59677 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.casting; + +import org.apache.paimon.utils.StringUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; + +/** + * Splits the comma-separated body of an array or row literal into its tokens, honouring quotes, + * escapes and nesting. + * + *

A separator only separates outside quotes and at bracket depth zero, so {@code "a,b"} and + * {@code [a, b]} each stay one token. Quotes and backslashes are grouping syntax and do not survive + * into the value, but whether a token was quoted does: quoting is how the literal text {@code null} + * and the empty string are written, which are otherwise unrepresentable. + * + *

Whitespace around a token is dropped; whitespace inside quotes is kept. + */ +class TokenSplitter { + + /** One token of a literal body, plus whether quotes contributed to it. */ + static class Token { + + private final String value; + private final boolean quoted; + + Token(String value, boolean quoted) { + this.value = value; + this.quoted = quoted; + } + + String value() { + return value; + } + + /** Whether the token was written with quotes, which makes its value a literal string. */ + boolean quoted() { + return quoted; + } + } + + private TokenSplitter() {} + + static List split(String content) { + List tokens = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + Stack bracketStack = new Stack<>(); + boolean inQuotes = false; + boolean escaped = false; + boolean quoted = false; + // length of current up to the last character that was not unquoted whitespace + int end = 0; + + for (char c : content.toCharArray()) { + if (escaped) { + // an escaped character stands for itself; the backslash is syntax + escaped = false; + current.append(c); + end = current.length(); + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == '"') { + inQuotes = !inQuotes; + quoted = true; + continue; + } + if (!inQuotes) { + if (StringUtils.isOpenBracket(c)) { + bracketStack.push(c); + } else if (StringUtils.isCloseBracket(c) && !bracketStack.isEmpty()) { + bracketStack.pop(); + } else if (c == ',' && bracketStack.isEmpty()) { + addToken(tokens, current, end, quoted); + current.setLength(0); + end = 0; + quoted = false; + continue; + } else if (Character.isWhitespace(c) && end == 0) { + // leading whitespace outside quotes is not part of the token + continue; + } + } + current.append(c); + if (inQuotes || !Character.isWhitespace(c)) { + end = current.length(); + } + } + + addToken(tokens, current, end, quoted); + return tokens; + } + + private static void addToken( + List tokens, StringBuilder current, int end, boolean quoted) { + // an empty unquoted token is absent rather than empty, so a trailing or doubled + // separator does not invent a value + if (end > 0 || quoted) { + tokens.add(new Token(current.substring(0, end), quoted)); + } + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java index b34be09f5b6a..2d16b4ccd116 100644 --- a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java @@ -976,35 +976,76 @@ public void testRowToString() { } @Test - public void testStringToArrayPreservesQuotedSeparator() { + public void testStringToArrayQuotingAndEscaping() { ArrayType arrayType = new ArrayType(DataTypes.STRING()); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, arrayType); + + // quotes group a token across the separator and do not survive into the value compareCastResult( - CastExecutors.resolve(VarCharType.STRING_TYPE, arrayType), + cast, BinaryString.fromString("[\"a,b\", c]"), new GenericArray( new Object[] { BinaryString.fromString("a,b"), BinaryString.fromString("c") })); - // an empty quoted token is dropped like in the map rule: empty-string elements - // are not expressible in this mini-language + // quoting is how an empty string is written, so the element must be kept compareCastResult( - CastExecutors.resolve(VarCharType.STRING_TYPE, arrayType), + cast, BinaryString.fromString("[\"\", a]"), - new GenericArray(new Object[] {BinaryString.fromString("a")})); + new GenericArray( + new Object[] {BinaryString.fromString(""), BinaryString.fromString("a")})); + + // an unquoted null is the null element; a quoted one is the four-character string + compareCastResult( + cast, + BinaryString.fromString("[null, \"null\"]"), + new GenericArray(new Object[] {null, BinaryString.fromString("null")})); + + // a backslash escapes the next character and is itself syntax + compareCastResult( + cast, + BinaryString.fromString("[a\\,b, c]"), + new GenericArray( + new Object[] { + BinaryString.fromString("a,b"), BinaryString.fromString("c") + })); } @Test - public void testStringToRowPreservesQuotedSeparator() { + public void testStringToRowQuotingAndEscaping() { RowType rowType = DataTypes.ROW( DataTypes.FIELD(0, "f0", DataTypes.STRING()), DataTypes.FIELD(1, "f1", DataTypes.INT())); - GenericRow expected = GenericRow.of(BinaryString.fromString("a,b"), 2); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, rowType); + compareCastResult( - CastExecutors.resolve(VarCharType.STRING_TYPE, rowType), + cast, BinaryString.fromString("{\"a,b\", 2}"), - expected); + GenericRow.of(BinaryString.fromString("a,b"), 2)); + + // an empty quoted field stays a field, so the field count still matches + compareCastResult( + cast, + BinaryString.fromString("{\"\", 2}"), + GenericRow.of(BinaryString.fromString(""), 2)); + + // a quoted null is the string, an unquoted one is SQL NULL + compareCastResult( + cast, + BinaryString.fromString("{\"null\", 2}"), + GenericRow.of(BinaryString.fromString("null"), 2)); + compareCastResult(cast, BinaryString.fromString("{null, 2}"), GenericRow.of(null, 2)); + + compareCastResult( + cast, + BinaryString.fromString("{a\\,b, 2}"), + GenericRow.of(BinaryString.fromString("a,b"), 2)); } @Test From c522cdea34b764c90446e4b000c88f7027cc7424 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 14:18:56 +0800 Subject: [PATCH 3/4] fix: strip quotes and escapes only at the level that wrote them The splitter removed quote and backslash characters at every bracket depth, so a nested literal lost its own syntax before the element rule saw it: the inner separator stopped being protected and [["a,b"], [c]] became a 2-element inner array, while ARRAY> and ARRAY> turned into "Invalid map entry format" and "Row field count mismatch". Strip them at depth zero and pass them through verbatim inside a nested literal, tracking quote state at every depth so a quoted bracket still cannot move the depth. Two more from the same review: An escape now marks its token a literal like a quote does, so [\null] is the string and not SQL NULL. A field written as whitespace is empty rather than absent. Only a token that received no character at all is dropped, which keeps [a, ] at one element and [a,,b] at two while restoring {a, ,b} to three fields instead of failing the field count. Co-Authored-By: Claude Code --- .../paimon/casting/StringToArrayCastRule.java | 2 +- .../paimon/casting/StringToRowCastRule.java | 2 +- .../apache/paimon/casting/TokenSplitter.java | 67 +++++++++++------ .../paimon/casting/CastExecutorTest.java | 74 +++++++++++++++++++ 4 files changed, 122 insertions(+), 23 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java index 93f5657c2877..45a1cb1d97a2 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java @@ -116,7 +116,7 @@ private List parseArrayElements( String value = token.value(); // only an unquoted null is the null element; "null" is the four-character string Object element = - !token.quoted() && "null".equals(value) + !token.literal() && "null".equals(value) ? null : elementCastExecutor.cast(BinaryString.fromString(value)); elements.add(element); diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java index e7f830e6a914..1ceaf1c2bc60 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java @@ -148,7 +148,7 @@ private Object parseFieldValue( TokenSplitter.Token token, CastExecutor castExecutor) { String value = token.value(); // only an unquoted null is the null field; "null" is the four-character string - return !token.quoted() && "null".equals(value) + return !token.literal() && "null".equals(value) ? null : castExecutor.cast(BinaryString.fromString(value)); } diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java index 67aaddd59677..9e79ca585526 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java @@ -29,32 +29,36 @@ * escapes and nesting. * *

A separator only separates outside quotes and at bracket depth zero, so {@code "a,b"} and - * {@code [a, b]} each stay one token. Quotes and backslashes are grouping syntax and do not survive - * into the value, but whether a token was quoted does: quoting is how the literal text {@code null} - * and the empty string are written, which are otherwise unrepresentable. + * {@code [a, b]} each stay one token. + * + *

Quotes and backslashes are this level's syntax, so they are removed at depth zero and kept + * verbatim inside a nested literal, where they are the inner level's syntax and the rule for that + * element parses them again. Whether they appeared at depth zero is remembered: quoting or escaping + * is how the literal text {@code null} and the empty string are written, which are otherwise + * unrepresentable. * *

Whitespace around a token is dropped; whitespace inside quotes is kept. */ class TokenSplitter { - /** One token of a literal body, plus whether quotes contributed to it. */ + /** One token of a literal body, plus whether its value was written as a literal. */ static class Token { private final String value; - private final boolean quoted; + private final boolean literal; - Token(String value, boolean quoted) { + Token(String value, boolean literal) { this.value = value; - this.quoted = quoted; + this.literal = literal; } String value() { return value; } - /** Whether the token was written with quotes, which makes its value a literal string. */ - boolean quoted() { - return quoted; + /** Whether quotes or an escape made this a literal string rather than a bare word. */ + boolean literal() { + return literal; } } @@ -66,25 +70,43 @@ static List split(String content) { Stack bracketStack = new Stack<>(); boolean inQuotes = false; boolean escaped = false; - boolean quoted = false; + boolean literal = false; + // whether this token received any character at all, whitespace included + boolean present = false; // length of current up to the last character that was not unquoted whitespace int end = 0; for (char c : content.toCharArray()) { + boolean nested = !bracketStack.isEmpty(); if (escaped) { - // an escaped character stands for itself; the backslash is syntax + // the escapee stands for itself and is never read as syntax escaped = false; + present = true; current.append(c); end = current.length(); continue; } if (c == '\\') { escaped = true; + present = true; + if (nested) { + // the inner rule has to see the escape to protect its own separators + current.append(c); + end = current.length(); + } else { + literal = true; + } continue; } if (c == '"') { inQuotes = !inQuotes; - quoted = true; + present = true; + if (nested) { + current.append(c); + end = current.length(); + } else { + literal = true; + } continue; } if (!inQuotes) { @@ -93,32 +115,35 @@ static List split(String content) { } else if (StringUtils.isCloseBracket(c) && !bracketStack.isEmpty()) { bracketStack.pop(); } else if (c == ',' && bracketStack.isEmpty()) { - addToken(tokens, current, end, quoted); + addToken(tokens, current, end, literal, present); current.setLength(0); end = 0; - quoted = false; + literal = false; + present = false; continue; } else if (Character.isWhitespace(c) && end == 0) { // leading whitespace outside quotes is not part of the token + present = true; continue; } } + present = true; current.append(c); if (inQuotes || !Character.isWhitespace(c)) { end = current.length(); } } - addToken(tokens, current, end, quoted); + addToken(tokens, current, end, literal, present); return tokens; } private static void addToken( - List tokens, StringBuilder current, int end, boolean quoted) { - // an empty unquoted token is absent rather than empty, so a trailing or doubled - // separator does not invent a value - if (end > 0 || quoted) { - tokens.add(new Token(current.substring(0, end), quoted)); + List tokens, StringBuilder current, int end, boolean literal, boolean present) { + // a token that never received a character is absent rather than empty, so a trailing or + // doubled separator does not invent a value; one that received only whitespace is empty + if (present || literal) { + tokens.add(new Token(current.substring(0, end), literal)); } } } diff --git a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java index 2d16b4ccd116..4dfe210401ab 100644 --- a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java @@ -1048,6 +1048,80 @@ public void testStringToRowQuotingAndEscaping() { GenericRow.of(BinaryString.fromString("a,b"), 2)); } + @Test + public void testStringToNestedArrayKeepsInnerSyntax() { + // quotes and escapes belong to whichever level wrote them: the outer split must leave a + // nested literal's own syntax in place for the element rule to parse again, or the inner + // separator stops being protected and the element count changes + ArrayType nested = new ArrayType(new ArrayType(DataTypes.STRING())); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, nested); + + assertNestedElements(cast, "[[\"a,b\"], [c]]", new String[] {"a,b"}, new String[] {"c"}); + assertNestedElements(cast, "[[a\\,b], [c]]", new String[] {"a,b"}, new String[] {"c"}); + assertNestedElements(cast, "[[\"null\"], [a]]", new String[] {"null"}, new String[] {"a"}); + assertNestedElements(cast, "[[\"\"], [a]]", new String[] {""}, new String[] {"a"}); + assertNestedElements(cast, "[[\" a \"], [b]]", new String[] {" a "}, new String[] {"b"}); + assertNestedElements(cast, "[[1, 2], [3]]", new String[] {"1", "2"}, new String[] {"3"}); + } + + private static void assertNestedElements( + CastExecutor cast, String literal, String[]... expected) { + InternalArray outer = cast.cast(BinaryString.fromString(literal)); + assertThat(outer.size()).as("outer size of %s", literal).isEqualTo(expected.length); + for (int i = 0; i < expected.length; i++) { + InternalArray inner = outer.getArray(i); + assertThat(inner.size()) + .as("inner size of %s at %s", literal, i) + .isEqualTo(expected[i].length); + for (int j = 0; j < expected[i].length; j++) { + assertThat(inner.getString(j).toString()) + .as("element %s.%s of %s", i, j, literal) + .isEqualTo(expected[i][j]); + } + } + } + + @Test + public void testStringToArrayEscapedNullIsALiteral() { + ArrayType arrayType = new ArrayType(DataTypes.STRING()); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, arrayType); + + // escaping, like quoting, says the token is written text rather than the null literal + compareCastResult( + cast, + BinaryString.fromString("[\\null, x]"), + new GenericArray( + new Object[] { + BinaryString.fromString("null"), BinaryString.fromString("x") + })); + } + + @Test + public void testStringToRowKeepsWhitespaceOnlyField() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD(0, "f0", DataTypes.STRING()), + DataTypes.FIELD(1, "f1", DataTypes.STRING()), + DataTypes.FIELD(2, "f2", DataTypes.STRING())); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, rowType); + + // a field written as whitespace is an empty field, not an absent one: dropping it would + // turn a working cast into a field count mismatch + compareCastResult( + cast, + BinaryString.fromString("{a, ,b}"), + GenericRow.of( + BinaryString.fromString("a"), + BinaryString.fromString(""), + BinaryString.fromString("b"))); + } + @Test public void testSplitMapEntriesWithQuotes() { String content = "1, \"abc\""; From 621779af01790469f0b3078f8189c11dab54e1fd Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 16:59:04 +0800 Subject: [PATCH 4/4] fix: treat a whitespace-only token as no token, wherever it sits An empty-but-present field only existed for interior fields, and it handed the empty string to the element cast, which failed the whole array. --- .../apache/paimon/casting/TokenSplitter.java | 20 ++++--------- .../paimon/casting/CastExecutorTest.java | 30 ++++++++++++++++--- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java index 9e79ca585526..54059675db1d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java @@ -71,8 +71,6 @@ static List split(String content) { boolean inQuotes = false; boolean escaped = false; boolean literal = false; - // whether this token received any character at all, whitespace included - boolean present = false; // length of current up to the last character that was not unquoted whitespace int end = 0; @@ -81,14 +79,12 @@ static List split(String content) { if (escaped) { // the escapee stands for itself and is never read as syntax escaped = false; - present = true; current.append(c); end = current.length(); continue; } if (c == '\\') { escaped = true; - present = true; if (nested) { // the inner rule has to see the escape to protect its own separators current.append(c); @@ -100,7 +96,6 @@ static List split(String content) { } if (c == '"') { inQuotes = !inQuotes; - present = true; if (nested) { current.append(c); end = current.length(); @@ -115,34 +110,31 @@ static List split(String content) { } else if (StringUtils.isCloseBracket(c) && !bracketStack.isEmpty()) { bracketStack.pop(); } else if (c == ',' && bracketStack.isEmpty()) { - addToken(tokens, current, end, literal, present); + addToken(tokens, current, end, literal); current.setLength(0); end = 0; literal = false; - present = false; continue; } else if (Character.isWhitespace(c) && end == 0) { // leading whitespace outside quotes is not part of the token - present = true; continue; } } - present = true; current.append(c); if (inQuotes || !Character.isWhitespace(c)) { end = current.length(); } } - addToken(tokens, current, end, literal, present); + addToken(tokens, current, end, literal); return tokens; } private static void addToken( - List tokens, StringBuilder current, int end, boolean literal, boolean present) { - // a token that never received a character is absent rather than empty, so a trailing or - // doubled separator does not invent a value; one that received only whitespace is empty - if (present || literal) { + List tokens, StringBuilder current, int end, boolean literal) { + // whitespace is not part of a token, so one made only of whitespace was never written; + // quoting is how an empty value is written + if (end > 0 || literal) { tokens.add(new Token(current.substring(0, end), literal)); } } diff --git a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java index 4dfe210401ab..8325687c21b2 100644 --- a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java @@ -1101,7 +1101,7 @@ public void testStringToArrayEscapedNullIsALiteral() { } @Test - public void testStringToRowKeepsWhitespaceOnlyField() { + public void testStringToRowWhitespaceOnlyFieldIsNotAField() { RowType rowType = DataTypes.ROW( DataTypes.FIELD(0, "f0", DataTypes.STRING()), @@ -1111,17 +1111,39 @@ public void testStringToRowKeepsWhitespaceOnlyField() { (CastExecutor) CastExecutors.resolve(VarCharType.STRING_TYPE, rowType); - // a field written as whitespace is an empty field, not an absent one: dropping it would - // turn a working cast into a field count mismatch + // whitespace is not part of a token, so a field made only of whitespace was never + // written, and where it sits does not change that + for (String literal : new String[] {"{a, ,b}", "{ ,a,b}", "{a,b, }"}) { + assertThatThrownBy(() -> cast.cast(BinaryString.fromString(literal))) + .as("%s", literal) + .hasMessageContaining("Row field count mismatch. Expected: 3, Actual: 2"); + } + + // quoting is how an empty field is written compareCastResult( cast, - BinaryString.fromString("{a, ,b}"), + BinaryString.fromString("{a, \"\", b}"), GenericRow.of( BinaryString.fromString("a"), BinaryString.fromString(""), BinaryString.fromString("b"))); } + @Test + public void testStringToArraySkipsAnEmptyElement() { + ArrayType arrayType = new ArrayType(DataTypes.INT()); + CastExecutor cast = + (CastExecutor) + CastExecutors.resolve(VarCharType.STRING_TYPE, arrayType); + + // an element written as nothing, with or without whitespace, is no element: handing the + // empty string to the int cast instead would fail the whole array + compareCastResult( + cast, BinaryString.fromString("[1,,3]"), new GenericArray(new Integer[] {1, 3})); + compareCastResult( + cast, BinaryString.fromString("[1, , 3]"), new GenericArray(new Integer[] {1, 3})); + } + @Test public void testSplitMapEntriesWithQuotes() { String content = "1, \"abc\"";