From 8fa106f0d8acff235604dbe8f5262815c848b80e Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 06:55:05 +0800 Subject: [PATCH 1/8] [api] Use Locale.ROOT in StringUtils case conversions toLowerCaseIfNeed, toLowerCase, and toUpperCase converted with the JVM default locale. Under a Turkish or Azeri default locale, 'I' lowercases to a dotless glyph and 'i' uppercases to a dotted capital, so the case-sensitive=false identifier matching used by CDC table mapping, computed columns, and the Arrow readers silently broke for columns containing 'I'/'i'. Convert with Locale.ROOT, and align the record side of the same CDC flow: CdcRecord.fieldNameLowerCase also lowercased with the default locale, so fixing only the schema side would newly diverge the two halves of the record-schema join under tr/az (previously both sides mangled identically and data still flowed). Assisted-by: GLM-5.3 --- .../org/apache/paimon/utils/StringUtils.java | 9 ++-- .../apache/paimon/utils/StringUtilsTest.java | 16 ++++++ .../paimon/flink/sink/cdc/CdcRecord.java | 6 ++- .../paimon/flink/sink/cdc/CdcRecordTest.java | 53 +++++++++++++++++++ 4 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordTest.java diff --git a/paimon-api/src/main/java/org/apache/paimon/utils/StringUtils.java b/paimon-api/src/main/java/org/apache/paimon/utils/StringUtils.java index 4a46deca4d37..9e321436737d 100644 --- a/paimon-api/src/main/java/org/apache/paimon/utils/StringUtils.java +++ b/paimon-api/src/main/java/org/apache/paimon/utils/StringUtils.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Random; import java.util.concurrent.ThreadLocalRandom; @@ -668,7 +669,9 @@ public static String quote(String str) { } public static String toLowerCaseIfNeed(String str, boolean caseSensitive) { - return caseSensitive ? str : str.toLowerCase(); + // Locale.ROOT: identifier matching must not depend on the JVM default locale + // (e.g. Turkish lowercases 'I' to a dotless glyph and breaks column mapping) + return caseSensitive ? str : str.toLowerCase(Locale.ROOT); } public static boolean isNumeric(final CharSequence cs) { @@ -733,14 +736,14 @@ public static String toUpperCase(String value) { if (value == null) { return null; } - return value.toUpperCase(); + return value.toUpperCase(Locale.ROOT); } public static String toLowerCase(String value) { if (value == null) { return null; } - return value.toLowerCase(); + return value.toLowerCase(Locale.ROOT); } public static boolean isOpenBracket(char c) { diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/StringUtilsTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/StringUtilsTest.java index c4cb9b95bc4f..c0bf33c59704 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/StringUtilsTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/StringUtilsTest.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Random; import static org.assertj.core.api.Assertions.assertThat; @@ -451,6 +452,21 @@ void testToLowerCaseIfNeed(String input, boolean caseSensitive, String expected) String result = StringUtils.toLowerCaseIfNeed(input, caseSensitive); assertThat(result).isEqualTo(expected); } + + @Test + void testToLowerCaseIfNeedIndependentOfDefaultLocale() { + Locale original = Locale.getDefault(); + try { + // Turkish lowercases 'I' to a dotless glyph under its locale; + // identifier matching must stay locale-independent + Locale.setDefault(new Locale("tr", "TR")); + assertThat(StringUtils.toLowerCaseIfNeed("INDEX", false)).isEqualTo("index"); + assertThat(StringUtils.toLowerCase("INDEX")).isEqualTo("index"); + assertThat(StringUtils.toUpperCase("ıindex")).isEqualTo("IINDEX"); + } finally { + Locale.setDefault(original); + } + } } @Nested diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java index b23d0d6f06de..3093ebbfb54c 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java @@ -24,6 +24,7 @@ import java.io.Serializable; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -58,7 +59,10 @@ public Map data() { public CdcRecord fieldNameLowerCase() { Map newData = new HashMap<>(); for (Map.Entry entry : data.entrySet()) { - newData.put(entry.getKey().toLowerCase(), entry.getValue()); + // Locale.ROOT: must match the schema-side toLowerCaseIfNeed conversion; + // a Turkish default locale would lowercase 'I' to a dotless glyph and + // silently null out the column on the record-schema join + newData.put(entry.getKey().toLowerCase(Locale.ROOT), entry.getValue()); } return new CdcRecord(kind, newData); } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordTest.java new file mode 100644 index 000000000000..eadbe0fc3630 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink.cdc; + +import org.apache.paimon.types.RowKind; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Test for {@link CdcRecord}. */ +class CdcRecordTest { + + @Test + void fieldNameLowerCaseIndependentOfDefaultLocale() { + Locale original = Locale.getDefault(); + try { + // Turkish lowercases 'I' to a dotless glyph under its locale; the record-side + // keys must match the schema-side Locale.ROOT conversion or the column join + // silently nulls out + Locale.setDefault(new Locale("tr", "TR")); + Map data = new HashMap<>(); + data.put("INDEX", "v"); + CdcRecord record = new CdcRecord(RowKind.INSERT, data); + + CdcRecord converted = record.fieldNameLowerCase(); + + assertThat(converted.data()).containsEntry("index", "v").doesNotContainKey("ındex"); + } finally { + Locale.setDefault(original); + } + } +} From c215217cd0cff9c6a767abd67aa7b433b1fe7bf1 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 13:24:35 +0800 Subject: [PATCH 2/8] fix: use Locale.ROOT for every machine-facing case conversion in api/common/core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixing only StringUtils left the same bug in sites that actually throw. Under a Turkish default locale 'i' uppercases to a dotted capital, so PartitionMarkDoneAction.valueOf("SUCCESS_FILE") gets "SUCCESS_FİLE" and IllegalArgumentException, RowKind.fromShortString("+i") stops matching "+I", JdbcProtocol.valueOf loses SQLITE and MARIADB, and Solaris detection in OperatingSystem stops recognising its own name. The DLF request signers, option key lookups, format identifiers and system table names have the same exposure. Every converted site here is machine-facing: enum names, protocol tokens, option keys, header names, OS names, format identifiers, hex digits. None should follow the JVM default locale. BinaryString.toLowerCase/toUpperCase are left alone: their ASCII path uses Character.toLowerCase and their fallback already pins Locale.ROOT, so the SQL upper()/lower() transforms over user data were never locale-dependent. Co-Authored-By: Claude Code --- .../java/org/apache/paimon/CoreOptions.java | 7 +- .../apache/paimon/options/OptionsUtils.java | 2 +- .../org/apache/paimon/rest/HttpClient.java | 5 +- .../rest/auth/DLFAuthProviderFactory.java | 3 +- .../paimon/rest/auth/DLFDefaultSigner.java | 15 ++-- .../paimon/rest/auth/DLFOpenApiSigner.java | 2 +- .../paimon/types/DataTypeJsonParser.java | 5 +- .../java/org/apache/paimon/types/RowKind.java | 4 +- .../paimon/TurkishLocaleParsingTest.java | 72 +++++++++++++++++++ .../compression/BlockCompressionFactory.java | 4 +- .../org/apache/paimon/format/FileFormat.java | 7 +- .../java/org/apache/paimon/fs/FileIO.java | 3 +- .../globalindex/HybridSearchRanker.java | 3 +- .../statistics/SimpleColStatsCollector.java | 3 +- .../paimon/utils/BlobDescriptorUtils.java | 3 +- .../org/apache/paimon/utils/JNIUtils.java | 3 +- .../apache/paimon/utils/OperatingSystem.java | 4 +- .../apache/paimon/catalog/CatalogUtils.java | 3 +- .../jdbc/DistributedLockDialectFactory.java | 4 +- .../table/ChainTableFileStoreTable.java | 3 +- .../org/apache/paimon/table/FormatTable.java | 3 +- .../table/source/DataEvolutionVectorScan.java | 3 +- .../table/system/SystemTableLoader.java | 3 +- .../paimon/utils/PartitionPathUtils.java | 3 +- 24 files changed, 134 insertions(+), 33 deletions(-) create mode 100644 paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 7b0665c50296..d61aec287f41 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -3273,7 +3273,7 @@ public Map statsModePerLevel() { } public static String normalizeFileFormat(String fileFormat) { - return StringUtils.isEmpty(fileFormat) ? fileFormat : fileFormat.toLowerCase(); + return StringUtils.isEmpty(fileFormat) ? fileFormat : fileFormat.toLowerCase(Locale.ROOT); } public String dataFilePrefix() { @@ -4238,7 +4238,10 @@ public String partitionMarkDoneCustomClass() { public Set partitionMarkDoneActions() { return Arrays.stream(options.get(PARTITION_MARK_DONE_ACTION).split(",")) - .map(x -> PartitionMarkDoneAction.valueOf(x.replace('-', '_').toUpperCase())) + .map( + x -> + PartitionMarkDoneAction.valueOf( + x.replace('-', '_').toUpperCase(Locale.ROOT))) .collect(Collectors.toCollection(HashSet::new)); } diff --git a/paimon-api/src/main/java/org/apache/paimon/options/OptionsUtils.java b/paimon-api/src/main/java/org/apache/paimon/options/OptionsUtils.java index a625454f3996..e0ca14be9b1a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/options/OptionsUtils.java +++ b/paimon-api/src/main/java/org/apache/paimon/options/OptionsUtils.java @@ -199,7 +199,7 @@ static Boolean convertToBoolean(Object o) { return (Boolean) o; } - switch (o.toString().toUpperCase()) { + switch (o.toString().toUpperCase(Locale.ROOT)) { case "TRUE": return true; case "FALSE": diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java index 8205dfe21295..2bee7d2261a5 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.function.Function; @@ -221,7 +222,9 @@ private static String extractRequestId(ClassicHttpResponse response) { .filter( h -> h.getName() != null - && h.getName().toLowerCase().contains("request-id")) + && h.getName() + .toLowerCase(Locale.ROOT) + .contains("request-id")) .map(Header::getValue) .filter(Objects::nonNull) .findFirst() diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java index 1ce7d0a79aa4..3dfac337dcd8 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java @@ -22,6 +22,7 @@ import org.apache.paimon.rest.RESTCatalogOptions; import org.apache.paimon.utils.StringUtils; +import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -96,7 +97,7 @@ protected static String parseSigningAlgoFromUri(String uri) { } // Check for aliyun openapi endpoints - if (uri.toLowerCase().contains("dlfnext")) { + if (uri.toLowerCase(Locale.ROOT).contains("dlfnext")) { return DLFOpenApiSigner.IDENTIFIER; } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java index 76daf5c72f62..d572afa88c95 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java @@ -34,6 +34,7 @@ import java.util.Base64; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.TreeMap; @@ -62,12 +63,12 @@ public class DLFDefaultSigner implements DLFRequestSigner { private static final String NEW_LINE = "\n"; private static final List SIGNED_HEADERS = Arrays.asList( - DLF_CONTENT_MD5_HEADER_KEY.toLowerCase(), - DLF_CONTENT_TYPE_KEY.toLowerCase(), - DLF_CONTENT_SHA56_HEADER_KEY.toLowerCase(), - DLF_DATE_HEADER_KEY.toLowerCase(), - DLF_AUTH_VERSION_HEADER_KEY.toLowerCase(), - DLF_SECURITY_TOKEN_HEADER_KEY.toLowerCase()); + DLF_CONTENT_MD5_HEADER_KEY.toLowerCase(Locale.ROOT), + DLF_CONTENT_TYPE_KEY.toLowerCase(Locale.ROOT), + DLF_CONTENT_SHA56_HEADER_KEY.toLowerCase(Locale.ROOT), + DLF_DATE_HEADER_KEY.toLowerCase(Locale.ROOT), + DLF_AUTH_VERSION_HEADER_KEY.toLowerCase(Locale.ROOT), + DLF_SECURITY_TOKEN_HEADER_KEY.toLowerCase(Locale.ROOT)); private final String region; @@ -215,7 +216,7 @@ private static TreeMap buildSortedSignedHeadersMap( TreeMap orderMap = new TreeMap<>(); if (headers != null) { for (Map.Entry header : headers.entrySet()) { - String key = header.getKey().toLowerCase(); + String key = header.getKey().toLowerCase(Locale.ROOT); if (SIGNED_HEADERS.contains(key)) { orderMap.put(key, StringUtils.trim(header.getValue())); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java index aaf37138283d..0d4858f5817a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java @@ -184,7 +184,7 @@ public String identifier() { private static String buildCanonicalizedHeaders(Map headers) { TreeMap sortedHeaders = new TreeMap<>(); for (Map.Entry entry : headers.entrySet()) { - String key = entry.getKey().toLowerCase(); + String key = entry.getKey().toLowerCase(Locale.ROOT); if (key.startsWith("x-acs-")) { sortedHeaders.put(key, StringUtils.trim(entry.getValue())); } diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java index 5076b577812b..5f26799ab932 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -213,7 +214,7 @@ private static List tokenize(String chars) { builder.setLength(0); cursor = consumeIdentifier(builder, chars, cursor); final String token = builder.toString(); - final String normalizedToken = token.toUpperCase(); + final String normalizedToken = token.toUpperCase(Locale.ROOT); if (KEYWORDS.contains(normalizedToken)) { tokens.add(new Token(TokenType.KEYWORD, cursor, normalizedToken)); } else { @@ -344,7 +345,7 @@ private enum Keyword { private static final Set KEYWORDS = Stream.of(Keyword.values()) - .map(k -> k.toString().toUpperCase()) + .map(k -> k.toString().toUpperCase(Locale.ROOT)) .collect(Collectors.toSet()); private static class Token { diff --git a/paimon-api/src/main/java/org/apache/paimon/types/RowKind.java b/paimon-api/src/main/java/org/apache/paimon/types/RowKind.java index 17096dfbfc6f..f7206d272294 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/RowKind.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/RowKind.java @@ -20,6 +20,8 @@ import org.apache.paimon.annotation.Public; +import java.util.Locale; + /** * Lists all kinds of changes that a row can describe in a changelog. * @@ -135,7 +137,7 @@ public static RowKind fromByteValue(byte value) { * @see #shortString() for mapping of string and {@link RowKind}. */ public static RowKind fromShortString(String value) { - switch (value.toUpperCase()) { + switch (value.toUpperCase(Locale.ROOT)) { case "+I": return INSERT; case "-U": diff --git a/paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java b/paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java new file mode 100644 index 000000000000..483366806e7d --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon; + +import org.apache.paimon.options.Options; +import org.apache.paimon.types.RowKind; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.apache.paimon.CoreOptions.PARTITION_MARK_DONE_ACTION; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Parsing an option value, an enum name or a protocol token uppercases or lowercases it first. + * Under a Turkish default locale 'i' maps to a dotted capital and 'I' to a dotless small letter, so + * those conversions must pin {@link Locale#ROOT} or the token no longer matches what it is compared + * against. + */ +class TurkishLocaleParsingTest { + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void partitionMarkDoneActionsParse() { + // SUCCESS_FILE and DONE_PARTITION both contain an 'i': a locale-sensitive uppercase + // turns them into names no enum constant has, and valueOf throws + Options options = new Options(); + options.set(PARTITION_MARK_DONE_ACTION, "success-file,done-partition"); + + assertThat(new CoreOptions(options).partitionMarkDoneActions()) + .containsExactlyInAnyOrder( + CoreOptions.PartitionMarkDoneAction.SUCCESS_FILE, + CoreOptions.PartitionMarkDoneAction.DONE_PARTITION); + } + + @Test + void rowKindFromLowerCaseShortString() { + assertThat(RowKind.fromShortString("+i")).isEqualTo(RowKind.INSERT); + assertThat(RowKind.fromShortString("-d")).isEqualTo(RowKind.DELETE); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/compression/BlockCompressionFactory.java b/paimon-common/src/main/java/org/apache/paimon/compression/BlockCompressionFactory.java index 60dd1657bc57..563ee625bb40 100644 --- a/paimon-common/src/main/java/org/apache/paimon/compression/BlockCompressionFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/compression/BlockCompressionFactory.java @@ -23,6 +23,8 @@ import javax.annotation.Nullable; +import java.util.Locale; + /** * Each compression codec has an implementation of {@link BlockCompressionFactory} to create * compressors and decompressors. @@ -38,7 +40,7 @@ public interface BlockCompressionFactory { /** Creates {@link BlockCompressionFactory} according to the configuration. */ @Nullable static BlockCompressionFactory create(CompressOptions compression) { - switch (compression.compress().toUpperCase()) { + switch (compression.compress().toUpperCase(Locale.ROOT)) { case "NONE": return null; case "ZSTD": diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java index d6bfe7a0d3fe..1c45e671cf01 100644 --- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java +++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -88,15 +89,15 @@ public static FileFormat fromIdentifier(String identifier, Options options) { /** Create a {@link FileFormat} from format identifier and format options. */ public static FileFormat fromIdentifier(String identifier, FormatContext context) { return FormatFactoryUtil.discoverFactory( - FileFormat.class.getClassLoader(), identifier.toLowerCase()) + FileFormat.class.getClassLoader(), identifier.toLowerCase(Locale.ROOT)) .create(context); } protected Options getIdentifierPrefixOptions(Options options) { Map result = new HashMap<>(); - String prefix = formatIdentifier.toLowerCase() + "."; + String prefix = formatIdentifier.toLowerCase(Locale.ROOT) + "."; for (String key : options.keySet()) { - if (key.toLowerCase().startsWith(prefix)) { + if (key.toLowerCase(Locale.ROOT).startsWith(prefix)) { result.put(prefix + key.substring(prefix.length()), options.get(key)); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java index c20e2c70bb18..06e0bfb709c4 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java @@ -48,6 +48,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Queue; @@ -562,7 +563,7 @@ static FileIO get(Path path, CatalogContext config) throws IOException { for (String[] keys : loader.requiredOptions()) { boolean found = false; for (String key : keys) { - if (options.contains(key.toLowerCase())) { + if (options.contains(key.toLowerCase(Locale.ROOT))) { found = true; break; } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java index e3c910aa4b32..4f506f5d4e42 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java @@ -25,6 +25,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.PriorityQueue; @@ -63,7 +64,7 @@ public static String normalizeRanker(String ranker) { if (ranker == null || ranker.trim().isEmpty()) { return RRF_RANKER; } - String normalized = ranker.trim().toLowerCase(); + String normalized = ranker.trim().toLowerCase(Locale.ROOT); if (!RRF_RANKER.equals(normalized) && !WEIGHTED_SCORE_RANKER.equals(normalized) && !MRR_RANKER.equals(normalized)) { diff --git a/paimon-common/src/main/java/org/apache/paimon/statistics/SimpleColStatsCollector.java b/paimon-common/src/main/java/org/apache/paimon/statistics/SimpleColStatsCollector.java index a9281020444e..c72bea629597 100644 --- a/paimon-common/src/main/java/org/apache/paimon/statistics/SimpleColStatsCollector.java +++ b/paimon-common/src/main/java/org/apache/paimon/statistics/SimpleColStatsCollector.java @@ -22,6 +22,7 @@ import org.apache.paimon.format.SimpleColStats; import java.util.Arrays; +import java.util.Locale; import java.util.regex.Matcher; import static org.apache.paimon.statistics.TruncateSimpleColStatsCollector.TRUNCATE_PATTERN; @@ -62,7 +63,7 @@ static SimpleColStatsCollector[] create(SimpleColStatsCollector.Factory[] factor } static Factory from(String option) { - String upper = option.toUpperCase(); + String upper = option.toUpperCase(Locale.ROOT); switch (upper) { case "NONE": return NoneSimpleColStatsCollector::new; diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java index 1cc1366bc889..ceb328053d65 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.net.URI; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -61,7 +62,7 @@ public static void validateTableRoot(Path tableRoot, BlobDescriptor descriptor) } private static String normalizeScheme(@Nullable String scheme) { - return scheme == null ? null : scheme.toLowerCase(); + return scheme == null ? null : scheme.toLowerCase(Locale.ROOT); } /** diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/JNIUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/JNIUtils.java index c9b4c657a4d7..eff716be3dc9 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/JNIUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/JNIUtils.java @@ -24,6 +24,7 @@ import java.io.InputStream; import java.security.AccessController; import java.security.PrivilegedAction; +import java.util.Locale; import java.util.logging.Logger; /** @@ -37,7 +38,7 @@ public class JNIUtils { private static boolean inited = false; private static String osName() { - String osName = System.getProperty("os.name").toLowerCase().replace(' ', '_'); + String osName = System.getProperty("os.name").toLowerCase(Locale.ROOT).replace(' ', '_'); if (osName.startsWith("win")) { return "win"; } else { diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/OperatingSystem.java b/paimon-common/src/main/java/org/apache/paimon/utils/OperatingSystem.java index 9e923b3faa2a..3188def893f4 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/OperatingSystem.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/OperatingSystem.java @@ -18,6 +18,8 @@ package org.apache.paimon.utils; +import java.util.Locale; + /** An enumeration indicating the operating system that the JVM runs on. */ public enum OperatingSystem { LINUX, @@ -115,7 +117,7 @@ private static OperatingSystem readOSFromSystemProperties() { if (osName.startsWith(FREEBSD_OS_PREFIX)) { return FREE_BSD; } - String osNameLowerCase = osName.toLowerCase(); + String osNameLowerCase = osName.toLowerCase(Locale.ROOT); if (osNameLowerCase.contains(SOLARIS_OS_INFIX_1) || osNameLowerCase.contains(SOLARIS_OS_INFIX_2)) { return SOLARIS; diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index 916f63350c7b..98acf9fd8de3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -62,6 +62,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -408,7 +409,7 @@ public static Table loadTable( private static Table createGlobalSystemTable(String tableName, Catalog catalog) throws Catalog.TableNotExistException { - switch (tableName.toLowerCase()) { + switch (tableName.toLowerCase(Locale.ROOT)) { case ALL_TABLE_OPTIONS: List tables = listAllTables(catalog); Map> allOptions = new HashMap<>(); diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/DistributedLockDialectFactory.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/DistributedLockDialectFactory.java index e0c167a4df05..f324aa41e1f7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/jdbc/DistributedLockDialectFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/DistributedLockDialectFactory.java @@ -18,9 +18,11 @@ package org.apache.paimon.jdbc; +import java.util.Locale; + class DistributedLockDialectFactory { static JdbcDistributedLockDialect create(String protocol) { - JdbcProtocol type = JdbcProtocol.valueOf(protocol.toUpperCase()); + JdbcProtocol type = JdbcProtocol.valueOf(protocol.toUpperCase(Locale.ROOT)); switch (type) { case SQLITE: return new SqlLiteDistributedLockDialect(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java index ba8f8bf48cff..d30d627fb459 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java @@ -35,6 +35,7 @@ import org.apache.paimon.types.RowType; import java.io.IOException; +import java.util.Locale; import java.util.Map; /** @@ -106,7 +107,7 @@ private static String describeUnsupportedMode( } return "from-timestamp"; default: - return effectiveMode.name().toLowerCase().replace('_', '-'); + return effectiveMode.name().toLowerCase(Locale.ROOT).replace('_', '-'); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java index acb487e2573b..7f3ee12d6ef2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -104,7 +105,7 @@ enum Format { /** Parses a file format string to a corresponding {@link Format} enum constant. */ static Format parseFormat(String fileFormat) { try { - return Format.valueOf(fileFormat.toUpperCase()); + return Format.valueOf(fileFormat.toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { throw new UnsupportedOperationException( "Format table unsupported file format: " diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java index fdb57385abf3..876f49d8226a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java @@ -40,6 +40,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -311,7 +312,7 @@ private String option(String key) { if (value == null) { value = table.options().get(key); } - return value == null ? null : value.toLowerCase().trim(); + return value == null ? null : value.toLowerCase(Locale.ROOT).trim(); } private static boolean containsField(GlobalIndexMeta meta, int fieldId) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java index 776730946850..c5ed1e2ecdcc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -103,7 +104,7 @@ public class SystemTableLoader { @Nullable public static Table load(String type, FileStoreTable dataTable) { - String name = type.toLowerCase(); + String name = type.toLowerCase(Locale.ROOT); if (PHYSICAL_METADATA_TABLES.contains(name) && dataTable.coreOptions().queryAuthEnabled()) { throw new UnsupportedOperationException( String.format( diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java index 9e979a98a9da..aec2b21aaed2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java @@ -42,6 +42,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; @@ -302,7 +303,7 @@ private static void escapeChar(char c, StringBuilder sb) { if (c < 16) { sb.append('0'); } - sb.append(Integer.toHexString(c).toUpperCase()); + sb.append(Integer.toHexString(c).toUpperCase(Locale.ROOT)); } public static String unescapePathName(String path) { From aee1f5a66217628fa08384c744324119a676fb7c Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 14:09:32 +0800 Subject: [PATCH 3/8] fix: keep prefix slicing length-safe, drop a test case that pins nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getIdentifierPrefixOptions lowercased the key to test the prefix and then sliced the original by the prefix length, which assumes lowercasing preserves length. It does not: ROOT maps 'İ' to 'i' plus a combining dot. Match case-insensitively on the original key instead. RowKind.fromShortString("-d") passes whichever conversion the code uses, since Turkish differs from ROOT on 'i' and 'I' alone. Only the "+i" case can catch the bug, so keep that one and say why. Co-Authored-By: Claude Code --- .../test/java/org/apache/paimon/TurkishLocaleParsingTest.java | 3 ++- .../src/main/java/org/apache/paimon/format/FileFormat.java | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java b/paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java index 483366806e7d..a397c25b7ee6 100644 --- a/paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java @@ -66,7 +66,8 @@ void partitionMarkDoneActionsParse() { @Test void rowKindFromLowerCaseShortString() { + // "+i" is the only short string this can catch: Turkish differs from ROOT on 'i' and + // 'I' alone, so "-d" or "-u" would pass whichever conversion the code uses assertThat(RowKind.fromShortString("+i")).isEqualTo(RowKind.INSERT); - assertThat(RowKind.fromShortString("-d")).isEqualTo(RowKind.DELETE); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java index 1c45e671cf01..28c2712c03e0 100644 --- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java +++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java @@ -97,7 +97,9 @@ protected Options getIdentifierPrefixOptions(Options options) { Map result = new HashMap<>(); String prefix = formatIdentifier.toLowerCase(Locale.ROOT) + "."; for (String key : options.keySet()) { - if (key.toLowerCase(Locale.ROOT).startsWith(prefix)) { + // match case-insensitively on the original key: lowercasing can change a string's + // length (ROOT maps 'İ' to two characters), and the suffix is sliced by prefix length + if (key.regionMatches(true, 0, prefix, 0, prefix.length())) { result.put(prefix + key.substring(prefix.length()), options.get(key)); } } From 788491bb209f43028d17e4c41624c65aa438b6e3 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 16:52:03 +0800 Subject: [PATCH 4/8] fix: finish the locale sweep and keep prefix slicing length-safe Pins Locale.ROOT on the remaining paimon-flink-cdc conversions, and matches the format prefix against the identifier as written so the option key is sliced at an offset it has. --- .../org/apache/paimon/format/FileFormat.java | 9 +- .../format/FileFormatPrefixOptionsTest.java | 101 ++++++++++++++++++ .../paimon/flink/action/cdc/Expression.java | 7 +- .../action/cdc/SyncDatabaseActionBase.java | 11 +- .../flink/action/cdc/TableNameConverter.java | 15 +-- .../paimon/flink/action/cdc/TypeMapping.java | 3 +- .../action/cdc/format/DataFormatFactory.java | 4 +- .../cdc/mongodb/MongoDBActionUtils.java | 3 +- .../cdc/mongodb/MongoDBSchemaUtils.java | 4 +- .../strategy/MongoVersionStrategy.java | 4 +- .../action/cdc/mysql/MySqlTypeUtils.java | 11 +- .../cdc/postgres/PostgresTypeUtils.java | 4 +- .../cdc/mysql/TurkishLocaleTypeNameTest.java | 60 +++++++++++ 13 files changed, 209 insertions(+), 27 deletions(-) create mode 100644 paimon-common/src/test/java/org/apache/paimon/format/FileFormatPrefixOptionsTest.java create mode 100644 paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/TurkishLocaleTypeNameTest.java diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java index 28c2712c03e0..1e6b56c3f9d2 100644 --- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java +++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java @@ -95,12 +95,13 @@ public static FileFormat fromIdentifier(String identifier, FormatContext context protected Options getIdentifierPrefixOptions(Options options) { Map result = new HashMap<>(); - String prefix = formatIdentifier.toLowerCase(Locale.ROOT) + "."; + // match against the identifier as written so the suffix is sliced at an offset the key + // actually has: lower-casing can lengthen a string, and U+0130 lower-cases to two chars + String prefix = formatIdentifier + "."; + String lowerCasePrefix = formatIdentifier.toLowerCase(Locale.ROOT) + "."; for (String key : options.keySet()) { - // match case-insensitively on the original key: lowercasing can change a string's - // length (ROOT maps 'İ' to two characters), and the suffix is sliced by prefix length if (key.regionMatches(true, 0, prefix, 0, prefix.length())) { - result.put(prefix + key.substring(prefix.length()), options.get(key)); + result.put(lowerCasePrefix + key.substring(prefix.length()), options.get(key)); } } return new Options(result); diff --git a/paimon-common/src/test/java/org/apache/paimon/format/FileFormatPrefixOptionsTest.java b/paimon-common/src/test/java/org/apache/paimon/format/FileFormatPrefixOptionsTest.java new file mode 100644 index 000000000000..23b6deda7892 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/format/FileFormatPrefixOptionsTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format; + +import org.apache.paimon.options.Options; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.types.RowType; + +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +/** Tests for the format-prefixed option filtering in {@link FileFormat}. */ +class FileFormatPrefixOptionsTest { + + /** U+0130, whose lower case under {@link Locale#ROOT} is the two characters below. */ + private static final String DOTTED_CAPITAL_I = "İ"; + + private static final String LOWER_CASE_DOTTED_I = "i̇"; + + @Test + void prefixIsMatchedCaseInsensitively() { + Options options = new Options(); + options.set("ORC.compression", "zstd"); + options.set("avro.codec", "snappy"); + + Options prefixed = new TestFileFormat("orc").getIdentifierPrefixOptions(options); + + assertThat(prefixed.toMap()).containsExactly(entry("orc.compression", "zstd")); + } + + @Test + void anIdentifierThatLowerCasesLongerKeepsItsOptions() { + assertThat(DOTTED_CAPITAL_I.toLowerCase(Locale.ROOT)).isEqualTo(LOWER_CASE_DOTTED_I); + + Options options = new Options(); + options.set(DOTTED_CAPITAL_I + ".compression", "zstd"); + + Options prefixed = new TestFileFormat(DOTTED_CAPITAL_I).getIdentifierPrefixOptions(options); + + // slicing the key at the length of the lower-cased prefix would drop the "c" + assertThat(prefixed.toMap()) + .containsExactly(entry(LOWER_CASE_DOTTED_I + ".compression", "zstd")); + } + + @Test + void aKeyThatIsOnlyThePrefixYieldsAnEmptySuffix() { + Options options = new Options(); + options.set(DOTTED_CAPITAL_I + ".", "zstd"); + + Options prefixed = new TestFileFormat(DOTTED_CAPITAL_I).getIdentifierPrefixOptions(options); + + // this key is shorter than the lower-cased prefix, so slicing at its length overruns it + assertThat(prefixed.toMap()).containsExactly(entry(LOWER_CASE_DOTTED_I + ".", "zstd")); + } + + private static class TestFileFormat extends FileFormat { + + private TestFileFormat(String formatIdentifier) { + super(formatIdentifier); + } + + @Override + public FormatReaderFactory createReaderFactory( + RowType dataSchemaRowType, + RowType projectedRowType, + @Nullable List filters) { + throw new UnsupportedOperationException(); + } + + @Override + public FormatWriterFactory createWriterFactory(RowType type) { + throw new UnsupportedOperationException(); + } + + @Override + public void validateDataFields(RowType rowType) {} + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/Expression.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/Expression.java index 087fe15e67a2..9c82b0faa46f 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/Expression.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/Expression.java @@ -37,6 +37,7 @@ import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -191,11 +192,11 @@ public ExpressionCreator getCreator() { Arrays.stream(ExpressionFunction.values()) .collect( Collectors.toMap( - value -> value.name().toLowerCase(), + value -> value.name().toLowerCase(Locale.ROOT), ExpressionFunction::getCreator)); public static ExpressionCreator creator(String exprName) { - return EXPRESSION_FUNCTIONS.get(exprName.toLowerCase()); + return EXPRESSION_FUNCTIONS.get(exprName.toLowerCase(Locale.ROOT)); } } @@ -253,7 +254,7 @@ static Expression create( String exprName, String... args) { - ExpressionCreator function = ExpressionFunction.creator(exprName.toLowerCase()); + ExpressionCreator function = ExpressionFunction.creator(exprName.toLowerCase(Locale.ROOT)); if (function == null) { throw new UnsupportedOperationException( String.format( diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java index d7393485ad73..d0ca252fbee0 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.regex.Pattern; @@ -113,7 +114,10 @@ public SyncDatabaseActionBase withDbPrefix(Map dbPrefix) { dbPrefix.entrySet().stream() .collect( HashMap::new, - (m, e) -> m.put(e.getKey().toLowerCase(), e.getValue()), + (m, e) -> + m.put( + e.getKey().toLowerCase(Locale.ROOT), + e.getValue()), HashMap::putAll); } return this; @@ -125,7 +129,10 @@ public SyncDatabaseActionBase withDbSuffix(Map dbSuffix) { dbSuffix.entrySet().stream() .collect( HashMap::new, - (m, e) -> m.put(e.getKey().toLowerCase(), e.getValue()), + (m, e) -> + m.put( + e.getKey().toLowerCase(Locale.ROOT), + e.getValue()), HashMap::putAll); } return this; diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java index 7dd63ed2273e..44cd51be83e9 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java @@ -22,6 +22,7 @@ import java.io.Serializable; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import static org.apache.paimon.utils.StringUtils.toLowerCaseIfNeed; @@ -78,8 +79,8 @@ public TableNameConverter( public String convert(String originDbName, String originTblName) { // top priority: table mapping - if (tableMapping.containsKey(originTblName.toLowerCase())) { - String mappedName = tableMapping.get(originTblName.toLowerCase()); + if (tableMapping.containsKey(originTblName.toLowerCase(Locale.ROOT))) { + String mappedName = tableMapping.get(originTblName.toLowerCase(Locale.ROOT)); return toLowerCaseIfNeed(mappedName, caseSensitive); } @@ -87,11 +88,11 @@ public String convert(String originDbName, String originTblName) { String tblSuffix = suffix; // second priority: prefix and postfix specified by db - if (dbPrefix.containsKey(originDbName.toLowerCase())) { - tblPrefix = dbPrefix.get(originDbName.toLowerCase()); + if (dbPrefix.containsKey(originDbName.toLowerCase(Locale.ROOT))) { + tblPrefix = dbPrefix.get(originDbName.toLowerCase(Locale.ROOT)); } - if (dbSuffix.containsKey(originDbName.toLowerCase())) { - tblSuffix = dbSuffix.get(originDbName.toLowerCase()); + if (dbSuffix.containsKey(originDbName.toLowerCase(Locale.ROOT))) { + tblSuffix = dbSuffix.get(originDbName.toLowerCase(Locale.ROOT)); } // third priority: normal prefix and suffix @@ -117,7 +118,7 @@ private Map lowerMapKey(Map map) { } for (String key : map.keySet()) { - lowerKeyMap.put(key.toLowerCase(), map.get(key)); + lowerKeyMap.put(key.toLowerCase(Locale.ROOT), map.get(key)); } return lowerKeyMap; diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java index 499c3823b9e1..429f3f3d2fa9 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java @@ -21,6 +21,7 @@ import java.io.Serializable; import java.util.Arrays; import java.util.Collections; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -96,7 +97,7 @@ public static TypeMappingMode mode(String option) { } public String configString() { - return name().toLowerCase().replace("_", "-"); + return name().toLowerCase(Locale.ROOT).replace("_", "-"); } } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/DataFormatFactory.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/DataFormatFactory.java index df1e92d90792..0893c0012435 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/DataFormatFactory.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/DataFormatFactory.java @@ -22,13 +22,15 @@ import org.apache.paimon.factories.FactoryException; import org.apache.paimon.factories.FactoryUtil; +import java.util.Locale; + /** Factory to create {@link DataFormat}. */ public interface DataFormatFactory extends Factory { DataFormat create(); static DataFormat createDataFormat(String format) { - String identifier = format.toLowerCase(); + String identifier = format.toLowerCase(Locale.ROOT); DataFormatFactory dataFormatFactory; try { dataFormatFactory = diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBActionUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBActionUtils.java index ddc69a180c71..0fa299811f93 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBActionUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBActionUtils.java @@ -33,6 +33,7 @@ import org.apache.kafka.connect.json.JsonConverterConfig; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -118,7 +119,7 @@ public static MongoDBSource buildMongodbSource( .collectionList(tableList); String startupMode = mongodbConfig.get(SourceOptions.SCAN_STARTUP_MODE); - switch (startupMode.toLowerCase()) { + switch (startupMode.toLowerCase(Locale.ROOT)) { case INITIAL_MODE: sourceBuilder.startupOptions(StartupOptions.initial()); break; diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSchemaUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSchemaUtils.java index 9dbb6eef7df5..0c3571889887 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSchemaUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSchemaUtils.java @@ -38,6 +38,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Objects; import static org.apache.flink.cdc.connectors.mongodb.internal.MongoDBEnvelope.encodeValue; @@ -153,7 +154,8 @@ private static String buildConnectionString( } private static SchemaAcquisitionMode getModeFromConfig(Configuration mongodbConfig) { - return SchemaAcquisitionMode.valueOf(mongodbConfig.get(START_MODE).toUpperCase()); + return SchemaAcquisitionMode.valueOf( + mongodbConfig.get(START_MODE).toUpperCase(Locale.ROOT)); } private static List getColumnNames(Document document) { diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java index 0a72f711604e..f0ad32eac27b 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -86,7 +87,8 @@ default Map getExtractRow( return null; } SchemaAcquisitionMode mode = - SchemaAcquisitionMode.valueOf(mongodbConfig.get(START_MODE).toUpperCase()); + SchemaAcquisitionMode.valueOf( + mongodbConfig.get(START_MODE).toUpperCase(Locale.ROOT)); ObjectNode objectNode = JsonSerdeUtil.asSpecificNodeType(jsonNode.asText(), ObjectNode.class); JsonNode idNode = objectNode.get(ID_FIELD); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java index 989236d8c854..efb2367a8e30 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java @@ -37,6 +37,7 @@ import java.nio.ByteBuffer; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -153,8 +154,8 @@ public static Tuple3 getTypeInfo(String typeName) { if (leftBracketIndex != -1) { int rightBracketIndex = typeName.indexOf(RIGHT_BRACKETS); shortType = - typeName.substring(0, leftBracketIndex).trim().toUpperCase() - + typeName.substring(rightBracketIndex + 1).toUpperCase(); + typeName.substring(0, leftBracketIndex).trim().toUpperCase(Locale.ROOT) + + typeName.substring(rightBracketIndex + 1).toUpperCase(Locale.ROOT); String insideBrackets = typeName.substring(leftBracketIndex + 1, rightBracketIndex).trim(); @@ -167,7 +168,7 @@ public static Tuple3 getTypeInfo(String typeName) { length = Integer.parseInt(insideBrackets); } } else { - shortType = typeName.toUpperCase(); + shortType = typeName.toUpperCase(Locale.ROOT); if (isDecimalType(shortType)) { // when missing precision and scale of the decimal, we // use the max precision and scale to avoid parse error @@ -187,7 +188,7 @@ public static DataType toDataType( return DataTypes.STRING(); } - switch (type.toUpperCase()) { + switch (type.toUpperCase(Locale.ROOT)) { case BIT: if (length == null || length == 1) { return DataTypes.BOOLEAN(); @@ -335,7 +336,7 @@ public static DataType toDataType( } public static boolean isGeoType(String type) { - switch (type.toUpperCase()) { + switch (type.toUpperCase(Locale.ROOT)) { case GEOMETRY: case POINT: case LINESTRING: diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresTypeUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresTypeUtils.java index 7fae4c4327be..9ec33e498a94 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresTypeUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresTypeUtils.java @@ -26,6 +26,8 @@ import javax.annotation.Nullable; +import java.util.Locale; + import static org.apache.paimon.flink.action.cdc.TypeMapping.TypeMappingMode.TO_STRING; /** Converts from Postgres type to {@link DataType}. */ @@ -83,7 +85,7 @@ public static DataType toDataType( } precision = precision == null ? 0 : precision; scale = scale == null ? 0 : scale; - switch (typeName.toLowerCase()) { + switch (typeName.toLowerCase(Locale.ROOT)) { case PG_BIT: case PG_VARBIT: if (precision <= 1) { diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/TurkishLocaleTypeNameTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/TurkishLocaleTypeNameTest.java new file mode 100644 index 000000000000..911d27467a16 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/TurkishLocaleTypeNameTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.mysql; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * A source type name is case-folded before it is matched against the names this class knows. Under + * a Turkish default locale 'i' maps to a dotted capital, so those conversions must pin {@link + * Locale#ROOT} or a type whose name contains an 'i' stops being recognized. + */ +class TurkishLocaleTypeNameTest { + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void lowerCaseTypeNameKeepsItsShortType() { + // a locale-sensitive uppercase turns "int" into "İNT", which no case arm names + assertThat(MySqlTypeUtils.getTypeInfo("int").f0).isEqualTo("INT"); + } + + @Test + void lowerCaseGeoTypeIsStillRecognized() { + assertThat(MySqlTypeUtils.isGeoType("point")).isTrue(); + assertThat(MySqlTypeUtils.isGeoType("linestring")).isTrue(); + } +} From bb81158fe121347cf961c06dd7a00c19971b2e1c Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 17:25:56 +0800 Subject: [PATCH 5/8] fix: pin Locale.ROOT on every remaining case conversion in main sources Covers paimon-filesystems, paimon-hive, paimon-flink-common, paimon-spark, paimon-lumina, the vendored OrcFile copy, and the docs, benchmark and CI tooling. Also makes the Hive clone copy of getIdentifierPrefixOptions length-safe, like the FileFormat original. --- .../benchmark/metric/cpu/OperatingSystem.java | 4 +- .../ConfigOptionsDocGenerator.java | 3 +- .../org/apache/paimon/cosn/COSNFileIO.java | 11 +-- .../org/apache/paimon/jindo/JindoFileIO.java | 11 +-- .../java/org/apache/paimon/obs/OBSFileIO.java | 11 +-- .../java/org/apache/paimon/oss/OSSFileIO.java | 11 +-- .../paimon/flink/FormatCatalogTable.java | 3 +- .../paimon/flink/action/ActionFactory.java | 3 +- .../paimon/flink/action/CloneAction.java | 3 +- .../flink/action/MultiTablesSinkMode.java | 5 +- .../flink/clone/CloneFileFormatUtils.java | 4 +- .../paimon/flink/lookup/PartitionLoader.java | 3 +- .../procedure/CreateGlobalIndexProcedure.java | 3 +- .../procedure/DropGlobalIndexProcedure.java | 3 +- .../source/aggregate/LocalAggregator.java | 3 +- .../flink/action/MultiTablesSinkModeTest.java | 63 +++++++++++++++++ .../src/main/java/org/apache/orc/OrcFile.java | 5 +- .../org/apache/paimon/hive/HiveCatalog.java | 7 +- .../apache/paimon/hive/HiveTableUtils.java | 5 +- .../paimon/hive/clone/HiveCloneUtils.java | 3 +- .../hive/clone/HiveTableCloneExtractor.java | 16 +++-- .../paimon/hive/pool/CachedClientPool.java | 2 +- .../clone/HiveTableCloneExtractorTest.java | 69 +++++++++++++++++++ .../hive/pool/TestCachedClientPool.java | 18 +++++ .../org/apache/paimon/hive/HiveSchema.java | 5 +- .../apache/paimon/hive/PaimonMetaHook.java | 9 +-- .../hive/mapred/PaimonRecordReader.java | 3 +- .../lumina/index/LuminaVectorMetric.java | 4 +- .../lumina/index/LuminaVectorMetricTest.java | 56 +++++++++++++++ .../org/apache/paimon/spark/SparkCatalog.java | 2 +- .../spark/catalog/FormatTableCatalog.java | 4 +- .../tools/ci/licensecheck/JarFileChecker.java | 2 +- 32 files changed, 299 insertions(+), 55 deletions(-) create mode 100644 paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MultiTablesSinkModeTest.java create mode 100644 paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/clone/HiveTableCloneExtractorTest.java create mode 100644 paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorMetricTest.java diff --git a/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/OperatingSystem.java b/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/OperatingSystem.java index 0e15ca154cf6..0fd6cd92a827 100644 --- a/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/OperatingSystem.java +++ b/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/OperatingSystem.java @@ -18,6 +18,8 @@ package org.apache.paimon.benchmark.metric.cpu; +import java.util.Locale; + /** An enumeration indicating the operating system that the JVM runs on. */ public enum OperatingSystem { LINUX, @@ -115,7 +117,7 @@ private static OperatingSystem readOSFromSystemProperties() { if (osName.startsWith(FREEBSD_OS_PREFIX)) { return FREE_BSD; } - String osNameLowerCase = osName.toLowerCase(); + String osNameLowerCase = osName.toLowerCase(Locale.ROOT); if (osNameLowerCase.contains(SOLARIS_OS_INFIX_1) || osNameLowerCase.contains(SOLARIS_OS_INFIX_2)) { return SOLARIS; diff --git a/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java b/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java index ebb167cbacc4..28e38b050752 100644 --- a/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java +++ b/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java @@ -55,6 +55,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.regex.Matcher; @@ -263,7 +264,7 @@ private static void createTable( @VisibleForTesting static String toSnakeCase(String name) { - return name.replaceAll("(.)([A-Z][a-z])", "$1_$2").toLowerCase(); + return name.replaceAll("(.)([A-Z][a-z])", "$1_$2").toLowerCase(Locale.ROOT); } @VisibleForTesting diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java index 0f0096751d9e..ddec6d38e008 100644 --- a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java +++ b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java @@ -33,6 +33,7 @@ import java.io.UncheckedIOException; import java.net.URI; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -52,8 +53,10 @@ public class COSNFileIO extends HadoopCompliantFileIO { private static final Map CASE_SENSITIVE_KEYS = new HashMap() { { - put(COSN_USER_INFO_KEY_ID.toLowerCase(), COSN_USER_INFO_KEY_ID); - put(COSN_USER_INFO_KEY_SECRET.toLowerCase(), COSN_USER_INFO_KEY_SECRET); + put(COSN_USER_INFO_KEY_ID.toLowerCase(Locale.ROOT), COSN_USER_INFO_KEY_ID); + put( + COSN_USER_INFO_KEY_SECRET.toLowerCase(Locale.ROOT), + COSN_USER_INFO_KEY_SECRET); } }; /** @@ -77,8 +80,8 @@ public void configure(CatalogContext context) { for (String prefix : CONFIG_PREFIXES) { if (key.startsWith(prefix)) { String value = context.options().get(key); - if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase())) { - key = CASE_SENSITIVE_KEYS.get(key.toLowerCase()); + if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase(Locale.ROOT))) { + key = CASE_SENSITIVE_KEYS.get(key.toLowerCase(Locale.ROOT)); } hadoopOptions.set(key, value); LOG.debug( diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 1e4646d4b9f9..271672ab7e04 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -48,6 +48,7 @@ import java.net.URI; import java.time.Duration; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -83,9 +84,9 @@ public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsP private static final Map CASE_SENSITIVE_KEYS = new HashMap() { { - put(OSS_ACCESS_KEY_ID.toLowerCase(), OSS_ACCESS_KEY_ID); - put(OSS_ACCESS_KEY_SECRET.toLowerCase(), OSS_ACCESS_KEY_SECRET); - put(OSS_SECURITY_TOKEN.toLowerCase(), OSS_SECURITY_TOKEN); + put(OSS_ACCESS_KEY_ID.toLowerCase(Locale.ROOT), OSS_ACCESS_KEY_ID); + put(OSS_ACCESS_KEY_SECRET.toLowerCase(Locale.ROOT), OSS_ACCESS_KEY_SECRET); + put(OSS_SECURITY_TOKEN.toLowerCase(Locale.ROOT), OSS_SECURITY_TOKEN); } }; @@ -122,8 +123,8 @@ public void configure(CatalogContext context) { for (String prefix : CONFIG_PREFIXES) { if (key.startsWith(prefix)) { String value = context.options().get(key); - if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase())) { - key = CASE_SENSITIVE_KEYS.get(key.toLowerCase()); + if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase(Locale.ROOT))) { + key = CASE_SENSITIVE_KEYS.get(key.toLowerCase(Locale.ROOT)); } hadoopOptions.set(key, value); diff --git a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/OBSFileIO.java b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/OBSFileIO.java index 4f425ca59065..408bcc0a7b89 100644 --- a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/OBSFileIO.java +++ b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/OBSFileIO.java @@ -33,6 +33,7 @@ import java.io.UncheckedIOException; import java.net.URI; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -60,9 +61,9 @@ public class OBSFileIO extends HadoopCompliantFileIO { private static final Map CASE_SENSITIVE_KEYS = new HashMap() { { - put(OBS_ACCESS_KEY_ID.toLowerCase(), OBS_ACCESS_KEY_ID); - put(OBS_ACCESS_KEY_SECRET.toLowerCase(), OBS_ACCESS_KEY_SECRET); - put(OBS_SESSION_TOKEN.toLowerCase(), OBS_SESSION_TOKEN); + put(OBS_ACCESS_KEY_ID.toLowerCase(Locale.ROOT), OBS_ACCESS_KEY_ID); + put(OBS_ACCESS_KEY_SECRET.toLowerCase(Locale.ROOT), OBS_ACCESS_KEY_SECRET); + put(OBS_SESSION_TOKEN.toLowerCase(Locale.ROOT), OBS_SESSION_TOKEN); } }; @@ -92,8 +93,8 @@ public void configure(CatalogContext context) { for (String prefix : CONFIG_PREFIXES) { if (key.startsWith(prefix)) { String value = context.options().get(key); - if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase())) { - key = CASE_SENSITIVE_KEYS.get(key.toLowerCase()); + if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase(Locale.ROOT))) { + key = CASE_SENSITIVE_KEYS.get(key.toLowerCase(Locale.ROOT)); } hadoopOptions.set(key, value); diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java index 5a3b48a5c54b..3249cbfcef77 100644 --- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java @@ -58,6 +58,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -115,9 +116,9 @@ public class OSSFileIO extends HadoopCompliantFileIO implements HadoopOptionsPro private static final Map CASE_SENSITIVE_KEYS = new HashMap() { { - put(OSS_ACCESS_KEY_ID.toLowerCase(), OSS_ACCESS_KEY_ID); - put(OSS_ACCESS_KEY_SECRET.toLowerCase(), OSS_ACCESS_KEY_SECRET); - put(OSS_SECURITY_TOKEN.toLowerCase(), OSS_SECURITY_TOKEN); + put(OSS_ACCESS_KEY_ID.toLowerCase(Locale.ROOT), OSS_ACCESS_KEY_ID); + put(OSS_ACCESS_KEY_SECRET.toLowerCase(Locale.ROOT), OSS_ACCESS_KEY_SECRET); + put(OSS_SECURITY_TOKEN.toLowerCase(Locale.ROOT), OSS_SECURITY_TOKEN); } }; @@ -147,8 +148,8 @@ public void configure(CatalogContext context) { for (String prefix : CONFIG_PREFIXES) { if (key.startsWith(prefix)) { String value = context.options().get(key); - if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase())) { - key = CASE_SENSITIVE_KEYS.get(key.toLowerCase()); + if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase(Locale.ROOT))) { + key = CASE_SENSITIVE_KEYS.get(key.toLowerCase(Locale.ROOT)); } hadoopOptions.set(key, value); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java index 5dbce5f0a42b..fa0322090d4b 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -88,7 +89,7 @@ public CatalogTable copy(Map map) { public Map getOptions() { if (cachedOptions == null) { cachedOptions = new HashMap<>(); - String format = table.format().name().toLowerCase(); + String format = table.format().name().toLowerCase(Locale.ROOT); Map options = table.options(); options.forEach( (k, v) -> { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java index 4e71b14d25f8..1101bdc5b765 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java @@ -31,6 +31,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -71,7 +72,7 @@ public interface ActionFactory extends Factory { static Optional createAction(String[] args) { // to be compatible with old usage - String action = args[0].toLowerCase().replaceAll("-", "_"); + String action = args[0].toLowerCase(Locale.ROOT).replaceAll("-", "_"); String[] actionArgs = Arrays.copyOfRange(args, 1, args.length); ActionFactory actionFactory; try { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java index 4ee87710e6fe..8b3441943541 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java @@ -29,6 +29,7 @@ import javax.annotation.Nullable; import java.util.List; +import java.util.Locale; import java.util.Map; /** Clone source table to target table. */ @@ -104,7 +105,7 @@ public CloneAction( this.preferFileFormat = StringUtils.isNullOrWhitespaceOnly(preferFileFormat) ? preferFileFormat - : preferFileFormat.toLowerCase(); + : preferFileFormat.toLowerCase(Locale.ROOT); this.cloneFrom = cloneFrom; this.metaOnly = metaOnly; this.cloneIfExists = cloneIfExists; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/MultiTablesSinkMode.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/MultiTablesSinkMode.java index d5c514199589..d5c0d06be2f1 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/MultiTablesSinkMode.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/MultiTablesSinkMode.java @@ -21,6 +21,7 @@ import javax.annotation.Nullable; import java.io.Serializable; +import java.util.Locale; /** * There are two modes for database sink. @@ -40,7 +41,7 @@ public static MultiTablesSinkMode fromString(@Nullable String mode) { return DIVIDED; } - switch (mode.toLowerCase()) { + switch (mode.toLowerCase(Locale.ROOT)) { case "divided": return DIVIDED; case "combined": @@ -51,6 +52,6 @@ public static MultiTablesSinkMode fromString(@Nullable String mode) { } public String configString() { - return name().toLowerCase().replace("_", "-"); + return name().toLowerCase(Locale.ROOT).replace("_", "-"); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneFileFormatUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneFileFormatUtils.java index b1c73fce7d9d..88fbd11fba24 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneFileFormatUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneFileFormatUtils.java @@ -21,6 +21,8 @@ import org.apache.paimon.flink.action.CloneAction; import org.apache.paimon.utils.StringUtils; +import java.util.Locale; + /** Utils for file format in {@link CloneAction}. */ public class CloneFileFormatUtils { @@ -28,7 +30,7 @@ public static void validateFileFormat(String fileFormat) { if (StringUtils.isNullOrWhitespaceOnly(fileFormat)) { return; } - String fileFormatLower = fileFormat.toLowerCase(); + String fileFormatLower = fileFormat.toLowerCase(Locale.ROOT); String[] supportedFileFormat = new String[] {"parquet", "orc", "avro"}; for (String supportedFormat : supportedFileFormat) { if (fileFormatLower.equals(supportedFormat)) { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionLoader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionLoader.java index 49d236a69302..688321e31fd6 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionLoader.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionLoader.java @@ -34,6 +34,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Locale; /** Specify partitions for lookup tables. */ public abstract class PartitionLoader implements Serializable { @@ -95,7 +96,7 @@ public static PartitionLoader of(Table table) { FlinkConnectorOptions.SCAN_PARTITIONS.key()); int maxPartitionNum = -1; - switch (scanPartitions.toLowerCase()) { + switch (scanPartitions.toLowerCase(Locale.ROOT)) { case MAX_PT: maxPartitionNum = 1; break; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedure.java index 8fb6fac49eca..f93d19a0dcf8 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedure.java @@ -37,6 +37,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; @@ -114,7 +115,7 @@ public String[] call( // Parse options Options userOptions = createUserOptions(table, options); - indexType = indexType.toLowerCase().trim(); + indexType = indexType.toLowerCase(Locale.ROOT).trim(); if (indexColumns.size() > 1) { // Fail fast before submitting the job: index types that do not support multi-column // throw from GlobalIndexerFactory#create, which happens before any indexer side effect. diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DropGlobalIndexProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DropGlobalIndexProcedure.java index 098c30cbe48a..d74d0d2ad784 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DropGlobalIndexProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DropGlobalIndexProcedure.java @@ -46,6 +46,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; @@ -112,7 +113,7 @@ public String[] call( PartitionPredicate partitionPredicate = parsePartitionPredicate(table, partitions); // Normalize index type - final String indexTypeLower = indexType.toLowerCase().trim(); + final String indexTypeLower = indexType.toLowerCase(Locale.ROOT).trim(); // Get latest snapshot Snapshot snapshot = diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/aggregate/LocalAggregator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/aggregate/LocalAggregator.java index b1a6dc8cf5c2..009407f08516 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/aggregate/LocalAggregator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/aggregate/LocalAggregator.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; /** Local aggregator based on split statistics. */ @@ -229,7 +230,7 @@ private DataType resultType() { } private String resultName() { - return kind.name().toLowerCase(); + return kind.name().toLowerCase(Locale.ROOT); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MultiTablesSinkModeTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MultiTablesSinkModeTest.java new file mode 100644 index 000000000000..42e23207692b --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MultiTablesSinkModeTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The mode name is case-folded before it is matched, so the conversion has to pin {@link + * Locale#ROOT} or a name containing an 'i' stops matching under a Turkish default. + */ +class MultiTablesSinkModeTest { + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void upperCaseModeNameStillParses() { + // DIVIDED contains an 'I', which a locale-sensitive lowercase turns into a dotless one + assertThat(MultiTablesSinkMode.fromString("DIVIDED")) + .isEqualTo(MultiTablesSinkMode.DIVIDED); + assertThat(MultiTablesSinkMode.fromString("divided")) + .isEqualTo(MultiTablesSinkMode.DIVIDED); + } + + @Test + void configStringRoundTrips() { + for (MultiTablesSinkMode mode : MultiTablesSinkMode.values()) { + assertThat(MultiTablesSinkMode.fromString(mode.configString())).isEqualTo(mode); + } + } +} diff --git a/paimon-format/src/main/java/org/apache/orc/OrcFile.java b/paimon-format/src/main/java/org/apache/orc/OrcFile.java index f482174edc1f..06e59cd1c9ae 100644 --- a/paimon-format/src/main/java/org/apache/orc/OrcFile.java +++ b/paimon-format/src/main/java/org/apache/orc/OrcFile.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Properties; @@ -513,7 +514,9 @@ protected WriterOptions(Properties tableProperties, Configuration conf) { blockPaddingValue = OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf); compressValue = CompressionKind.valueOf( - OrcConf.COMPRESS.getString(tableProperties, conf).toUpperCase()); + OrcConf.COMPRESS + .getString(tableProperties, conf) + .toUpperCase(Locale.ROOT)); enforceBufferSize = OrcConf.ENFORCE_COMPRESSION_BUFFER_SIZE.getBoolean(tableProperties, conf); String versionName = OrcConf.WRITE_FORMAT.getString(tableProperties, conf); diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index b3ab44f64eb5..8420010267af 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -94,6 +94,7 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -1170,8 +1171,8 @@ private boolean usingExternalTable(Map tableOptions) { String externalPropValue = tableOptions.getOrDefault( - HIVE_EXTERNAL_TABLE_PROP.toLowerCase(), - tableOptions.get(HIVE_EXTERNAL_TABLE_PROP.toUpperCase())); + HIVE_EXTERNAL_TABLE_PROP.toLowerCase(Locale.ROOT), + tableOptions.get(HIVE_EXTERNAL_TABLE_PROP.toUpperCase(Locale.ROOT))); return CatalogTableType.EXTERNAL.equals(tableType) || "TRUE".equalsIgnoreCase(externalPropValue); } @@ -1705,7 +1706,7 @@ private Table newHmsTable( } else { // format table table.getParameters().put(TABLE_TYPE_PROP, provider.name()); - table.getParameters().put(FILE_FORMAT.key(), provider.name().toLowerCase()); + table.getParameters().put(FILE_FORMAT.key(), provider.name().toLowerCase(Locale.ROOT)); table.getParameters().put(TYPE.key(), FORMAT_TABLE.toString()); } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java index 56ac5b82f44f..3a8e8cc8b6c8 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import static org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM; import static org.apache.paimon.CoreOptions.FILE_FORMAT; @@ -103,7 +104,7 @@ public static Schema tryToFormatSchema(Table hiveTable) { String serLib = serdeInfo.getSerializationLib() == null ? "" - : serdeInfo.getSerializationLib().toLowerCase(); + : serdeInfo.getSerializationLib().toLowerCase(Locale.ROOT); String inputFormat = sd.getInputFormat() == null ? "" : sd.getInputFormat(); if (serLib.contains("parquet")) { format = Format.PARQUET; @@ -132,7 +133,7 @@ public static Schema tryToFormatSchema(Table hiveTable) { rowType.getFields().forEach(f -> builder.column(f.name(), f.type(), f.description())); options.set(PATH, location); options.set(TYPE, FORMAT_TABLE); - options.set(FILE_FORMAT, format.name().toLowerCase()); + options.set(FILE_FORMAT, format.name().toLowerCase(Locale.ROOT)); return builder.partitionKeys(partitionKeys) .options(options.toMap()) .comment(comment) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveCloneUtils.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveCloneUtils.java index f1d412ace1d6..1c6bb5a349d0 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveCloneUtils.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveCloneUtils.java @@ -43,6 +43,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Predicate; @@ -205,7 +206,7 @@ private static String parseFormat(StorageDescriptor sd) { String serLib = serdeInfo.getSerializationLib() == null ? "" - : serdeInfo.getSerializationLib().toLowerCase(); + : serdeInfo.getSerializationLib().toLowerCase(Locale.ROOT); String inputFormat = sd.getInputFormat() == null ? "" : sd.getInputFormat(); if (serLib.contains("avro")) { return "avro"; diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveTableCloneExtractor.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveTableCloneExtractor.java index 0d3baf29420d..61ddb4096c07 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveTableCloneExtractor.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveTableCloneExtractor.java @@ -45,6 +45,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -226,10 +227,13 @@ private static String parseCompression( public static Map getIdentifierPrefixOptions( String formatIdentifier, Map options) { Map result = new HashMap<>(); - String prefix = formatIdentifier.toLowerCase() + "."; + // match against the identifier as written so the suffix is sliced at an offset the key + // actually has: lower-casing can lengthen a string, and U+0130 lower-cases to two chars + String prefix = formatIdentifier + "."; + String lowerCasePrefix = formatIdentifier.toLowerCase(Locale.ROOT) + "."; for (String key : options.keySet()) { - if (key.toLowerCase().startsWith(prefix)) { - result.put(prefix + key.substring(prefix.length()), options.get(key)); + if (key.regionMatches(true, 0, prefix, 0, prefix.length())) { + result.put(lowerCasePrefix + key.substring(prefix.length()), options.get(key)); } } return result; @@ -238,9 +242,11 @@ public static Map getIdentifierPrefixOptions( public static Map getOptionsWhenCloneSplits(Table table, String format) { Map result = new HashMap<>(); if (FormatTable.Format.JSON.name().equalsIgnoreCase(format)) { - result.put(FILE_FORMAT.key(), FormatTable.Format.PARQUET.name().toLowerCase()); + result.put( + FILE_FORMAT.key(), FormatTable.Format.PARQUET.name().toLowerCase(Locale.ROOT)); } else if (FormatTable.Format.CSV.name().equalsIgnoreCase(format)) { - result.put(FILE_FORMAT.key(), FormatTable.Format.PARQUET.name().toLowerCase()); + result.put( + FILE_FORMAT.key(), FormatTable.Format.PARQUET.name().toLowerCase(Locale.ROOT)); } else { result.put(FILE_FORMAT.key(), format); } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java index 483b21106ba0..3da673dd2508 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java @@ -164,7 +164,7 @@ static Key extractKey(String clientClassName, String cacheKeys, Configuration co key); confElements.put(key, conf.get(key)); } else { - KeyElementType type = KeyElementType.valueOf(trimmed.toUpperCase()); + KeyElementType type = KeyElementType.valueOf(trimmed.toUpperCase(Locale.ROOT)); switch (type) { case UGI: case USER_NAME: diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/clone/HiveTableCloneExtractorTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/clone/HiveTableCloneExtractorTest.java new file mode 100644 index 000000000000..dcceed0a1546 --- /dev/null +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/clone/HiveTableCloneExtractorTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hive.clone; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +/** Tests for the format-prefixed option filtering in {@link HiveTableCloneExtractor}. */ +class HiveTableCloneExtractorTest { + + /** U+0130, whose lower case under {@link Locale#ROOT} is the two characters below. */ + private static final String DOTTED_CAPITAL_I = "İ"; + + private static final String LOWER_CASE_DOTTED_I = "i̇"; + + @Test + void prefixIsMatchedCaseInsensitively() { + Map options = new HashMap<>(); + options.put("ORC.compression", "zstd"); + options.put("avro.codec", "snappy"); + + assertThat(HiveTableCloneExtractor.getIdentifierPrefixOptions("orc", options)) + .containsExactly(entry("orc.compression", "zstd")); + } + + @Test + void anIdentifierThatLowerCasesLongerKeepsItsOptions() { + assertThat(DOTTED_CAPITAL_I.toLowerCase(Locale.ROOT)).isEqualTo(LOWER_CASE_DOTTED_I); + + Map options = new HashMap<>(); + options.put(DOTTED_CAPITAL_I + ".compression", "zstd"); + + // slicing the key at the length of the lower-cased prefix would drop the "c" + assertThat(HiveTableCloneExtractor.getIdentifierPrefixOptions(DOTTED_CAPITAL_I, options)) + .containsExactly(entry(LOWER_CASE_DOTTED_I + ".compression", "zstd")); + } + + @Test + void aKeyThatIsOnlyThePrefixYieldsAnEmptySuffix() { + Map options = new HashMap<>(); + options.put(DOTTED_CAPITAL_I + ".", "zstd"); + + // this key is shorter than the lower-cased prefix, so slicing at its length overruns it + assertThat(HiveTableCloneExtractor.getIdentifierPrefixOptions(DOTTED_CAPITAL_I, options)) + .containsExactly(entry(LOWER_CASE_DOTTED_I + ".", "zstd")); + } +} diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/pool/TestCachedClientPool.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/pool/TestCachedClientPool.java index 0b1456538281..ea551e6e44c2 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/pool/TestCachedClientPool.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/pool/TestCachedClientPool.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.security.PrivilegedAction; import java.util.List; +import java.util.Locale; import java.util.UUID; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORECONNECTURLKEY; @@ -59,6 +60,23 @@ public void testExtractKeyUnknownElementErrorMessage() { .hasMessage("Unknown key element conf"); } + @Test + public void testExtractKeyUnderATurkishDefaultLocale() { + // UGI contains an 'I', so a locale-sensitive uppercase asks the enum for UGİ + Locale original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + try { + assertThat( + CachedClientPool.extractKey( + HiveMetaStoreClient.class.getName(), + "ugi", + new Configuration())) + .isNotNull(); + } finally { + Locale.setDefault(original); + } + } + @Test public void testCacheKeyNotSame() { // client1 use cache key type:paimon diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java index aac33db8700b..7321a58aa343 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -172,10 +173,10 @@ public static HiveSchema extract(@Nullable Configuration configuration, Properti paimonSchema.fields().stream() .collect( Collectors.toMap( - dataField -> dataField.name().toLowerCase(), + dataField -> dataField.name().toLowerCase(Locale.ROOT), Function.identity())); for (int i = 0; i < columnNames.size(); i++) { - String columnName = columnNames.get(i).toLowerCase(); + String columnName = columnNames.get(i).toLowerCase(Locale.ROOT); if (Objects.equals(columnName, tagToPartField)) { // ignore tagToPartField, it should just be a string type continue; diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java index c032c2ff43e8..071195fde6ab 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -78,8 +79,8 @@ public void preCreateTable(Table table) throws MetaException { table.getSd().setInputFormat(PaimonInputFormat.class.getCanonicalName()); table.getSd().setOutputFormat(PaimonOutputFormat.class.getCanonicalName()); - table.setDbName(table.getDbName().toLowerCase()); - table.setTableName(table.getTableName().toLowerCase()); + table.setDbName(table.getDbName().toLowerCase(Locale.ROOT)); + table.setTableName(table.getTableName().toLowerCase(Locale.ROOT)); String location = LocationKeyExtractor.getPaimonLocation(conf, table); Identifier identifier = Identifier.create(table.getDbName(), table.getTableName()); if (location == null) { @@ -115,7 +116,7 @@ public void preCreateTable(Table table) throws MetaException { .forEachRemaining( fieldSchema -> schemaBuilder.column( - fieldSchema.getName().toLowerCase(), + fieldSchema.getName().toLowerCase(Locale.ROOT), toPaimonType(fieldSchema.getType()), fieldSchema.getComment())); // partition columns @@ -128,7 +129,7 @@ public void preCreateTable(Table table) throws MetaException { .forEachRemaining( fieldSchema -> schemaBuilder.column( - fieldSchema.getName().toLowerCase(), + fieldSchema.getName().toLowerCase(Locale.ROOT), toPaimonType(fieldSchema.getType()), fieldSchema.getComment())); diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java index 3232868b5327..1cd93a9f248e 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -88,7 +89,7 @@ public PaimonRecordReader( .boxed() .collect( Collectors.toMap( - index -> paimonColumns.get(index).toLowerCase(), + index -> paimonColumns.get(index).toLowerCase(Locale.ROOT), index -> index, (existing, replacement) -> existing, LinkedHashMap::new)); diff --git a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorMetric.java b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorMetric.java index f4240d701381..212d8a890b7b 100644 --- a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorMetric.java +++ b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorMetric.java @@ -18,6 +18,8 @@ package org.apache.paimon.lumina.index; +import java.util.Locale; + /** Enumeration of supported Lumina vector similarity metrics. */ public enum LuminaVectorMetric { @@ -45,7 +47,7 @@ public String getLuminaName() { } public static LuminaVectorMetric fromString(String name) { - return valueOf(name.toUpperCase()); + return valueOf(name.toUpperCase(Locale.ROOT)); } /** Resolves a Lumina native metric name (e.g. "l2") to the corresponding enum constant. */ diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorMetricTest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorMetricTest.java new file mode 100644 index 000000000000..9b46ea0ebee4 --- /dev/null +++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorMetricTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.lumina.index; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * A metric name is upper-cased before {@code valueOf}, so the conversion has to pin {@link + * Locale#ROOT} or a name containing an 'i' names no enum constant under a Turkish default. + */ +class LuminaVectorMetricTest { + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void everyMetricNameParsesFromItsLowerCaseForm() { + // COSINE contains an 'I': a locale-sensitive uppercase asks for COSİNE + for (LuminaVectorMetric metric : LuminaVectorMetric.values()) { + assertThat(LuminaVectorMetric.fromString(metric.name().toLowerCase(Locale.ROOT))) + .isEqualTo(metric); + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java index ab94e9987af8..03f0a5e8cde2 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java @@ -614,7 +614,7 @@ private Schema toInitialSchema( if (!usePaimon(provider)) { if (isFormatTable(provider)) { normalizedProperties.put(TYPE.key(), FORMAT_TABLE.toString()); - normalizedProperties.put(FILE_FORMAT.key(), provider.toLowerCase()); + normalizedProperties.put(FILE_FORMAT.key(), provider.toLowerCase(Locale.ROOT)); } else { throw new UnsupportedOperationException( String.format( diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java index 3893e670685b..aa7b199f8996 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java @@ -50,13 +50,15 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; /** Catalog supports format table. */ public interface FormatTableCatalog { default boolean isFormatTable(@Nullable String provide) { - return provide != null && SparkSource.FORMAT_NAMES().contains(provide.toLowerCase()); + return provide != null + && SparkSource.FORMAT_NAMES().contains(provide.toLowerCase(Locale.ROOT)); } default Table toSparkFormatTable(Identifier ident, FormatTable formatTable) { diff --git a/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java b/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java index 06f33273e289..5a4be336f5bb 100644 --- a/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java +++ b/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java @@ -329,7 +329,7 @@ private static int getNumNestedMetaInfDirectories(Path jar, Path jarRoot) throws } private static String getFileName(Path path) { - return path.getFileName().toString().toLowerCase(); + return path.getFileName().toString().toLowerCase(Locale.ROOT); } private static boolean pathStartsWith(Path file, String path) { From 2a7577463021e92bf43879a9099ad50c21ed0460 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 17:53:53 +0800 Subject: [PATCH 6/8] fix: finish the sweep in Scala sources and pin the locale on generated names The bulk edit matched only paren-bearing calls, so it left every Scala call and, with it, the pre-lowered format-name list that isFormatTable compares against. Also pins Locale.ROOT on the String.format calls that build file names. --- .../paimon/lookup/sort/db/LocalKvDb.java | 4 +- .../org/apache/paimon/disk/FileIOChannel.java | 4 +- .../paimon/iceberg/IcebergPathFactory.java | 4 +- .../lookup/local/LocalKvStateFactory.java | 3 +- .../spark/catalog/FormatTableCatalog.java | 15 ++-- .../org/apache/paimon/spark/SparkSource.scala | 3 +- .../analysis/PaimonFunctionResolver.scala | 4 +- .../analysis/ReplacePaimonFunctions.scala | 4 +- .../catalyst/plans/logical/CopyOptions.scala | 8 ++- .../paimon/spark/write/WriteHelper.scala | 6 +- .../PaimonSqlExtensionsAstBuilder.scala | 4 +- .../spark/catalog/FormatTableCatalogTest.java | 69 +++++++++++++++++++ 12 files changed, 112 insertions(+), 16 deletions(-) create mode 100644 paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/catalog/FormatTableCatalogTest.java diff --git a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/LocalKvDb.java b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/LocalKvDb.java index bf2e5698f7aa..a0b8a7b8d655 100644 --- a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/LocalKvDb.java +++ b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/LocalKvDb.java @@ -46,6 +46,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.PriorityQueue; import java.util.TreeMap; @@ -946,7 +947,8 @@ public void accept(MemorySlice key, byte[] value) throws IOException { private File newSstFile() { long sequence = fileSequence.getAndIncrement(); - return new File(dataDirectory, String.format("sst-%s-%06d.db", uuid, sequence)); + return new File( + dataDirectory, String.format(Locale.ROOT, "sst-%s-%06d.db", uuid, sequence)); } private void ensureOpen() { diff --git a/paimon-core/src/main/java/org/apache/paimon/disk/FileIOChannel.java b/paimon-core/src/main/java/org/apache/paimon/disk/FileIOChannel.java index 4b72029dddb6..8a3dd0098ff4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/disk/FileIOChannel.java +++ b/paimon-core/src/main/java/org/apache/paimon/disk/FileIOChannel.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.IOException; import java.nio.channels.FileChannel; +import java.util.Locale; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; @@ -163,7 +164,8 @@ public Enumerator(File[] basePaths, Random random) { public FileIOChannel.ID next() { int bucketNum = GLOBAL_NUMBER.getAndIncrement() % paths.length; - String filename = String.format("%s.%06d.channel", namePrefix, (localCounter++)); + String filename = + String.format(Locale.ROOT, "%s.%06d.channel", namePrefix, (localCounter++)); return new FileIOChannel.ID(new File(paths[bucketNum], filename), bucketNum); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java index 9025dbe87ad0..da44d5ac466b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java @@ -25,6 +25,7 @@ import org.apache.paimon.utils.PathFactory; import java.io.IOException; +import java.util.Locale; import java.util.UUID; import java.util.stream.Stream; @@ -65,7 +66,8 @@ public Path toManifestListPath(String manifestListName) { } public Path toMetadataPath(long snapshotId) { - return new Path(metadataDirectory(), String.format("v%d.metadata.json", snapshotId)); + return new Path( + metadataDirectory(), String.format(Locale.ROOT, "v%d.metadata.json", snapshotId)); } public Path toMetadataPath(String metadataName) { diff --git a/paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvStateFactory.java b/paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvStateFactory.java index b3041a988ffe..3461bb3df7db 100644 --- a/paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvStateFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvStateFactory.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.time.Duration; import java.util.LinkedHashMap; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.function.LongSupplier; @@ -152,7 +153,7 @@ private LocalKvDb createDatabase(String name, @Nullable LocalKvDb.MergeOperator checkArgument(!databases.containsKey(name), "State '%s' already exists.", name); File stateDirectory = - new File(rootDirectory, String.format("state-%06d", databases.size())); + new File(rootDirectory, String.format(Locale.ROOT, "state-%06d", databases.size())); LocalKvDb db = LocalKvDb.builder(stateDirectory) .cacheManager(cacheManager) diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java index aa7b199f8996..096216a31e1d 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java @@ -22,7 +22,6 @@ import org.apache.paimon.format.csv.CsvOptions; import org.apache.paimon.format.text.TextOptions; import org.apache.paimon.options.Options; -import org.apache.paimon.spark.SparkSource; import org.apache.paimon.spark.SparkTypeUtils; import org.apache.paimon.spark.format.PaimonFormatTable; import org.apache.paimon.table.FormatTable; @@ -50,15 +49,23 @@ import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.Map; /** Catalog supports format table. */ public interface FormatTableCatalog { default boolean isFormatTable(@Nullable String provide) { - return provide != null - && SparkSource.FORMAT_NAMES().contains(provide.toLowerCase(Locale.ROOT)); + if (provide == null) { + return false; + } + // compare against the enum names rather than a pre-lowered list, so the answer does not + // depend on the locale that was in effect when that list was built + for (FormatTable.Format format : FormatTable.Format.values()) { + if (format.name().equalsIgnoreCase(provide)) { + return true; + } + } + return false; } default Table toSparkFormatTable(Identifier ident, FormatTable formatTable) { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala index 0e7c1b8b6059..9767f4a57e88 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala @@ -39,6 +39,7 @@ import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap import java.util.{Map => JMap} +import java.util.Locale import scala.collection.JavaConverters._ @@ -130,7 +131,7 @@ object SparkSource { val NAME = "paimon" - val FORMAT_NAMES: Seq[String] = Format.values.map(_.toString.toLowerCase).toSeq + val FORMAT_NAMES: Seq[String] = Format.values.map(_.toString.toLowerCase(Locale.ROOT)).toSeq // Spark dataframe read options private val CATALOG = "catalog" diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala index 61426a52ec73..3d4ee948762f 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala @@ -33,6 +33,8 @@ import org.apache.spark.sql.connector.catalog.CatalogPlugin import org.apache.spark.sql.types.{BinaryType, DataType, DayTimeIntervalType, LongType, NullType, StringType} import org.apache.spark.unsafe.types.UTF8String +import java.util.Locale + case class PaimonFunctionResolver(spark: SparkSession) extends Rule[LogicalPlan] { protected lazy val catalogManager = spark.sessionState.catalogManager @@ -122,7 +124,7 @@ case class PaimonFunctionResolver(spark: SparkSession) extends Rule[LogicalPlan] case actual if actual == expectedType => expression case actual => throw new UnsupportedOperationException( - s"$argumentName must be ${expectedType.simpleString.toUpperCase} type, " + + s"$argumentName must be ${expectedType.simpleString.toUpperCase(Locale.ROOT)} type, " + s"but found ${actual.simpleString}") } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala index d81560ffe631..fe6b96e5b3c8 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala @@ -40,6 +40,8 @@ import org.apache.spark.sql.connector.catalog.PaimonCatalogImplicits._ import org.apache.spark.sql.types.{BinaryType, DataType, DayTimeIntervalType, NullType, StringType} import org.apache.spark.unsafe.types.UTF8String +import java.util.Locale + import scala.collection.JavaConverters._ object ReplacePaimonFunctions { @@ -269,7 +271,7 @@ case class ReplacePaimonFunctions(spark: SparkSession) extends Rule[LogicalPlan] case actual if actual == expectedType => expression case actual => throw new UnsupportedOperationException( - s"$argumentName must be ${expectedType.simpleString.toUpperCase} type, " + + s"$argumentName must be ${expectedType.simpleString.toUpperCase(Locale.ROOT)} type, " + s"but found ${actual.simpleString}") } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CopyOptions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CopyOptions.scala index 3e5e35b2b78d..71a988d61f56 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CopyOptions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CopyOptions.scala @@ -18,6 +18,8 @@ package org.apache.paimon.spark.catalyst.plans.logical +import java.util.Locale + sealed trait OnErrorMode object OnErrorMode { @@ -57,7 +59,7 @@ case class CopyFileFormat(formatType: FileFormatType, options: Map[String, Strin options.foreach { case (k, v) => k match { - case "MULTI_LINE" => mapped("multiLine") = v.toLowerCase + case "MULTI_LINE" => mapped("multiLine") = v.toLowerCase(Locale.ROOT) case "COMPRESSION" => mapped("compression") = v case _ => } @@ -84,7 +86,7 @@ case class CopyFileFormat(formatType: FileFormatType, options: Map[String, Strin case (k, v) => k match { case "FIELD_DELIMITER" => mapped("sep") = v - case "HEADER" => mapped("header") = v.toLowerCase + case "HEADER" => mapped("header") = v.toLowerCase(Locale.ROOT) case "QUOTE" => mapped("quote") = v case "ESCAPE" => mapped("escape") = v case "COMPRESSION" => mapped("compression") = v @@ -226,7 +228,7 @@ object CopyFileFormat { val LIST_SEPARATOR: String = "\u001f" def parseFormatType(typeStr: String): FileFormatType = { - typeStr.toUpperCase match { + typeStr.toUpperCase(Locale.ROOT) match { case "CSV" => FileFormatType.CSV case "JSON" => FileFormatType.JSON case "PARQUET" => FileFormatType.PARQUET diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/WriteHelper.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/WriteHelper.scala index 9de665c12b16..5393697a50d0 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/WriteHelper.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/WriteHelper.scala @@ -33,6 +33,8 @@ import org.apache.spark.sql.connector.metric.CustomTaskMetric import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.metric.SQLMetrics +import java.util.Locale + import scala.collection.JavaConverters._ trait WriteHelper extends Logging { @@ -52,7 +54,9 @@ trait WriteHelper extends Logging { val executionMetrics = Compatibility.getExecutionMetrics(spark, executionId.toLong).distinct val metricUpdates = executionMetrics.flatMap { m => - commitMetrics.find(x => m.metricType.toLowerCase.contains(x.name.toLowerCase)) match { + commitMetrics.find( + x => + m.metricType.toLowerCase(Locale.ROOT).contains(x.name.toLowerCase(Locale.ROOT))) match { case Some(customTaskMetric) => Some((m.accumulatorId, customTaskMetric.value())) case None => None } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala index f5986492b65e..a4aaf3ab69b9 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala @@ -34,6 +34,8 @@ import org.apache.spark.sql.catalyst.parser.extensions.PaimonSqlExtensionsParser import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.execution.command.{CreateTableLikeCommand => SparkCreateTableLikeCommand} +import java.util.Locale + import scala.collection.JavaConverters._ import scala.collection.mutable @@ -239,7 +241,7 @@ class PaimonSqlExtensionsAstBuilder(delegate: ParserInterface) opts.foreach { opt => - val key = opt.key.getText.toUpperCase + val key = opt.key.getText.toUpperCase(Locale.ROOT) if (!seen.add(key)) { throw new IllegalArgumentException(s"Duplicate FILE_FORMAT option: $key") } diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/catalog/FormatTableCatalogTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/catalog/FormatTableCatalogTest.java new file mode 100644 index 000000000000..574338989896 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/catalog/FormatTableCatalogTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalog; + +import org.apache.paimon.table.FormatTable; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * {@code MOSAIC} contains an 'I', so a provider name folded under a Turkish default locale no + * longer matches the enum constant it names. + */ +class FormatTableCatalogTest { + + private final FormatTableCatalog catalog = new FormatTableCatalog() {}; + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void everyFormatIsRecognizedInEitherCase() { + for (FormatTable.Format format : FormatTable.Format.values()) { + assertThat(catalog.isFormatTable(format.name())).as("%s", format).isTrue(); + assertThat(catalog.isFormatTable(format.name().toLowerCase(Locale.ROOT))) + .as("%s lower case", format) + .isTrue(); + } + } + + @Test + void aNameThatIsNotAFormatIsRejected() { + assertThat(catalog.isFormatTable(null)).isFalse(); + assertThat(catalog.isFormatTable("")).isFalse(); + assertThat(catalog.isFormatTable("paimon")).isFalse(); + assertThat(catalog.isFormatTable("orcish")).isFalse(); + } +} From d741c20d8c61f87a9631f8a032d2b4997824485b Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 14 Sep 2026 11:06:39 +0800 Subject: [PATCH 7/8] fix: pin Locale.ROOT in the method-reference case conversions too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep matched `.toLowerCase()` textually and missed `String::toLowerCase` method references, so seven sites kept the JVM default locale. One of them is a correctness bug rather than an inconsistency: CdcActionCommonUtils.buildPaimonSchema folds field names through toLowerCaseIfNeed (Locale.ROOT) and key lists through listCaseConvert (default locale), so on a Turkish JVM with a case-insensitive catalog a source column ID with primary key ID yields field id and key ıd. The strict path rejects the schema; the non-strict database-sync path drops the requested partitioning instead. The other six are the same class: type-mapping option parsing, the Kafka offset-reset hint, Hive partition key names, Hive predicate-pushdown column names, and FileIO option keys. TurkishLocaleSchemaKeyTest covers inferred keys, specified keys under both strict and non-strict checking, non-strict partition handling, and an upper-case type-mapping option. All four schema cases fail on the parent commit. --- .../java/org/apache/paimon/fs/FileIO.java | 2 +- .../action/cdc/CdcActionCommonUtils.java | 3 +- .../paimon/flink/action/cdc/TypeMapping.java | 2 +- .../action/cdc/kafka/KafkaActionUtils.java | 2 +- .../cdc/TurkishLocaleSchemaKeyTest.java | 143 ++++++++++++++++++ .../apache/paimon/hive/PaimonMetaHook.java | 2 +- .../SearchArgumentToPredicateConverter.java | 9 +- 7 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleSchemaKeyTest.java diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java index 06e0bfb709c4..a3309180b017 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java @@ -557,7 +557,7 @@ static FileIO get(Path path, CatalogContext config) throws IOException { if (loader != null) { Set options = config.options().keySet().stream() - .map(String::toLowerCase) + .map(s -> s.toLowerCase(Locale.ROOT)) .collect(Collectors.toSet()); Set missOptions = new HashSet<>(); for (String[] keys : loader.requiredOptions()) { diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java index 23265ae6447f..5af8c6ed1238 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; @@ -118,7 +119,7 @@ public static boolean schemaCompatible( public static List listCaseConvert(List origin, boolean caseSensitive) { return caseSensitive ? origin - : origin.stream().map(String::toLowerCase).collect(Collectors.toList()); + : origin.stream().map(s -> s.toLowerCase(Locale.ROOT)).collect(Collectors.toList()); } public static Schema buildPaimonSchema( diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java index 429f3f3d2fa9..ce10ae6dfcf5 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java @@ -50,7 +50,7 @@ public static TypeMapping parse(String[] rawOptions) { Set typeMappingModes = Arrays.stream(rawOptions) .map(String::trim) - .map(String::toLowerCase) + .map(s -> s.toLowerCase(Locale.ROOT)) .map(TypeMappingMode::mode) .collect(Collectors.toSet()); return new TypeMapping(typeMappingModes); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaActionUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaActionUtils.java index c34c34038d3c..8f85335edd04 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaActionUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaActionUtils.java @@ -176,7 +176,7 @@ private static OffsetResetStrategy getResetStrategy(String offsetResetConfig) { offsetResetConfig, Arrays.stream(OffsetResetStrategy.values()) .map(Enum::name) - .map(String::toLowerCase) + .map(s -> s.toLowerCase(Locale.ROOT)) .collect(Collectors.joining(","))))); } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleSchemaKeyTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleSchemaKeyTest.java new file mode 100644 index 000000000000..6ad27fb293ea --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleSchemaKeyTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc; + +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +/** + * The CDC action layer case-folds identifiers and option values before matching them. {@link + * CdcActionCommonUtils#buildPaimonSchema} does it through separate helpers for field names and for + * key lists, and {@link TypeMapping#parse} does it for `--type-mapping` values. Under a Turkish + * default locale 'I' lowercases to a dotless glyph, so every one of those conversions has to pin + * {@link Locale#ROOT}: otherwise a key list holds a name no field has, and an option spelled in + * upper case stops matching any mode. + */ +class TurkishLocaleSchemaKeyTest { + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void primaryKeyInferredFromSourceSchemaNamesTheField() { + Schema source = Schema.newBuilder().column("ID", DataTypes.INT()).primaryKey("ID").build(); + + Schema result = build(Collections.emptyList(), Collections.emptyList(), source, true, true); + + assertThat(result.fields().get(0).name()).isEqualTo("id"); + assertThat(result.primaryKeys()).containsExactly("id"); + } + + @Test + void specifiedPrimaryKeyNamesTheField() { + Schema source = Schema.newBuilder().column("ID", DataTypes.INT()).build(); + + Schema result = + build( + Collections.emptyList(), + Collections.singletonList("ID"), + source, + false, + false); + + assertThat(result.primaryKeys()).containsExactly("id"); + } + + @Test + void specifiedPrimaryKeyPassesStrictChecking() { + Schema source = Schema.newBuilder().column("ID", DataTypes.INT()).build(); + + assertThatCode( + () -> + build( + Collections.emptyList(), + Collections.singletonList("ID"), + source, + true, + false)) + .doesNotThrowAnyException(); + } + + @Test + void specifiedPartitionKeySurvivesNonStrictChecking() { + Schema source = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("CITY", DataTypes.STRING()) + .build(); + + Schema result = + build( + Collections.singletonList("CITY"), + Collections.emptyList(), + source, + false, + false); + + assertThat(result.partitionKeys()).containsExactly("city"); + } + + @Test + void upperCaseTypeMappingOptionStillMatchesItsMode() { + TypeMapping mapping = TypeMapping.parse(new String[] {"TINYINT1-NOT-BOOL"}); + + assertThat(mapping.containsMode(TypeMapping.TypeMappingMode.TINYINT1_NOT_BOOL)).isTrue(); + } + + private static Schema build( + List specifiedPartitionKeys, + List specifiedPrimaryKeys, + Schema sourceSchema, + boolean strictlyCheckSpecified, + boolean syncPKeysFromSourceSchema) { + return CdcActionCommonUtils.buildPaimonSchema( + "T", + specifiedPartitionKeys, + specifiedPrimaryKeys, + Collections.emptyList(), + Collections.emptyMap(), + sourceSchema, + new CdcMetadataConverter[0], + false, + strictlyCheckSpecified, + false, + syncPKeysFromSourceSchema); + } +} diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java index 071195fde6ab..5c96a5e80876 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java @@ -136,7 +136,7 @@ public void preCreateTable(Table table) throws MetaException { List partitionKeys = table.getPartitionKeys().stream() .map(FieldSchema::getName) - .map(String::toLowerCase) + .map(s -> s.toLowerCase(Locale.ROOT)) .collect(Collectors.toList()); schemaBuilder.partitionKeys(partitionKeys); } diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java index 38830af66a3e..202cceae296f 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -62,11 +63,15 @@ public SearchArgumentToPredicateConverter( this.root = searchArgument.getExpression(); this.leaves = searchArgument.getLeaves(); this.hiveColumnNames = - columnNames.stream().map(String::toLowerCase).collect(Collectors.toList()); + columnNames.stream() + .map(s -> s.toLowerCase(Locale.ROOT)) + .collect(Collectors.toList()); this.columnTypes = columnTypes; if (readColumnNames != null) { readColumnNames = - readColumnNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + readColumnNames.stream() + .map(s -> s.toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()); } this.readColumnNames = readColumnNames; From 620f15f2943535ab47a973f594536b5f382439c0 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 14 Sep 2026 11:25:12 +0800 Subject: [PATCH 8/8] test: assert the resulting keys in the strict-checking case assertThatCode(...).doesNotThrowAnyException() could not tell "the strict path accepted the key" from "it accepted the key and stored a different one". Assert the primary keys instead. Renamed the class: it also covers a --type-mapping option value, which is not a schema key. --- ...java => TurkishLocaleCaseFoldingTest.java} | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) rename paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/{TurkishLocaleSchemaKeyTest.java => TurkishLocaleCaseFoldingTest.java} (83%) diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleSchemaKeyTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleCaseFoldingTest.java similarity index 83% rename from paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleSchemaKeyTest.java rename to paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleCaseFoldingTest.java index 6ad27fb293ea..2ff3505a0388 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleSchemaKeyTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleCaseFoldingTest.java @@ -30,17 +30,16 @@ import java.util.Locale; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; /** * The CDC action layer case-folds identifiers and option values before matching them. {@link * CdcActionCommonUtils#buildPaimonSchema} does it through separate helpers for field names and for - * key lists, and {@link TypeMapping#parse} does it for `--type-mapping` values. Under a Turkish - * default locale 'I' lowercases to a dotless glyph, so every one of those conversions has to pin - * {@link Locale#ROOT}: otherwise a key list holds a name no field has, and an option spelled in - * upper case stops matching any mode. + * key lists, and {@link TypeMapping#parse} does it for {@code --type-mapping} values. Under a + * Turkish default locale 'I' lowercases to a dotless glyph, so every one of those conversions has + * to pin {@link Locale#ROOT}: otherwise a key list holds a name no field has, and an option spelled + * in upper case stops matching any mode. */ -class TurkishLocaleSchemaKeyTest { +class TurkishLocaleCaseFoldingTest { private Locale original; @@ -84,15 +83,15 @@ void specifiedPrimaryKeyNamesTheField() { void specifiedPrimaryKeyPassesStrictChecking() { Schema source = Schema.newBuilder().column("ID", DataTypes.INT()).build(); - assertThatCode( - () -> - build( - Collections.emptyList(), - Collections.singletonList("ID"), - source, - true, - false)) - .doesNotThrowAnyException(); + Schema result = + build( + Collections.emptyList(), + Collections.singletonList("ID"), + source, + true, + false); + + assertThat(result.primaryKeys()).containsExactly("id"); } @Test