diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 7b0665c50296..d61aec287f41 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -3273,7 +3273,7 @@ public Map statsModePerLevel() { } public static String normalizeFileFormat(String fileFormat) { - return StringUtils.isEmpty(fileFormat) ? fileFormat : fileFormat.toLowerCase(); + return StringUtils.isEmpty(fileFormat) ? fileFormat : fileFormat.toLowerCase(Locale.ROOT); } public String dataFilePrefix() { @@ -4238,7 +4238,10 @@ public String partitionMarkDoneCustomClass() { public Set partitionMarkDoneActions() { return Arrays.stream(options.get(PARTITION_MARK_DONE_ACTION).split(",")) - .map(x -> PartitionMarkDoneAction.valueOf(x.replace('-', '_').toUpperCase())) + .map( + x -> + PartitionMarkDoneAction.valueOf( + x.replace('-', '_').toUpperCase(Locale.ROOT))) .collect(Collectors.toCollection(HashSet::new)); } diff --git a/paimon-api/src/main/java/org/apache/paimon/options/OptionsUtils.java b/paimon-api/src/main/java/org/apache/paimon/options/OptionsUtils.java index a625454f3996..e0ca14be9b1a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/options/OptionsUtils.java +++ b/paimon-api/src/main/java/org/apache/paimon/options/OptionsUtils.java @@ -199,7 +199,7 @@ static Boolean convertToBoolean(Object o) { return (Boolean) o; } - switch (o.toString().toUpperCase()) { + switch (o.toString().toUpperCase(Locale.ROOT)) { case "TRUE": return true; case "FALSE": diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java index 8205dfe21295..2bee7d2261a5 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.function.Function; @@ -221,7 +222,9 @@ private static String extractRequestId(ClassicHttpResponse response) { .filter( h -> h.getName() != null - && h.getName().toLowerCase().contains("request-id")) + && h.getName() + .toLowerCase(Locale.ROOT) + .contains("request-id")) .map(Header::getValue) .filter(Objects::nonNull) .findFirst() diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java index 1ce7d0a79aa4..3dfac337dcd8 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java @@ -22,6 +22,7 @@ import org.apache.paimon.rest.RESTCatalogOptions; import org.apache.paimon.utils.StringUtils; +import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -96,7 +97,7 @@ protected static String parseSigningAlgoFromUri(String uri) { } // Check for aliyun openapi endpoints - if (uri.toLowerCase().contains("dlfnext")) { + if (uri.toLowerCase(Locale.ROOT).contains("dlfnext")) { return DLFOpenApiSigner.IDENTIFIER; } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java index 76daf5c72f62..d572afa88c95 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java @@ -34,6 +34,7 @@ import java.util.Base64; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.TreeMap; @@ -62,12 +63,12 @@ public class DLFDefaultSigner implements DLFRequestSigner { private static final String NEW_LINE = "\n"; private static final List SIGNED_HEADERS = Arrays.asList( - DLF_CONTENT_MD5_HEADER_KEY.toLowerCase(), - DLF_CONTENT_TYPE_KEY.toLowerCase(), - DLF_CONTENT_SHA56_HEADER_KEY.toLowerCase(), - DLF_DATE_HEADER_KEY.toLowerCase(), - DLF_AUTH_VERSION_HEADER_KEY.toLowerCase(), - DLF_SECURITY_TOKEN_HEADER_KEY.toLowerCase()); + DLF_CONTENT_MD5_HEADER_KEY.toLowerCase(Locale.ROOT), + DLF_CONTENT_TYPE_KEY.toLowerCase(Locale.ROOT), + DLF_CONTENT_SHA56_HEADER_KEY.toLowerCase(Locale.ROOT), + DLF_DATE_HEADER_KEY.toLowerCase(Locale.ROOT), + DLF_AUTH_VERSION_HEADER_KEY.toLowerCase(Locale.ROOT), + DLF_SECURITY_TOKEN_HEADER_KEY.toLowerCase(Locale.ROOT)); private final String region; @@ -215,7 +216,7 @@ private static TreeMap buildSortedSignedHeadersMap( TreeMap orderMap = new TreeMap<>(); if (headers != null) { for (Map.Entry header : headers.entrySet()) { - String key = header.getKey().toLowerCase(); + String key = header.getKey().toLowerCase(Locale.ROOT); if (SIGNED_HEADERS.contains(key)) { orderMap.put(key, StringUtils.trim(header.getValue())); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java index aaf37138283d..0d4858f5817a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java @@ -184,7 +184,7 @@ public String identifier() { private static String buildCanonicalizedHeaders(Map headers) { TreeMap sortedHeaders = new TreeMap<>(); for (Map.Entry entry : headers.entrySet()) { - String key = entry.getKey().toLowerCase(); + String key = entry.getKey().toLowerCase(Locale.ROOT); if (key.startsWith("x-acs-")) { sortedHeaders.put(key, StringUtils.trim(entry.getValue())); } diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java index 5076b577812b..5f26799ab932 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -213,7 +214,7 @@ private static List tokenize(String chars) { builder.setLength(0); cursor = consumeIdentifier(builder, chars, cursor); final String token = builder.toString(); - final String normalizedToken = token.toUpperCase(); + final String normalizedToken = token.toUpperCase(Locale.ROOT); if (KEYWORDS.contains(normalizedToken)) { tokens.add(new Token(TokenType.KEYWORD, cursor, normalizedToken)); } else { @@ -344,7 +345,7 @@ private enum Keyword { private static final Set KEYWORDS = Stream.of(Keyword.values()) - .map(k -> k.toString().toUpperCase()) + .map(k -> k.toString().toUpperCase(Locale.ROOT)) .collect(Collectors.toSet()); private static class Token { diff --git a/paimon-api/src/main/java/org/apache/paimon/types/RowKind.java b/paimon-api/src/main/java/org/apache/paimon/types/RowKind.java index 17096dfbfc6f..f7206d272294 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/RowKind.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/RowKind.java @@ -20,6 +20,8 @@ import org.apache.paimon.annotation.Public; +import java.util.Locale; + /** * Lists all kinds of changes that a row can describe in a changelog. * @@ -135,7 +137,7 @@ public static RowKind fromByteValue(byte value) { * @see #shortString() for mapping of string and {@link RowKind}. */ public static RowKind fromShortString(String value) { - switch (value.toUpperCase()) { + switch (value.toUpperCase(Locale.ROOT)) { case "+I": return INSERT; case "-U": diff --git a/paimon-api/src/main/java/org/apache/paimon/utils/StringUtils.java b/paimon-api/src/main/java/org/apache/paimon/utils/StringUtils.java index 4a46deca4d37..9e321436737d 100644 --- a/paimon-api/src/main/java/org/apache/paimon/utils/StringUtils.java +++ b/paimon-api/src/main/java/org/apache/paimon/utils/StringUtils.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Random; import java.util.concurrent.ThreadLocalRandom; @@ -668,7 +669,9 @@ public static String quote(String str) { } public static String toLowerCaseIfNeed(String str, boolean caseSensitive) { - return caseSensitive ? str : str.toLowerCase(); + // Locale.ROOT: identifier matching must not depend on the JVM default locale + // (e.g. Turkish lowercases 'I' to a dotless glyph and breaks column mapping) + return caseSensitive ? str : str.toLowerCase(Locale.ROOT); } public static boolean isNumeric(final CharSequence cs) { @@ -733,14 +736,14 @@ public static String toUpperCase(String value) { if (value == null) { return null; } - return value.toUpperCase(); + return value.toUpperCase(Locale.ROOT); } public static String toLowerCase(String value) { if (value == null) { return null; } - return value.toLowerCase(); + return value.toLowerCase(Locale.ROOT); } public static boolean isOpenBracket(char c) { diff --git a/paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java b/paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java new file mode 100644 index 000000000000..a397c25b7ee6 --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/TurkishLocaleParsingTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon; + +import org.apache.paimon.options.Options; +import org.apache.paimon.types.RowKind; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.apache.paimon.CoreOptions.PARTITION_MARK_DONE_ACTION; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Parsing an option value, an enum name or a protocol token uppercases or lowercases it first. + * Under a Turkish default locale 'i' maps to a dotted capital and 'I' to a dotless small letter, so + * those conversions must pin {@link Locale#ROOT} or the token no longer matches what it is compared + * against. + */ +class TurkishLocaleParsingTest { + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void partitionMarkDoneActionsParse() { + // SUCCESS_FILE and DONE_PARTITION both contain an 'i': a locale-sensitive uppercase + // turns them into names no enum constant has, and valueOf throws + Options options = new Options(); + options.set(PARTITION_MARK_DONE_ACTION, "success-file,done-partition"); + + assertThat(new CoreOptions(options).partitionMarkDoneActions()) + .containsExactlyInAnyOrder( + CoreOptions.PartitionMarkDoneAction.SUCCESS_FILE, + CoreOptions.PartitionMarkDoneAction.DONE_PARTITION); + } + + @Test + void rowKindFromLowerCaseShortString() { + // "+i" is the only short string this can catch: Turkish differs from ROOT on 'i' and + // 'I' alone, so "-d" or "-u" would pass whichever conversion the code uses + assertThat(RowKind.fromShortString("+i")).isEqualTo(RowKind.INSERT); + } +} diff --git a/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/OperatingSystem.java b/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/OperatingSystem.java index 0e15ca154cf6..0fd6cd92a827 100644 --- a/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/OperatingSystem.java +++ b/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/OperatingSystem.java @@ -18,6 +18,8 @@ package org.apache.paimon.benchmark.metric.cpu; +import java.util.Locale; + /** An enumeration indicating the operating system that the JVM runs on. */ public enum OperatingSystem { LINUX, @@ -115,7 +117,7 @@ private static OperatingSystem readOSFromSystemProperties() { if (osName.startsWith(FREEBSD_OS_PREFIX)) { return FREE_BSD; } - String osNameLowerCase = osName.toLowerCase(); + String osNameLowerCase = osName.toLowerCase(Locale.ROOT); if (osNameLowerCase.contains(SOLARIS_OS_INFIX_1) || osNameLowerCase.contains(SOLARIS_OS_INFIX_2)) { return SOLARIS; diff --git a/paimon-common/src/main/java/org/apache/paimon/compression/BlockCompressionFactory.java b/paimon-common/src/main/java/org/apache/paimon/compression/BlockCompressionFactory.java index 60dd1657bc57..563ee625bb40 100644 --- a/paimon-common/src/main/java/org/apache/paimon/compression/BlockCompressionFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/compression/BlockCompressionFactory.java @@ -23,6 +23,8 @@ import javax.annotation.Nullable; +import java.util.Locale; + /** * Each compression codec has an implementation of {@link BlockCompressionFactory} to create * compressors and decompressors. @@ -38,7 +40,7 @@ public interface BlockCompressionFactory { /** Creates {@link BlockCompressionFactory} according to the configuration. */ @Nullable static BlockCompressionFactory create(CompressOptions compression) { - switch (compression.compress().toUpperCase()) { + switch (compression.compress().toUpperCase(Locale.ROOT)) { case "NONE": return null; case "ZSTD": diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java index d6bfe7a0d3fe..1e6b56c3f9d2 100644 --- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java +++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -88,16 +89,19 @@ public static FileFormat fromIdentifier(String identifier, Options options) { /** Create a {@link FileFormat} from format identifier and format options. */ public static FileFormat fromIdentifier(String identifier, FormatContext context) { return FormatFactoryUtil.discoverFactory( - FileFormat.class.getClassLoader(), identifier.toLowerCase()) + FileFormat.class.getClassLoader(), identifier.toLowerCase(Locale.ROOT)) .create(context); } protected Options getIdentifierPrefixOptions(Options options) { Map result = new HashMap<>(); - String prefix = formatIdentifier.toLowerCase() + "."; + // match against the identifier as written so the suffix is sliced at an offset the key + // actually has: lower-casing can lengthen a string, and U+0130 lower-cases to two chars + String prefix = formatIdentifier + "."; + String lowerCasePrefix = formatIdentifier.toLowerCase(Locale.ROOT) + "."; for (String key : options.keySet()) { - if (key.toLowerCase().startsWith(prefix)) { - result.put(prefix + key.substring(prefix.length()), options.get(key)); + if (key.regionMatches(true, 0, prefix, 0, prefix.length())) { + result.put(lowerCasePrefix + key.substring(prefix.length()), options.get(key)); } } return new Options(result); diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java index c20e2c70bb18..a3309180b017 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java @@ -48,6 +48,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Queue; @@ -556,13 +557,13 @@ static FileIO get(Path path, CatalogContext config) throws IOException { if (loader != null) { Set options = config.options().keySet().stream() - .map(String::toLowerCase) + .map(s -> s.toLowerCase(Locale.ROOT)) .collect(Collectors.toSet()); Set missOptions = new HashSet<>(); for (String[] keys : loader.requiredOptions()) { boolean found = false; for (String key : keys) { - if (options.contains(key.toLowerCase())) { + if (options.contains(key.toLowerCase(Locale.ROOT))) { found = true; break; } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java index e3c910aa4b32..4f506f5d4e42 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java @@ -25,6 +25,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.PriorityQueue; @@ -63,7 +64,7 @@ public static String normalizeRanker(String ranker) { if (ranker == null || ranker.trim().isEmpty()) { return RRF_RANKER; } - String normalized = ranker.trim().toLowerCase(); + String normalized = ranker.trim().toLowerCase(Locale.ROOT); if (!RRF_RANKER.equals(normalized) && !WEIGHTED_SCORE_RANKER.equals(normalized) && !MRR_RANKER.equals(normalized)) { diff --git a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/LocalKvDb.java b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/LocalKvDb.java index bf2e5698f7aa..a0b8a7b8d655 100644 --- a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/LocalKvDb.java +++ b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/LocalKvDb.java @@ -46,6 +46,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.PriorityQueue; import java.util.TreeMap; @@ -946,7 +947,8 @@ public void accept(MemorySlice key, byte[] value) throws IOException { private File newSstFile() { long sequence = fileSequence.getAndIncrement(); - return new File(dataDirectory, String.format("sst-%s-%06d.db", uuid, sequence)); + return new File( + dataDirectory, String.format(Locale.ROOT, "sst-%s-%06d.db", uuid, sequence)); } private void ensureOpen() { diff --git a/paimon-common/src/main/java/org/apache/paimon/statistics/SimpleColStatsCollector.java b/paimon-common/src/main/java/org/apache/paimon/statistics/SimpleColStatsCollector.java index a9281020444e..c72bea629597 100644 --- a/paimon-common/src/main/java/org/apache/paimon/statistics/SimpleColStatsCollector.java +++ b/paimon-common/src/main/java/org/apache/paimon/statistics/SimpleColStatsCollector.java @@ -22,6 +22,7 @@ import org.apache.paimon.format.SimpleColStats; import java.util.Arrays; +import java.util.Locale; import java.util.regex.Matcher; import static org.apache.paimon.statistics.TruncateSimpleColStatsCollector.TRUNCATE_PATTERN; @@ -62,7 +63,7 @@ static SimpleColStatsCollector[] create(SimpleColStatsCollector.Factory[] factor } static Factory from(String option) { - String upper = option.toUpperCase(); + String upper = option.toUpperCase(Locale.ROOT); switch (upper) { case "NONE": return NoneSimpleColStatsCollector::new; diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java index 1cc1366bc889..ceb328053d65 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.net.URI; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -61,7 +62,7 @@ public static void validateTableRoot(Path tableRoot, BlobDescriptor descriptor) } private static String normalizeScheme(@Nullable String scheme) { - return scheme == null ? null : scheme.toLowerCase(); + return scheme == null ? null : scheme.toLowerCase(Locale.ROOT); } /** diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/JNIUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/JNIUtils.java index c9b4c657a4d7..eff716be3dc9 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/JNIUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/JNIUtils.java @@ -24,6 +24,7 @@ import java.io.InputStream; import java.security.AccessController; import java.security.PrivilegedAction; +import java.util.Locale; import java.util.logging.Logger; /** @@ -37,7 +38,7 @@ public class JNIUtils { private static boolean inited = false; private static String osName() { - String osName = System.getProperty("os.name").toLowerCase().replace(' ', '_'); + String osName = System.getProperty("os.name").toLowerCase(Locale.ROOT).replace(' ', '_'); if (osName.startsWith("win")) { return "win"; } else { diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/OperatingSystem.java b/paimon-common/src/main/java/org/apache/paimon/utils/OperatingSystem.java index 9e923b3faa2a..3188def893f4 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/OperatingSystem.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/OperatingSystem.java @@ -18,6 +18,8 @@ package org.apache.paimon.utils; +import java.util.Locale; + /** An enumeration indicating the operating system that the JVM runs on. */ public enum OperatingSystem { LINUX, @@ -115,7 +117,7 @@ private static OperatingSystem readOSFromSystemProperties() { if (osName.startsWith(FREEBSD_OS_PREFIX)) { return FREE_BSD; } - String osNameLowerCase = osName.toLowerCase(); + String osNameLowerCase = osName.toLowerCase(Locale.ROOT); if (osNameLowerCase.contains(SOLARIS_OS_INFIX_1) || osNameLowerCase.contains(SOLARIS_OS_INFIX_2)) { return SOLARIS; diff --git a/paimon-common/src/test/java/org/apache/paimon/format/FileFormatPrefixOptionsTest.java b/paimon-common/src/test/java/org/apache/paimon/format/FileFormatPrefixOptionsTest.java new file mode 100644 index 000000000000..23b6deda7892 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/format/FileFormatPrefixOptionsTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format; + +import org.apache.paimon.options.Options; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.types.RowType; + +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +/** Tests for the format-prefixed option filtering in {@link FileFormat}. */ +class FileFormatPrefixOptionsTest { + + /** U+0130, whose lower case under {@link Locale#ROOT} is the two characters below. */ + private static final String DOTTED_CAPITAL_I = "İ"; + + private static final String LOWER_CASE_DOTTED_I = "i̇"; + + @Test + void prefixIsMatchedCaseInsensitively() { + Options options = new Options(); + options.set("ORC.compression", "zstd"); + options.set("avro.codec", "snappy"); + + Options prefixed = new TestFileFormat("orc").getIdentifierPrefixOptions(options); + + assertThat(prefixed.toMap()).containsExactly(entry("orc.compression", "zstd")); + } + + @Test + void anIdentifierThatLowerCasesLongerKeepsItsOptions() { + assertThat(DOTTED_CAPITAL_I.toLowerCase(Locale.ROOT)).isEqualTo(LOWER_CASE_DOTTED_I); + + Options options = new Options(); + options.set(DOTTED_CAPITAL_I + ".compression", "zstd"); + + Options prefixed = new TestFileFormat(DOTTED_CAPITAL_I).getIdentifierPrefixOptions(options); + + // slicing the key at the length of the lower-cased prefix would drop the "c" + assertThat(prefixed.toMap()) + .containsExactly(entry(LOWER_CASE_DOTTED_I + ".compression", "zstd")); + } + + @Test + void aKeyThatIsOnlyThePrefixYieldsAnEmptySuffix() { + Options options = new Options(); + options.set(DOTTED_CAPITAL_I + ".", "zstd"); + + Options prefixed = new TestFileFormat(DOTTED_CAPITAL_I).getIdentifierPrefixOptions(options); + + // this key is shorter than the lower-cased prefix, so slicing at its length overruns it + assertThat(prefixed.toMap()).containsExactly(entry(LOWER_CASE_DOTTED_I + ".", "zstd")); + } + + private static class TestFileFormat extends FileFormat { + + private TestFileFormat(String formatIdentifier) { + super(formatIdentifier); + } + + @Override + public FormatReaderFactory createReaderFactory( + RowType dataSchemaRowType, + RowType projectedRowType, + @Nullable List filters) { + throw new UnsupportedOperationException(); + } + + @Override + public FormatWriterFactory createWriterFactory(RowType type) { + throw new UnsupportedOperationException(); + } + + @Override + public void validateDataFields(RowType rowType) {} + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/StringUtilsTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/StringUtilsTest.java index c4cb9b95bc4f..c0bf33c59704 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/StringUtilsTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/StringUtilsTest.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Random; import static org.assertj.core.api.Assertions.assertThat; @@ -451,6 +452,21 @@ void testToLowerCaseIfNeed(String input, boolean caseSensitive, String expected) String result = StringUtils.toLowerCaseIfNeed(input, caseSensitive); assertThat(result).isEqualTo(expected); } + + @Test + void testToLowerCaseIfNeedIndependentOfDefaultLocale() { + Locale original = Locale.getDefault(); + try { + // Turkish lowercases 'I' to a dotless glyph under its locale; + // identifier matching must stay locale-independent + Locale.setDefault(new Locale("tr", "TR")); + assertThat(StringUtils.toLowerCaseIfNeed("INDEX", false)).isEqualTo("index"); + assertThat(StringUtils.toLowerCase("INDEX")).isEqualTo("index"); + assertThat(StringUtils.toUpperCase("ıindex")).isEqualTo("IINDEX"); + } finally { + Locale.setDefault(original); + } + } } @Nested diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index 916f63350c7b..98acf9fd8de3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -62,6 +62,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -408,7 +409,7 @@ public static Table loadTable( private static Table createGlobalSystemTable(String tableName, Catalog catalog) throws Catalog.TableNotExistException { - switch (tableName.toLowerCase()) { + switch (tableName.toLowerCase(Locale.ROOT)) { case ALL_TABLE_OPTIONS: List tables = listAllTables(catalog); Map> allOptions = new HashMap<>(); diff --git a/paimon-core/src/main/java/org/apache/paimon/disk/FileIOChannel.java b/paimon-core/src/main/java/org/apache/paimon/disk/FileIOChannel.java index 4b72029dddb6..8a3dd0098ff4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/disk/FileIOChannel.java +++ b/paimon-core/src/main/java/org/apache/paimon/disk/FileIOChannel.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.IOException; import java.nio.channels.FileChannel; +import java.util.Locale; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; @@ -163,7 +164,8 @@ public Enumerator(File[] basePaths, Random random) { public FileIOChannel.ID next() { int bucketNum = GLOBAL_NUMBER.getAndIncrement() % paths.length; - String filename = String.format("%s.%06d.channel", namePrefix, (localCounter++)); + String filename = + String.format(Locale.ROOT, "%s.%06d.channel", namePrefix, (localCounter++)); return new FileIOChannel.ID(new File(paths[bucketNum], filename), bucketNum); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java index 9025dbe87ad0..da44d5ac466b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java @@ -25,6 +25,7 @@ import org.apache.paimon.utils.PathFactory; import java.io.IOException; +import java.util.Locale; import java.util.UUID; import java.util.stream.Stream; @@ -65,7 +66,8 @@ public Path toManifestListPath(String manifestListName) { } public Path toMetadataPath(long snapshotId) { - return new Path(metadataDirectory(), String.format("v%d.metadata.json", snapshotId)); + return new Path( + metadataDirectory(), String.format(Locale.ROOT, "v%d.metadata.json", snapshotId)); } public Path toMetadataPath(String metadataName) { diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/DistributedLockDialectFactory.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/DistributedLockDialectFactory.java index e0c167a4df05..f324aa41e1f7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/jdbc/DistributedLockDialectFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/DistributedLockDialectFactory.java @@ -18,9 +18,11 @@ package org.apache.paimon.jdbc; +import java.util.Locale; + class DistributedLockDialectFactory { static JdbcDistributedLockDialect create(String protocol) { - JdbcProtocol type = JdbcProtocol.valueOf(protocol.toUpperCase()); + JdbcProtocol type = JdbcProtocol.valueOf(protocol.toUpperCase(Locale.ROOT)); switch (type) { case SQLITE: return new SqlLiteDistributedLockDialect(); diff --git a/paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvStateFactory.java b/paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvStateFactory.java index b3041a988ffe..3461bb3df7db 100644 --- a/paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvStateFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvStateFactory.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.time.Duration; import java.util.LinkedHashMap; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.function.LongSupplier; @@ -152,7 +153,7 @@ private LocalKvDb createDatabase(String name, @Nullable LocalKvDb.MergeOperator checkArgument(!databases.containsKey(name), "State '%s' already exists.", name); File stateDirectory = - new File(rootDirectory, String.format("state-%06d", databases.size())); + new File(rootDirectory, String.format(Locale.ROOT, "state-%06d", databases.size())); LocalKvDb db = LocalKvDb.builder(stateDirectory) .cacheManager(cacheManager) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java index ba8f8bf48cff..d30d627fb459 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java @@ -35,6 +35,7 @@ import org.apache.paimon.types.RowType; import java.io.IOException; +import java.util.Locale; import java.util.Map; /** @@ -106,7 +107,7 @@ private static String describeUnsupportedMode( } return "from-timestamp"; default: - return effectiveMode.name().toLowerCase().replace('_', '-'); + return effectiveMode.name().toLowerCase(Locale.ROOT).replace('_', '-'); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java index acb487e2573b..7f3ee12d6ef2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -104,7 +105,7 @@ enum Format { /** Parses a file format string to a corresponding {@link Format} enum constant. */ static Format parseFormat(String fileFormat) { try { - return Format.valueOf(fileFormat.toUpperCase()); + return Format.valueOf(fileFormat.toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { throw new UnsupportedOperationException( "Format table unsupported file format: " diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java index fdb57385abf3..876f49d8226a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java @@ -40,6 +40,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -311,7 +312,7 @@ private String option(String key) { if (value == null) { value = table.options().get(key); } - return value == null ? null : value.toLowerCase().trim(); + return value == null ? null : value.toLowerCase(Locale.ROOT).trim(); } private static boolean containsField(GlobalIndexMeta meta, int fieldId) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java index 776730946850..c5ed1e2ecdcc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -103,7 +104,7 @@ public class SystemTableLoader { @Nullable public static Table load(String type, FileStoreTable dataTable) { - String name = type.toLowerCase(); + String name = type.toLowerCase(Locale.ROOT); if (PHYSICAL_METADATA_TABLES.contains(name) && dataTable.coreOptions().queryAuthEnabled()) { throw new UnsupportedOperationException( String.format( diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java index 9e979a98a9da..aec2b21aaed2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java @@ -42,6 +42,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; @@ -302,7 +303,7 @@ private static void escapeChar(char c, StringBuilder sb) { if (c < 16) { sb.append('0'); } - sb.append(Integer.toHexString(c).toUpperCase()); + sb.append(Integer.toHexString(c).toUpperCase(Locale.ROOT)); } public static String unescapePathName(String path) { diff --git a/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java b/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java index ebb167cbacc4..28e38b050752 100644 --- a/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java +++ b/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java @@ -55,6 +55,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.regex.Matcher; @@ -263,7 +264,7 @@ private static void createTable( @VisibleForTesting static String toSnakeCase(String name) { - return name.replaceAll("(.)([A-Z][a-z])", "$1_$2").toLowerCase(); + return name.replaceAll("(.)([A-Z][a-z])", "$1_$2").toLowerCase(Locale.ROOT); } @VisibleForTesting diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java index 0f0096751d9e..ddec6d38e008 100644 --- a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java +++ b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java @@ -33,6 +33,7 @@ import java.io.UncheckedIOException; import java.net.URI; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -52,8 +53,10 @@ public class COSNFileIO extends HadoopCompliantFileIO { private static final Map CASE_SENSITIVE_KEYS = new HashMap() { { - put(COSN_USER_INFO_KEY_ID.toLowerCase(), COSN_USER_INFO_KEY_ID); - put(COSN_USER_INFO_KEY_SECRET.toLowerCase(), COSN_USER_INFO_KEY_SECRET); + put(COSN_USER_INFO_KEY_ID.toLowerCase(Locale.ROOT), COSN_USER_INFO_KEY_ID); + put( + COSN_USER_INFO_KEY_SECRET.toLowerCase(Locale.ROOT), + COSN_USER_INFO_KEY_SECRET); } }; /** @@ -77,8 +80,8 @@ public void configure(CatalogContext context) { for (String prefix : CONFIG_PREFIXES) { if (key.startsWith(prefix)) { String value = context.options().get(key); - if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase())) { - key = CASE_SENSITIVE_KEYS.get(key.toLowerCase()); + if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase(Locale.ROOT))) { + key = CASE_SENSITIVE_KEYS.get(key.toLowerCase(Locale.ROOT)); } hadoopOptions.set(key, value); LOG.debug( diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 1e4646d4b9f9..271672ab7e04 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -48,6 +48,7 @@ import java.net.URI; import java.time.Duration; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -83,9 +84,9 @@ public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsP private static final Map CASE_SENSITIVE_KEYS = new HashMap() { { - put(OSS_ACCESS_KEY_ID.toLowerCase(), OSS_ACCESS_KEY_ID); - put(OSS_ACCESS_KEY_SECRET.toLowerCase(), OSS_ACCESS_KEY_SECRET); - put(OSS_SECURITY_TOKEN.toLowerCase(), OSS_SECURITY_TOKEN); + put(OSS_ACCESS_KEY_ID.toLowerCase(Locale.ROOT), OSS_ACCESS_KEY_ID); + put(OSS_ACCESS_KEY_SECRET.toLowerCase(Locale.ROOT), OSS_ACCESS_KEY_SECRET); + put(OSS_SECURITY_TOKEN.toLowerCase(Locale.ROOT), OSS_SECURITY_TOKEN); } }; @@ -122,8 +123,8 @@ public void configure(CatalogContext context) { for (String prefix : CONFIG_PREFIXES) { if (key.startsWith(prefix)) { String value = context.options().get(key); - if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase())) { - key = CASE_SENSITIVE_KEYS.get(key.toLowerCase()); + if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase(Locale.ROOT))) { + key = CASE_SENSITIVE_KEYS.get(key.toLowerCase(Locale.ROOT)); } hadoopOptions.set(key, value); diff --git a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/OBSFileIO.java b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/OBSFileIO.java index 4f425ca59065..408bcc0a7b89 100644 --- a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/OBSFileIO.java +++ b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/OBSFileIO.java @@ -33,6 +33,7 @@ import java.io.UncheckedIOException; import java.net.URI; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -60,9 +61,9 @@ public class OBSFileIO extends HadoopCompliantFileIO { private static final Map CASE_SENSITIVE_KEYS = new HashMap() { { - put(OBS_ACCESS_KEY_ID.toLowerCase(), OBS_ACCESS_KEY_ID); - put(OBS_ACCESS_KEY_SECRET.toLowerCase(), OBS_ACCESS_KEY_SECRET); - put(OBS_SESSION_TOKEN.toLowerCase(), OBS_SESSION_TOKEN); + put(OBS_ACCESS_KEY_ID.toLowerCase(Locale.ROOT), OBS_ACCESS_KEY_ID); + put(OBS_ACCESS_KEY_SECRET.toLowerCase(Locale.ROOT), OBS_ACCESS_KEY_SECRET); + put(OBS_SESSION_TOKEN.toLowerCase(Locale.ROOT), OBS_SESSION_TOKEN); } }; @@ -92,8 +93,8 @@ public void configure(CatalogContext context) { for (String prefix : CONFIG_PREFIXES) { if (key.startsWith(prefix)) { String value = context.options().get(key); - if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase())) { - key = CASE_SENSITIVE_KEYS.get(key.toLowerCase()); + if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase(Locale.ROOT))) { + key = CASE_SENSITIVE_KEYS.get(key.toLowerCase(Locale.ROOT)); } hadoopOptions.set(key, value); diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java index 5a3b48a5c54b..3249cbfcef77 100644 --- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java @@ -58,6 +58,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -115,9 +116,9 @@ public class OSSFileIO extends HadoopCompliantFileIO implements HadoopOptionsPro private static final Map CASE_SENSITIVE_KEYS = new HashMap() { { - put(OSS_ACCESS_KEY_ID.toLowerCase(), OSS_ACCESS_KEY_ID); - put(OSS_ACCESS_KEY_SECRET.toLowerCase(), OSS_ACCESS_KEY_SECRET); - put(OSS_SECURITY_TOKEN.toLowerCase(), OSS_SECURITY_TOKEN); + put(OSS_ACCESS_KEY_ID.toLowerCase(Locale.ROOT), OSS_ACCESS_KEY_ID); + put(OSS_ACCESS_KEY_SECRET.toLowerCase(Locale.ROOT), OSS_ACCESS_KEY_SECRET); + put(OSS_SECURITY_TOKEN.toLowerCase(Locale.ROOT), OSS_SECURITY_TOKEN); } }; @@ -147,8 +148,8 @@ public void configure(CatalogContext context) { for (String prefix : CONFIG_PREFIXES) { if (key.startsWith(prefix)) { String value = context.options().get(key); - if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase())) { - key = CASE_SENSITIVE_KEYS.get(key.toLowerCase()); + if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase(Locale.ROOT))) { + key = CASE_SENSITIVE_KEYS.get(key.toLowerCase(Locale.ROOT)); } hadoopOptions.set(key, value); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java index 23265ae6447f..5af8c6ed1238 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; @@ -118,7 +119,7 @@ public static boolean schemaCompatible( public static List listCaseConvert(List origin, boolean caseSensitive) { return caseSensitive ? origin - : origin.stream().map(String::toLowerCase).collect(Collectors.toList()); + : origin.stream().map(s -> s.toLowerCase(Locale.ROOT)).collect(Collectors.toList()); } public static Schema buildPaimonSchema( diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/Expression.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/Expression.java index 087fe15e67a2..9c82b0faa46f 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/Expression.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/Expression.java @@ -37,6 +37,7 @@ import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -191,11 +192,11 @@ public ExpressionCreator getCreator() { Arrays.stream(ExpressionFunction.values()) .collect( Collectors.toMap( - value -> value.name().toLowerCase(), + value -> value.name().toLowerCase(Locale.ROOT), ExpressionFunction::getCreator)); public static ExpressionCreator creator(String exprName) { - return EXPRESSION_FUNCTIONS.get(exprName.toLowerCase()); + return EXPRESSION_FUNCTIONS.get(exprName.toLowerCase(Locale.ROOT)); } } @@ -253,7 +254,7 @@ static Expression create( String exprName, String... args) { - ExpressionCreator function = ExpressionFunction.creator(exprName.toLowerCase()); + ExpressionCreator function = ExpressionFunction.creator(exprName.toLowerCase(Locale.ROOT)); if (function == null) { throw new UnsupportedOperationException( String.format( diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java index d7393485ad73..d0ca252fbee0 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.regex.Pattern; @@ -113,7 +114,10 @@ public SyncDatabaseActionBase withDbPrefix(Map dbPrefix) { dbPrefix.entrySet().stream() .collect( HashMap::new, - (m, e) -> m.put(e.getKey().toLowerCase(), e.getValue()), + (m, e) -> + m.put( + e.getKey().toLowerCase(Locale.ROOT), + e.getValue()), HashMap::putAll); } return this; @@ -125,7 +129,10 @@ public SyncDatabaseActionBase withDbSuffix(Map dbSuffix) { dbSuffix.entrySet().stream() .collect( HashMap::new, - (m, e) -> m.put(e.getKey().toLowerCase(), e.getValue()), + (m, e) -> + m.put( + e.getKey().toLowerCase(Locale.ROOT), + e.getValue()), HashMap::putAll); } return this; diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java index 7dd63ed2273e..44cd51be83e9 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java @@ -22,6 +22,7 @@ import java.io.Serializable; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import static org.apache.paimon.utils.StringUtils.toLowerCaseIfNeed; @@ -78,8 +79,8 @@ public TableNameConverter( public String convert(String originDbName, String originTblName) { // top priority: table mapping - if (tableMapping.containsKey(originTblName.toLowerCase())) { - String mappedName = tableMapping.get(originTblName.toLowerCase()); + if (tableMapping.containsKey(originTblName.toLowerCase(Locale.ROOT))) { + String mappedName = tableMapping.get(originTblName.toLowerCase(Locale.ROOT)); return toLowerCaseIfNeed(mappedName, caseSensitive); } @@ -87,11 +88,11 @@ public String convert(String originDbName, String originTblName) { String tblSuffix = suffix; // second priority: prefix and postfix specified by db - if (dbPrefix.containsKey(originDbName.toLowerCase())) { - tblPrefix = dbPrefix.get(originDbName.toLowerCase()); + if (dbPrefix.containsKey(originDbName.toLowerCase(Locale.ROOT))) { + tblPrefix = dbPrefix.get(originDbName.toLowerCase(Locale.ROOT)); } - if (dbSuffix.containsKey(originDbName.toLowerCase())) { - tblSuffix = dbSuffix.get(originDbName.toLowerCase()); + if (dbSuffix.containsKey(originDbName.toLowerCase(Locale.ROOT))) { + tblSuffix = dbSuffix.get(originDbName.toLowerCase(Locale.ROOT)); } // third priority: normal prefix and suffix @@ -117,7 +118,7 @@ private Map lowerMapKey(Map map) { } for (String key : map.keySet()) { - lowerKeyMap.put(key.toLowerCase(), map.get(key)); + lowerKeyMap.put(key.toLowerCase(Locale.ROOT), map.get(key)); } return lowerKeyMap; diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java index 499c3823b9e1..ce10ae6dfcf5 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TypeMapping.java @@ -21,6 +21,7 @@ import java.io.Serializable; import java.util.Arrays; import java.util.Collections; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -49,7 +50,7 @@ public static TypeMapping parse(String[] rawOptions) { Set typeMappingModes = Arrays.stream(rawOptions) .map(String::trim) - .map(String::toLowerCase) + .map(s -> s.toLowerCase(Locale.ROOT)) .map(TypeMappingMode::mode) .collect(Collectors.toSet()); return new TypeMapping(typeMappingModes); @@ -96,7 +97,7 @@ public static TypeMappingMode mode(String option) { } public String configString() { - return name().toLowerCase().replace("_", "-"); + return name().toLowerCase(Locale.ROOT).replace("_", "-"); } } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/DataFormatFactory.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/DataFormatFactory.java index df1e92d90792..0893c0012435 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/DataFormatFactory.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/DataFormatFactory.java @@ -22,13 +22,15 @@ import org.apache.paimon.factories.FactoryException; import org.apache.paimon.factories.FactoryUtil; +import java.util.Locale; + /** Factory to create {@link DataFormat}. */ public interface DataFormatFactory extends Factory { DataFormat create(); static DataFormat createDataFormat(String format) { - String identifier = format.toLowerCase(); + String identifier = format.toLowerCase(Locale.ROOT); DataFormatFactory dataFormatFactory; try { dataFormatFactory = diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaActionUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaActionUtils.java index c34c34038d3c..8f85335edd04 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaActionUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaActionUtils.java @@ -176,7 +176,7 @@ private static OffsetResetStrategy getResetStrategy(String offsetResetConfig) { offsetResetConfig, Arrays.stream(OffsetResetStrategy.values()) .map(Enum::name) - .map(String::toLowerCase) + .map(s -> s.toLowerCase(Locale.ROOT)) .collect(Collectors.joining(","))))); } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBActionUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBActionUtils.java index ddc69a180c71..0fa299811f93 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBActionUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBActionUtils.java @@ -33,6 +33,7 @@ import org.apache.kafka.connect.json.JsonConverterConfig; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -118,7 +119,7 @@ public static MongoDBSource buildMongodbSource( .collectionList(tableList); String startupMode = mongodbConfig.get(SourceOptions.SCAN_STARTUP_MODE); - switch (startupMode.toLowerCase()) { + switch (startupMode.toLowerCase(Locale.ROOT)) { case INITIAL_MODE: sourceBuilder.startupOptions(StartupOptions.initial()); break; diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSchemaUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSchemaUtils.java index 9dbb6eef7df5..0c3571889887 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSchemaUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSchemaUtils.java @@ -38,6 +38,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Objects; import static org.apache.flink.cdc.connectors.mongodb.internal.MongoDBEnvelope.encodeValue; @@ -153,7 +154,8 @@ private static String buildConnectionString( } private static SchemaAcquisitionMode getModeFromConfig(Configuration mongodbConfig) { - return SchemaAcquisitionMode.valueOf(mongodbConfig.get(START_MODE).toUpperCase()); + return SchemaAcquisitionMode.valueOf( + mongodbConfig.get(START_MODE).toUpperCase(Locale.ROOT)); } private static List getColumnNames(Document document) { diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java index 0a72f711604e..f0ad32eac27b 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -86,7 +87,8 @@ default Map getExtractRow( return null; } SchemaAcquisitionMode mode = - SchemaAcquisitionMode.valueOf(mongodbConfig.get(START_MODE).toUpperCase()); + SchemaAcquisitionMode.valueOf( + mongodbConfig.get(START_MODE).toUpperCase(Locale.ROOT)); ObjectNode objectNode = JsonSerdeUtil.asSpecificNodeType(jsonNode.asText(), ObjectNode.class); JsonNode idNode = objectNode.get(ID_FIELD); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java index 989236d8c854..efb2367a8e30 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java @@ -37,6 +37,7 @@ import java.nio.ByteBuffer; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -153,8 +154,8 @@ public static Tuple3 getTypeInfo(String typeName) { if (leftBracketIndex != -1) { int rightBracketIndex = typeName.indexOf(RIGHT_BRACKETS); shortType = - typeName.substring(0, leftBracketIndex).trim().toUpperCase() - + typeName.substring(rightBracketIndex + 1).toUpperCase(); + typeName.substring(0, leftBracketIndex).trim().toUpperCase(Locale.ROOT) + + typeName.substring(rightBracketIndex + 1).toUpperCase(Locale.ROOT); String insideBrackets = typeName.substring(leftBracketIndex + 1, rightBracketIndex).trim(); @@ -167,7 +168,7 @@ public static Tuple3 getTypeInfo(String typeName) { length = Integer.parseInt(insideBrackets); } } else { - shortType = typeName.toUpperCase(); + shortType = typeName.toUpperCase(Locale.ROOT); if (isDecimalType(shortType)) { // when missing precision and scale of the decimal, we // use the max precision and scale to avoid parse error @@ -187,7 +188,7 @@ public static DataType toDataType( return DataTypes.STRING(); } - switch (type.toUpperCase()) { + switch (type.toUpperCase(Locale.ROOT)) { case BIT: if (length == null || length == 1) { return DataTypes.BOOLEAN(); @@ -335,7 +336,7 @@ public static DataType toDataType( } public static boolean isGeoType(String type) { - switch (type.toUpperCase()) { + switch (type.toUpperCase(Locale.ROOT)) { case GEOMETRY: case POINT: case LINESTRING: diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresTypeUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresTypeUtils.java index 7fae4c4327be..9ec33e498a94 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresTypeUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresTypeUtils.java @@ -26,6 +26,8 @@ import javax.annotation.Nullable; +import java.util.Locale; + import static org.apache.paimon.flink.action.cdc.TypeMapping.TypeMappingMode.TO_STRING; /** Converts from Postgres type to {@link DataType}. */ @@ -83,7 +85,7 @@ public static DataType toDataType( } precision = precision == null ? 0 : precision; scale = scale == null ? 0 : scale; - switch (typeName.toLowerCase()) { + switch (typeName.toLowerCase(Locale.ROOT)) { case PG_BIT: case PG_VARBIT: if (precision <= 1) { diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java index b23d0d6f06de..3093ebbfb54c 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java @@ -24,6 +24,7 @@ import java.io.Serializable; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -58,7 +59,10 @@ public Map data() { public CdcRecord fieldNameLowerCase() { Map newData = new HashMap<>(); for (Map.Entry entry : data.entrySet()) { - newData.put(entry.getKey().toLowerCase(), entry.getValue()); + // Locale.ROOT: must match the schema-side toLowerCaseIfNeed conversion; + // a Turkish default locale would lowercase 'I' to a dotless glyph and + // silently null out the column on the record-schema join + newData.put(entry.getKey().toLowerCase(Locale.ROOT), entry.getValue()); } return new CdcRecord(kind, newData); } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleCaseFoldingTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleCaseFoldingTest.java new file mode 100644 index 000000000000..2ff3505a0388 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TurkishLocaleCaseFoldingTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc; + +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The CDC action layer case-folds identifiers and option values before matching them. {@link + * CdcActionCommonUtils#buildPaimonSchema} does it through separate helpers for field names and for + * key lists, and {@link TypeMapping#parse} does it for {@code --type-mapping} values. Under a + * Turkish default locale 'I' lowercases to a dotless glyph, so every one of those conversions has + * to pin {@link Locale#ROOT}: otherwise a key list holds a name no field has, and an option spelled + * in upper case stops matching any mode. + */ +class TurkishLocaleCaseFoldingTest { + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void primaryKeyInferredFromSourceSchemaNamesTheField() { + Schema source = Schema.newBuilder().column("ID", DataTypes.INT()).primaryKey("ID").build(); + + Schema result = build(Collections.emptyList(), Collections.emptyList(), source, true, true); + + assertThat(result.fields().get(0).name()).isEqualTo("id"); + assertThat(result.primaryKeys()).containsExactly("id"); + } + + @Test + void specifiedPrimaryKeyNamesTheField() { + Schema source = Schema.newBuilder().column("ID", DataTypes.INT()).build(); + + Schema result = + build( + Collections.emptyList(), + Collections.singletonList("ID"), + source, + false, + false); + + assertThat(result.primaryKeys()).containsExactly("id"); + } + + @Test + void specifiedPrimaryKeyPassesStrictChecking() { + Schema source = Schema.newBuilder().column("ID", DataTypes.INT()).build(); + + Schema result = + build( + Collections.emptyList(), + Collections.singletonList("ID"), + source, + true, + false); + + assertThat(result.primaryKeys()).containsExactly("id"); + } + + @Test + void specifiedPartitionKeySurvivesNonStrictChecking() { + Schema source = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("CITY", DataTypes.STRING()) + .build(); + + Schema result = + build( + Collections.singletonList("CITY"), + Collections.emptyList(), + source, + false, + false); + + assertThat(result.partitionKeys()).containsExactly("city"); + } + + @Test + void upperCaseTypeMappingOptionStillMatchesItsMode() { + TypeMapping mapping = TypeMapping.parse(new String[] {"TINYINT1-NOT-BOOL"}); + + assertThat(mapping.containsMode(TypeMapping.TypeMappingMode.TINYINT1_NOT_BOOL)).isTrue(); + } + + private static Schema build( + List specifiedPartitionKeys, + List specifiedPrimaryKeys, + Schema sourceSchema, + boolean strictlyCheckSpecified, + boolean syncPKeysFromSourceSchema) { + return CdcActionCommonUtils.buildPaimonSchema( + "T", + specifiedPartitionKeys, + specifiedPrimaryKeys, + Collections.emptyList(), + Collections.emptyMap(), + sourceSchema, + new CdcMetadataConverter[0], + false, + strictlyCheckSpecified, + false, + syncPKeysFromSourceSchema); + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/TurkishLocaleTypeNameTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/TurkishLocaleTypeNameTest.java new file mode 100644 index 000000000000..911d27467a16 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/TurkishLocaleTypeNameTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.mysql; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * A source type name is case-folded before it is matched against the names this class knows. Under + * a Turkish default locale 'i' maps to a dotted capital, so those conversions must pin {@link + * Locale#ROOT} or a type whose name contains an 'i' stops being recognized. + */ +class TurkishLocaleTypeNameTest { + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void lowerCaseTypeNameKeepsItsShortType() { + // a locale-sensitive uppercase turns "int" into "İNT", which no case arm names + assertThat(MySqlTypeUtils.getTypeInfo("int").f0).isEqualTo("INT"); + } + + @Test + void lowerCaseGeoTypeIsStillRecognized() { + assertThat(MySqlTypeUtils.isGeoType("point")).isTrue(); + assertThat(MySqlTypeUtils.isGeoType("linestring")).isTrue(); + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordTest.java new file mode 100644 index 000000000000..eadbe0fc3630 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink.cdc; + +import org.apache.paimon.types.RowKind; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Test for {@link CdcRecord}. */ +class CdcRecordTest { + + @Test + void fieldNameLowerCaseIndependentOfDefaultLocale() { + Locale original = Locale.getDefault(); + try { + // Turkish lowercases 'I' to a dotless glyph under its locale; the record-side + // keys must match the schema-side Locale.ROOT conversion or the column join + // silently nulls out + Locale.setDefault(new Locale("tr", "TR")); + Map data = new HashMap<>(); + data.put("INDEX", "v"); + CdcRecord record = new CdcRecord(RowKind.INSERT, data); + + CdcRecord converted = record.fieldNameLowerCase(); + + assertThat(converted.data()).containsEntry("index", "v").doesNotContainKey("ındex"); + } finally { + Locale.setDefault(original); + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java index 5dbce5f0a42b..fa0322090d4b 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -88,7 +89,7 @@ public CatalogTable copy(Map map) { public Map getOptions() { if (cachedOptions == null) { cachedOptions = new HashMap<>(); - String format = table.format().name().toLowerCase(); + String format = table.format().name().toLowerCase(Locale.ROOT); Map options = table.options(); options.forEach( (k, v) -> { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java index 4e71b14d25f8..1101bdc5b765 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java @@ -31,6 +31,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -71,7 +72,7 @@ public interface ActionFactory extends Factory { static Optional createAction(String[] args) { // to be compatible with old usage - String action = args[0].toLowerCase().replaceAll("-", "_"); + String action = args[0].toLowerCase(Locale.ROOT).replaceAll("-", "_"); String[] actionArgs = Arrays.copyOfRange(args, 1, args.length); ActionFactory actionFactory; try { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java index 4ee87710e6fe..8b3441943541 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java @@ -29,6 +29,7 @@ import javax.annotation.Nullable; import java.util.List; +import java.util.Locale; import java.util.Map; /** Clone source table to target table. */ @@ -104,7 +105,7 @@ public CloneAction( this.preferFileFormat = StringUtils.isNullOrWhitespaceOnly(preferFileFormat) ? preferFileFormat - : preferFileFormat.toLowerCase(); + : preferFileFormat.toLowerCase(Locale.ROOT); this.cloneFrom = cloneFrom; this.metaOnly = metaOnly; this.cloneIfExists = cloneIfExists; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/MultiTablesSinkMode.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/MultiTablesSinkMode.java index d5c514199589..d5c0d06be2f1 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/MultiTablesSinkMode.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/MultiTablesSinkMode.java @@ -21,6 +21,7 @@ import javax.annotation.Nullable; import java.io.Serializable; +import java.util.Locale; /** * There are two modes for database sink. @@ -40,7 +41,7 @@ public static MultiTablesSinkMode fromString(@Nullable String mode) { return DIVIDED; } - switch (mode.toLowerCase()) { + switch (mode.toLowerCase(Locale.ROOT)) { case "divided": return DIVIDED; case "combined": @@ -51,6 +52,6 @@ public static MultiTablesSinkMode fromString(@Nullable String mode) { } public String configString() { - return name().toLowerCase().replace("_", "-"); + return name().toLowerCase(Locale.ROOT).replace("_", "-"); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneFileFormatUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneFileFormatUtils.java index b1c73fce7d9d..88fbd11fba24 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneFileFormatUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneFileFormatUtils.java @@ -21,6 +21,8 @@ import org.apache.paimon.flink.action.CloneAction; import org.apache.paimon.utils.StringUtils; +import java.util.Locale; + /** Utils for file format in {@link CloneAction}. */ public class CloneFileFormatUtils { @@ -28,7 +30,7 @@ public static void validateFileFormat(String fileFormat) { if (StringUtils.isNullOrWhitespaceOnly(fileFormat)) { return; } - String fileFormatLower = fileFormat.toLowerCase(); + String fileFormatLower = fileFormat.toLowerCase(Locale.ROOT); String[] supportedFileFormat = new String[] {"parquet", "orc", "avro"}; for (String supportedFormat : supportedFileFormat) { if (fileFormatLower.equals(supportedFormat)) { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionLoader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionLoader.java index 49d236a69302..688321e31fd6 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionLoader.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionLoader.java @@ -34,6 +34,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Locale; /** Specify partitions for lookup tables. */ public abstract class PartitionLoader implements Serializable { @@ -95,7 +96,7 @@ public static PartitionLoader of(Table table) { FlinkConnectorOptions.SCAN_PARTITIONS.key()); int maxPartitionNum = -1; - switch (scanPartitions.toLowerCase()) { + switch (scanPartitions.toLowerCase(Locale.ROOT)) { case MAX_PT: maxPartitionNum = 1; break; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedure.java index 8fb6fac49eca..f93d19a0dcf8 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedure.java @@ -37,6 +37,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; @@ -114,7 +115,7 @@ public String[] call( // Parse options Options userOptions = createUserOptions(table, options); - indexType = indexType.toLowerCase().trim(); + indexType = indexType.toLowerCase(Locale.ROOT).trim(); if (indexColumns.size() > 1) { // Fail fast before submitting the job: index types that do not support multi-column // throw from GlobalIndexerFactory#create, which happens before any indexer side effect. diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DropGlobalIndexProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DropGlobalIndexProcedure.java index 098c30cbe48a..d74d0d2ad784 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DropGlobalIndexProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DropGlobalIndexProcedure.java @@ -46,6 +46,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; @@ -112,7 +113,7 @@ public String[] call( PartitionPredicate partitionPredicate = parsePartitionPredicate(table, partitions); // Normalize index type - final String indexTypeLower = indexType.toLowerCase().trim(); + final String indexTypeLower = indexType.toLowerCase(Locale.ROOT).trim(); // Get latest snapshot Snapshot snapshot = diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/aggregate/LocalAggregator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/aggregate/LocalAggregator.java index b1a6dc8cf5c2..009407f08516 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/aggregate/LocalAggregator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/aggregate/LocalAggregator.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; /** Local aggregator based on split statistics. */ @@ -229,7 +230,7 @@ private DataType resultType() { } private String resultName() { - return kind.name().toLowerCase(); + return kind.name().toLowerCase(Locale.ROOT); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MultiTablesSinkModeTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MultiTablesSinkModeTest.java new file mode 100644 index 000000000000..42e23207692b --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MultiTablesSinkModeTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The mode name is case-folded before it is matched, so the conversion has to pin {@link + * Locale#ROOT} or a name containing an 'i' stops matching under a Turkish default. + */ +class MultiTablesSinkModeTest { + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void upperCaseModeNameStillParses() { + // DIVIDED contains an 'I', which a locale-sensitive lowercase turns into a dotless one + assertThat(MultiTablesSinkMode.fromString("DIVIDED")) + .isEqualTo(MultiTablesSinkMode.DIVIDED); + assertThat(MultiTablesSinkMode.fromString("divided")) + .isEqualTo(MultiTablesSinkMode.DIVIDED); + } + + @Test + void configStringRoundTrips() { + for (MultiTablesSinkMode mode : MultiTablesSinkMode.values()) { + assertThat(MultiTablesSinkMode.fromString(mode.configString())).isEqualTo(mode); + } + } +} diff --git a/paimon-format/src/main/java/org/apache/orc/OrcFile.java b/paimon-format/src/main/java/org/apache/orc/OrcFile.java index f482174edc1f..06e59cd1c9ae 100644 --- a/paimon-format/src/main/java/org/apache/orc/OrcFile.java +++ b/paimon-format/src/main/java/org/apache/orc/OrcFile.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Properties; @@ -513,7 +514,9 @@ protected WriterOptions(Properties tableProperties, Configuration conf) { blockPaddingValue = OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf); compressValue = CompressionKind.valueOf( - OrcConf.COMPRESS.getString(tableProperties, conf).toUpperCase()); + OrcConf.COMPRESS + .getString(tableProperties, conf) + .toUpperCase(Locale.ROOT)); enforceBufferSize = OrcConf.ENFORCE_COMPRESSION_BUFFER_SIZE.getBoolean(tableProperties, conf); String versionName = OrcConf.WRITE_FORMAT.getString(tableProperties, conf); diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index b3ab44f64eb5..8420010267af 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -94,6 +94,7 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -1170,8 +1171,8 @@ private boolean usingExternalTable(Map tableOptions) { String externalPropValue = tableOptions.getOrDefault( - HIVE_EXTERNAL_TABLE_PROP.toLowerCase(), - tableOptions.get(HIVE_EXTERNAL_TABLE_PROP.toUpperCase())); + HIVE_EXTERNAL_TABLE_PROP.toLowerCase(Locale.ROOT), + tableOptions.get(HIVE_EXTERNAL_TABLE_PROP.toUpperCase(Locale.ROOT))); return CatalogTableType.EXTERNAL.equals(tableType) || "TRUE".equalsIgnoreCase(externalPropValue); } @@ -1705,7 +1706,7 @@ private Table newHmsTable( } else { // format table table.getParameters().put(TABLE_TYPE_PROP, provider.name()); - table.getParameters().put(FILE_FORMAT.key(), provider.name().toLowerCase()); + table.getParameters().put(FILE_FORMAT.key(), provider.name().toLowerCase(Locale.ROOT)); table.getParameters().put(TYPE.key(), FORMAT_TABLE.toString()); } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java index 56ac5b82f44f..3a8e8cc8b6c8 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import static org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM; import static org.apache.paimon.CoreOptions.FILE_FORMAT; @@ -103,7 +104,7 @@ public static Schema tryToFormatSchema(Table hiveTable) { String serLib = serdeInfo.getSerializationLib() == null ? "" - : serdeInfo.getSerializationLib().toLowerCase(); + : serdeInfo.getSerializationLib().toLowerCase(Locale.ROOT); String inputFormat = sd.getInputFormat() == null ? "" : sd.getInputFormat(); if (serLib.contains("parquet")) { format = Format.PARQUET; @@ -132,7 +133,7 @@ public static Schema tryToFormatSchema(Table hiveTable) { rowType.getFields().forEach(f -> builder.column(f.name(), f.type(), f.description())); options.set(PATH, location); options.set(TYPE, FORMAT_TABLE); - options.set(FILE_FORMAT, format.name().toLowerCase()); + options.set(FILE_FORMAT, format.name().toLowerCase(Locale.ROOT)); return builder.partitionKeys(partitionKeys) .options(options.toMap()) .comment(comment) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveCloneUtils.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveCloneUtils.java index f1d412ace1d6..1c6bb5a349d0 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveCloneUtils.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveCloneUtils.java @@ -43,6 +43,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Predicate; @@ -205,7 +206,7 @@ private static String parseFormat(StorageDescriptor sd) { String serLib = serdeInfo.getSerializationLib() == null ? "" - : serdeInfo.getSerializationLib().toLowerCase(); + : serdeInfo.getSerializationLib().toLowerCase(Locale.ROOT); String inputFormat = sd.getInputFormat() == null ? "" : sd.getInputFormat(); if (serLib.contains("avro")) { return "avro"; diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveTableCloneExtractor.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveTableCloneExtractor.java index 0d3baf29420d..61ddb4096c07 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveTableCloneExtractor.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/clone/HiveTableCloneExtractor.java @@ -45,6 +45,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -226,10 +227,13 @@ private static String parseCompression( public static Map getIdentifierPrefixOptions( String formatIdentifier, Map options) { Map result = new HashMap<>(); - String prefix = formatIdentifier.toLowerCase() + "."; + // match against the identifier as written so the suffix is sliced at an offset the key + // actually has: lower-casing can lengthen a string, and U+0130 lower-cases to two chars + String prefix = formatIdentifier + "."; + String lowerCasePrefix = formatIdentifier.toLowerCase(Locale.ROOT) + "."; for (String key : options.keySet()) { - if (key.toLowerCase().startsWith(prefix)) { - result.put(prefix + key.substring(prefix.length()), options.get(key)); + if (key.regionMatches(true, 0, prefix, 0, prefix.length())) { + result.put(lowerCasePrefix + key.substring(prefix.length()), options.get(key)); } } return result; @@ -238,9 +242,11 @@ public static Map getIdentifierPrefixOptions( public static Map getOptionsWhenCloneSplits(Table table, String format) { Map result = new HashMap<>(); if (FormatTable.Format.JSON.name().equalsIgnoreCase(format)) { - result.put(FILE_FORMAT.key(), FormatTable.Format.PARQUET.name().toLowerCase()); + result.put( + FILE_FORMAT.key(), FormatTable.Format.PARQUET.name().toLowerCase(Locale.ROOT)); } else if (FormatTable.Format.CSV.name().equalsIgnoreCase(format)) { - result.put(FILE_FORMAT.key(), FormatTable.Format.PARQUET.name().toLowerCase()); + result.put( + FILE_FORMAT.key(), FormatTable.Format.PARQUET.name().toLowerCase(Locale.ROOT)); } else { result.put(FILE_FORMAT.key(), format); } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java index 483b21106ba0..3da673dd2508 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/pool/CachedClientPool.java @@ -164,7 +164,7 @@ static Key extractKey(String clientClassName, String cacheKeys, Configuration co key); confElements.put(key, conf.get(key)); } else { - KeyElementType type = KeyElementType.valueOf(trimmed.toUpperCase()); + KeyElementType type = KeyElementType.valueOf(trimmed.toUpperCase(Locale.ROOT)); switch (type) { case UGI: case USER_NAME: diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/clone/HiveTableCloneExtractorTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/clone/HiveTableCloneExtractorTest.java new file mode 100644 index 000000000000..dcceed0a1546 --- /dev/null +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/clone/HiveTableCloneExtractorTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hive.clone; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +/** Tests for the format-prefixed option filtering in {@link HiveTableCloneExtractor}. */ +class HiveTableCloneExtractorTest { + + /** U+0130, whose lower case under {@link Locale#ROOT} is the two characters below. */ + private static final String DOTTED_CAPITAL_I = "İ"; + + private static final String LOWER_CASE_DOTTED_I = "i̇"; + + @Test + void prefixIsMatchedCaseInsensitively() { + Map options = new HashMap<>(); + options.put("ORC.compression", "zstd"); + options.put("avro.codec", "snappy"); + + assertThat(HiveTableCloneExtractor.getIdentifierPrefixOptions("orc", options)) + .containsExactly(entry("orc.compression", "zstd")); + } + + @Test + void anIdentifierThatLowerCasesLongerKeepsItsOptions() { + assertThat(DOTTED_CAPITAL_I.toLowerCase(Locale.ROOT)).isEqualTo(LOWER_CASE_DOTTED_I); + + Map options = new HashMap<>(); + options.put(DOTTED_CAPITAL_I + ".compression", "zstd"); + + // slicing the key at the length of the lower-cased prefix would drop the "c" + assertThat(HiveTableCloneExtractor.getIdentifierPrefixOptions(DOTTED_CAPITAL_I, options)) + .containsExactly(entry(LOWER_CASE_DOTTED_I + ".compression", "zstd")); + } + + @Test + void aKeyThatIsOnlyThePrefixYieldsAnEmptySuffix() { + Map options = new HashMap<>(); + options.put(DOTTED_CAPITAL_I + ".", "zstd"); + + // this key is shorter than the lower-cased prefix, so slicing at its length overruns it + assertThat(HiveTableCloneExtractor.getIdentifierPrefixOptions(DOTTED_CAPITAL_I, options)) + .containsExactly(entry(LOWER_CASE_DOTTED_I + ".", "zstd")); + } +} diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/pool/TestCachedClientPool.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/pool/TestCachedClientPool.java index 0b1456538281..ea551e6e44c2 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/pool/TestCachedClientPool.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/pool/TestCachedClientPool.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.security.PrivilegedAction; import java.util.List; +import java.util.Locale; import java.util.UUID; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORECONNECTURLKEY; @@ -59,6 +60,23 @@ public void testExtractKeyUnknownElementErrorMessage() { .hasMessage("Unknown key element conf"); } + @Test + public void testExtractKeyUnderATurkishDefaultLocale() { + // UGI contains an 'I', so a locale-sensitive uppercase asks the enum for UGİ + Locale original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + try { + assertThat( + CachedClientPool.extractKey( + HiveMetaStoreClient.class.getName(), + "ugi", + new Configuration())) + .isNotNull(); + } finally { + Locale.setDefault(original); + } + } + @Test public void testCacheKeyNotSame() { // client1 use cache key type:paimon diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java index aac33db8700b..7321a58aa343 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -172,10 +173,10 @@ public static HiveSchema extract(@Nullable Configuration configuration, Properti paimonSchema.fields().stream() .collect( Collectors.toMap( - dataField -> dataField.name().toLowerCase(), + dataField -> dataField.name().toLowerCase(Locale.ROOT), Function.identity())); for (int i = 0; i < columnNames.size(); i++) { - String columnName = columnNames.get(i).toLowerCase(); + String columnName = columnNames.get(i).toLowerCase(Locale.ROOT); if (Objects.equals(columnName, tagToPartField)) { // ignore tagToPartField, it should just be a string type continue; diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java index c032c2ff43e8..5c96a5e80876 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -78,8 +79,8 @@ public void preCreateTable(Table table) throws MetaException { table.getSd().setInputFormat(PaimonInputFormat.class.getCanonicalName()); table.getSd().setOutputFormat(PaimonOutputFormat.class.getCanonicalName()); - table.setDbName(table.getDbName().toLowerCase()); - table.setTableName(table.getTableName().toLowerCase()); + table.setDbName(table.getDbName().toLowerCase(Locale.ROOT)); + table.setTableName(table.getTableName().toLowerCase(Locale.ROOT)); String location = LocationKeyExtractor.getPaimonLocation(conf, table); Identifier identifier = Identifier.create(table.getDbName(), table.getTableName()); if (location == null) { @@ -115,7 +116,7 @@ public void preCreateTable(Table table) throws MetaException { .forEachRemaining( fieldSchema -> schemaBuilder.column( - fieldSchema.getName().toLowerCase(), + fieldSchema.getName().toLowerCase(Locale.ROOT), toPaimonType(fieldSchema.getType()), fieldSchema.getComment())); // partition columns @@ -128,14 +129,14 @@ public void preCreateTable(Table table) throws MetaException { .forEachRemaining( fieldSchema -> schemaBuilder.column( - fieldSchema.getName().toLowerCase(), + fieldSchema.getName().toLowerCase(Locale.ROOT), toPaimonType(fieldSchema.getType()), fieldSchema.getComment())); List partitionKeys = table.getPartitionKeys().stream() .map(FieldSchema::getName) - .map(String::toLowerCase) + .map(s -> s.toLowerCase(Locale.ROOT)) .collect(Collectors.toList()); schemaBuilder.partitionKeys(partitionKeys); } diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java index 38830af66a3e..202cceae296f 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -62,11 +63,15 @@ public SearchArgumentToPredicateConverter( this.root = searchArgument.getExpression(); this.leaves = searchArgument.getLeaves(); this.hiveColumnNames = - columnNames.stream().map(String::toLowerCase).collect(Collectors.toList()); + columnNames.stream() + .map(s -> s.toLowerCase(Locale.ROOT)) + .collect(Collectors.toList()); this.columnTypes = columnTypes; if (readColumnNames != null) { readColumnNames = - readColumnNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + readColumnNames.stream() + .map(s -> s.toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()); } this.readColumnNames = readColumnNames; diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java index 3232868b5327..1cd93a9f248e 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -88,7 +89,7 @@ public PaimonRecordReader( .boxed() .collect( Collectors.toMap( - index -> paimonColumns.get(index).toLowerCase(), + index -> paimonColumns.get(index).toLowerCase(Locale.ROOT), index -> index, (existing, replacement) -> existing, LinkedHashMap::new)); diff --git a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorMetric.java b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorMetric.java index f4240d701381..212d8a890b7b 100644 --- a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorMetric.java +++ b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorMetric.java @@ -18,6 +18,8 @@ package org.apache.paimon.lumina.index; +import java.util.Locale; + /** Enumeration of supported Lumina vector similarity metrics. */ public enum LuminaVectorMetric { @@ -45,7 +47,7 @@ public String getLuminaName() { } public static LuminaVectorMetric fromString(String name) { - return valueOf(name.toUpperCase()); + return valueOf(name.toUpperCase(Locale.ROOT)); } /** Resolves a Lumina native metric name (e.g. "l2") to the corresponding enum constant. */ diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorMetricTest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorMetricTest.java new file mode 100644 index 000000000000..9b46ea0ebee4 --- /dev/null +++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorMetricTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.lumina.index; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * A metric name is upper-cased before {@code valueOf}, so the conversion has to pin {@link + * Locale#ROOT} or a name containing an 'i' names no enum constant under a Turkish default. + */ +class LuminaVectorMetricTest { + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void everyMetricNameParsesFromItsLowerCaseForm() { + // COSINE contains an 'I': a locale-sensitive uppercase asks for COSİNE + for (LuminaVectorMetric metric : LuminaVectorMetric.values()) { + assertThat(LuminaVectorMetric.fromString(metric.name().toLowerCase(Locale.ROOT))) + .isEqualTo(metric); + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java index ab94e9987af8..03f0a5e8cde2 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java @@ -614,7 +614,7 @@ private Schema toInitialSchema( if (!usePaimon(provider)) { if (isFormatTable(provider)) { normalizedProperties.put(TYPE.key(), FORMAT_TABLE.toString()); - normalizedProperties.put(FILE_FORMAT.key(), provider.toLowerCase()); + normalizedProperties.put(FILE_FORMAT.key(), provider.toLowerCase(Locale.ROOT)); } else { throw new UnsupportedOperationException( String.format( diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java index 3893e670685b..096216a31e1d 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java @@ -22,7 +22,6 @@ import org.apache.paimon.format.csv.CsvOptions; import org.apache.paimon.format.text.TextOptions; import org.apache.paimon.options.Options; -import org.apache.paimon.spark.SparkSource; import org.apache.paimon.spark.SparkTypeUtils; import org.apache.paimon.spark.format.PaimonFormatTable; import org.apache.paimon.table.FormatTable; @@ -56,7 +55,17 @@ public interface FormatTableCatalog { default boolean isFormatTable(@Nullable String provide) { - return provide != null && SparkSource.FORMAT_NAMES().contains(provide.toLowerCase()); + if (provide == null) { + return false; + } + // compare against the enum names rather than a pre-lowered list, so the answer does not + // depend on the locale that was in effect when that list was built + for (FormatTable.Format format : FormatTable.Format.values()) { + if (format.name().equalsIgnoreCase(provide)) { + return true; + } + } + return false; } default Table toSparkFormatTable(Identifier ident, FormatTable formatTable) { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala index 0e7c1b8b6059..9767f4a57e88 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala @@ -39,6 +39,7 @@ import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap import java.util.{Map => JMap} +import java.util.Locale import scala.collection.JavaConverters._ @@ -130,7 +131,7 @@ object SparkSource { val NAME = "paimon" - val FORMAT_NAMES: Seq[String] = Format.values.map(_.toString.toLowerCase).toSeq + val FORMAT_NAMES: Seq[String] = Format.values.map(_.toString.toLowerCase(Locale.ROOT)).toSeq // Spark dataframe read options private val CATALOG = "catalog" diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala index 61426a52ec73..3d4ee948762f 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala @@ -33,6 +33,8 @@ import org.apache.spark.sql.connector.catalog.CatalogPlugin import org.apache.spark.sql.types.{BinaryType, DataType, DayTimeIntervalType, LongType, NullType, StringType} import org.apache.spark.unsafe.types.UTF8String +import java.util.Locale + case class PaimonFunctionResolver(spark: SparkSession) extends Rule[LogicalPlan] { protected lazy val catalogManager = spark.sessionState.catalogManager @@ -122,7 +124,7 @@ case class PaimonFunctionResolver(spark: SparkSession) extends Rule[LogicalPlan] case actual if actual == expectedType => expression case actual => throw new UnsupportedOperationException( - s"$argumentName must be ${expectedType.simpleString.toUpperCase} type, " + + s"$argumentName must be ${expectedType.simpleString.toUpperCase(Locale.ROOT)} type, " + s"but found ${actual.simpleString}") } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala index d81560ffe631..fe6b96e5b3c8 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala @@ -40,6 +40,8 @@ import org.apache.spark.sql.connector.catalog.PaimonCatalogImplicits._ import org.apache.spark.sql.types.{BinaryType, DataType, DayTimeIntervalType, NullType, StringType} import org.apache.spark.unsafe.types.UTF8String +import java.util.Locale + import scala.collection.JavaConverters._ object ReplacePaimonFunctions { @@ -269,7 +271,7 @@ case class ReplacePaimonFunctions(spark: SparkSession) extends Rule[LogicalPlan] case actual if actual == expectedType => expression case actual => throw new UnsupportedOperationException( - s"$argumentName must be ${expectedType.simpleString.toUpperCase} type, " + + s"$argumentName must be ${expectedType.simpleString.toUpperCase(Locale.ROOT)} type, " + s"but found ${actual.simpleString}") } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CopyOptions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CopyOptions.scala index 3e5e35b2b78d..71a988d61f56 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CopyOptions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CopyOptions.scala @@ -18,6 +18,8 @@ package org.apache.paimon.spark.catalyst.plans.logical +import java.util.Locale + sealed trait OnErrorMode object OnErrorMode { @@ -57,7 +59,7 @@ case class CopyFileFormat(formatType: FileFormatType, options: Map[String, Strin options.foreach { case (k, v) => k match { - case "MULTI_LINE" => mapped("multiLine") = v.toLowerCase + case "MULTI_LINE" => mapped("multiLine") = v.toLowerCase(Locale.ROOT) case "COMPRESSION" => mapped("compression") = v case _ => } @@ -84,7 +86,7 @@ case class CopyFileFormat(formatType: FileFormatType, options: Map[String, Strin case (k, v) => k match { case "FIELD_DELIMITER" => mapped("sep") = v - case "HEADER" => mapped("header") = v.toLowerCase + case "HEADER" => mapped("header") = v.toLowerCase(Locale.ROOT) case "QUOTE" => mapped("quote") = v case "ESCAPE" => mapped("escape") = v case "COMPRESSION" => mapped("compression") = v @@ -226,7 +228,7 @@ object CopyFileFormat { val LIST_SEPARATOR: String = "\u001f" def parseFormatType(typeStr: String): FileFormatType = { - typeStr.toUpperCase match { + typeStr.toUpperCase(Locale.ROOT) match { case "CSV" => FileFormatType.CSV case "JSON" => FileFormatType.JSON case "PARQUET" => FileFormatType.PARQUET diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/WriteHelper.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/WriteHelper.scala index 9de665c12b16..5393697a50d0 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/WriteHelper.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/WriteHelper.scala @@ -33,6 +33,8 @@ import org.apache.spark.sql.connector.metric.CustomTaskMetric import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.metric.SQLMetrics +import java.util.Locale + import scala.collection.JavaConverters._ trait WriteHelper extends Logging { @@ -52,7 +54,9 @@ trait WriteHelper extends Logging { val executionMetrics = Compatibility.getExecutionMetrics(spark, executionId.toLong).distinct val metricUpdates = executionMetrics.flatMap { m => - commitMetrics.find(x => m.metricType.toLowerCase.contains(x.name.toLowerCase)) match { + commitMetrics.find( + x => + m.metricType.toLowerCase(Locale.ROOT).contains(x.name.toLowerCase(Locale.ROOT))) match { case Some(customTaskMetric) => Some((m.accumulatorId, customTaskMetric.value())) case None => None } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala index f5986492b65e..a4aaf3ab69b9 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala @@ -34,6 +34,8 @@ import org.apache.spark.sql.catalyst.parser.extensions.PaimonSqlExtensionsParser import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.execution.command.{CreateTableLikeCommand => SparkCreateTableLikeCommand} +import java.util.Locale + import scala.collection.JavaConverters._ import scala.collection.mutable @@ -239,7 +241,7 @@ class PaimonSqlExtensionsAstBuilder(delegate: ParserInterface) opts.foreach { opt => - val key = opt.key.getText.toUpperCase + val key = opt.key.getText.toUpperCase(Locale.ROOT) if (!seen.add(key)) { throw new IllegalArgumentException(s"Duplicate FILE_FORMAT option: $key") } diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/catalog/FormatTableCatalogTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/catalog/FormatTableCatalogTest.java new file mode 100644 index 000000000000..574338989896 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/catalog/FormatTableCatalogTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalog; + +import org.apache.paimon.table.FormatTable; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * {@code MOSAIC} contains an 'I', so a provider name folded under a Turkish default locale no + * longer matches the enum constant it names. + */ +class FormatTableCatalogTest { + + private final FormatTableCatalog catalog = new FormatTableCatalog() {}; + + private Locale original; + + @BeforeEach + void setUp() { + original = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + } + + @AfterEach + void tearDown() { + Locale.setDefault(original); + } + + @Test + void everyFormatIsRecognizedInEitherCase() { + for (FormatTable.Format format : FormatTable.Format.values()) { + assertThat(catalog.isFormatTable(format.name())).as("%s", format).isTrue(); + assertThat(catalog.isFormatTable(format.name().toLowerCase(Locale.ROOT))) + .as("%s lower case", format) + .isTrue(); + } + } + + @Test + void aNameThatIsNotAFormatIsRejected() { + assertThat(catalog.isFormatTable(null)).isFalse(); + assertThat(catalog.isFormatTable("")).isFalse(); + assertThat(catalog.isFormatTable("paimon")).isFalse(); + assertThat(catalog.isFormatTable("orcish")).isFalse(); + } +} diff --git a/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java b/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java index 06f33273e289..5a4be336f5bb 100644 --- a/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java +++ b/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java @@ -329,7 +329,7 @@ private static int getNumNestedMetaInfDirectories(Path jar, Path jarRoot) throws } private static String getFileName(Path path) { - return path.getFileName().toString().toLowerCase(); + return path.getFileName().toString().toLowerCase(Locale.ROOT); } private static boolean pathStartsWith(Path file, String path) {