Skip to content

[python] Support reading shared-shredding maps #9763

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 5 commits into from
Sep 13, 2026
Merged

[python] Support reading shared-shredding maps #9763

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2006f5f
[python] Support reading shared-shredding maps
XiaoHongbo-Hope Sep 12, 2026
851f447
[python] Restore ORC time values in shared-shredding maps
XiaoHongbo-Hope Sep 13, 2026
d089659
[python] Restore ORC timestamp precision in shredded maps
XiaoHongbo-Hope Sep 13, 2026
2d33d38
[python] Resolve shared map metadata through source paths
XiaoHongbo-Hope Sep 13, 2026
ee9898c
python: add shared-shredding map interoperability test
XiaoHongbo-Hope Sep 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1567,6 +1567,59 @@ public void testJavaReadMapBlobTable() throws Exception {
assertAdditionalMapBlobKeyTypes(table, "python");
}

/** Java writes shared-shredding MAP columns for Python to read. */
@Test
@EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
public void testJavaWriteSharedShreddingMapTable() throws Exception {
for (String format : Arrays.asList("parquet", "orc")) {
Identifier identifier = identifier("shared_shredding_map_java_test_" + format);
catalog.dropTable(identifier, true);
Schema schema =
Schema.newBuilder()
.column("id", DataTypes.INT())
.column(
"metrics",
DataTypes.MAP(DataTypes.STRING().notNull(), DataTypes.BIGINT()))
.option(BUCKET.key(), "-1")
.option(CoreOptions.FILE_FORMAT.key(), format)
.option(CoreOptions.WRITE_ONLY.key(), "true")
.option("fields.metrics.map.storage-layout", "shared-shredding")
.option("fields.metrics.map.shared-shredding.max-columns", "2")
.build();
catalog.createTable(identifier, schema, false);

Map<Object, Object> first = new LinkedHashMap<>();
first.put(BinaryString.fromString("hot"), 10L);
first.put(BinaryString.fromString("warm"), 20L);
first.put(BinaryString.fromString("overflow"), 30L);
Map<Object, Object> second = new LinkedHashMap<>();
second.put(BinaryString.fromString("hot"), null);
second.put(BinaryString.fromString("new"), 40L);

FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
try (BatchTableWrite write = writeBuilder.newWrite();
BatchTableCommit commit = writeBuilder.newCommit()) {
write.write(GenericRow.of(1, new GenericMap(first)));
write.write(GenericRow.of(2, new GenericMap(second)));
write.write(GenericRow.of(3, new GenericMap(Collections.emptyMap())));
write.write(GenericRow.of(4, null));
commit.commit(write.prepareCommit());
}

Map<Object, Object> later = new LinkedHashMap<>();
later.put(BinaryString.fromString("late"), 50L);
later.put(BinaryString.fromString("hot"), 60L);
table = (FileStoreTable) catalog.getTable(identifier);
writeBuilder = table.newBatchWriteBuilder();
try (BatchTableWrite write = writeBuilder.newWrite();
BatchTableCommit commit = writeBuilder.newCommit()) {
write.write(GenericRow.of(5, new GenericMap(later)));
commit.commit(write.prepareCommit());
}
}
}

private Map<Integer, Map<Integer, byte[]>> readMapBlobRows(FileStoreTable table)
throws Exception {
Map<Integer, Map<Integer, byte[]>> rows = new HashMap<>();
Expand Down
38 changes: 37 additions & 1 deletion paimon-python/dev/run_mixed_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ run_batched_java_write_tests() {
core_tests="${core_tests}+testBlobWriteAlterCompact"
core_tests="${core_tests}+testJavaWriteArrayBlobTable"
core_tests="${core_tests}+testJavaWriteMapBlobTable"
core_tests="${core_tests}+testJavaWriteSharedShreddingMapTable"
core_tests="${core_tests}+testDataEvolutionWrite"
core_tests="${core_tests}+testJavaWriteRowAppendTable"
if [[ "$PYTHON_MINOR" -ge 7 ]]; then
Expand Down Expand Up @@ -1011,6 +1012,28 @@ run_map_blob_interop_test() {
echo -e "${GREEN}✓ Java MAP<K, BLOB> read test completed successfully${NC}"
}

run_shared_shredding_map_test() {
echo -e "${YELLOW}=== Running shared-shredding MAP Test (Java Write → Python Read) ===${NC}"

if ! skip_batched_java_write; then
cd "$PROJECT_ROOT"
echo "Running Maven test for JavaPyE2ETest.testJavaWriteSharedShreddingMapTable..."
if ! mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteSharedShreddingMapTable -pl paimon-core -q -Drun.e2e.tests=true; then
echo -e "${RED}✗ Java shared-shredding MAP write test failed${NC}"
return 1
fi
echo -e "${GREEN}✓ Java shared-shredding MAP write test completed successfully${NC}"
fi

cd "$PAIMON_PYTHON_DIR"
echo "Running Python shared-shredding MAP read test..."
if ! python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest::test_read_shared_shredding_map_written_by_java -v; then
echo -e "${RED}✗ Python shared-shredding MAP read test failed${NC}"
return 1
fi
echo -e "${GREEN}✓ Python shared-shredding MAP read test completed successfully${NC}"
}

# Function to run VARIANT test (Java write, Python read)
run_java_variant_write_py_read_test() {
echo -e "${YELLOW}=== Running VARIANT Test (Java Write, Python Read) ===${NC}"
Expand Down Expand Up @@ -1129,6 +1152,7 @@ main() {
local blob_alter_compact_result=0
local array_blob_interop_result=0
local map_blob_interop_result=0
local shared_shredding_map_result=0
local data_evolution_result=0
local data_evolution_deletion_vector_result=0
local data_evolution_py_write_result=0
Expand Down Expand Up @@ -1373,6 +1397,12 @@ main() {

echo ""

if ! run_shared_shredding_map_test; then
shared_shredding_map_result=1
fi

echo ""

# Run data evolution test (Java write, Python read). Lance variant skips
# itself on <3.8 (get_file_format_params + gated Java lance read).
if ! run_data_evolution_test; then
Expand Down Expand Up @@ -1573,6 +1603,12 @@ main() {
echo -e "${RED}✗ MAP<K, BLOB> Interoperability Test (Java ↔ Python): FAILED${NC}"
fi

if [[ $shared_shredding_map_result -eq 0 ]]; then
echo -e "${GREEN}✓ Shared-shredding MAP Test (Java Write → Python Read): PASSED${NC}"
else
echo -e "${RED}✗ Shared-shredding MAP Test (Java Write → Python Read): FAILED${NC}"
fi

if [[ $data_evolution_result -eq 0 ]]; then
echo -e "${GREEN}✓ Data Evolution Test (Java Write, Python Read): PASSED${NC}"
else
Expand Down Expand Up @@ -1614,7 +1650,7 @@ main() {
# Clean up warehouse directory after all tests
cleanup_warehouse

if [[ $java_write_result -eq 0 && $python_read_result -eq 0 && $python_write_result -eq 0 && $java_read_result -eq 0 && $pk_dv_result -eq 0 && $btree_index_result -eq 0 && $btree_raw_fallback_result -eq 0 && $bitmap_index_result -eq 0 && $compressed_global_index_result -eq 0 && $compressed_text_result -eq 0 && $native_fulltext_result -eq 0 && $lumina_vector_result -eq 0 && $lumina_vector_btree_result -eq 0 && $vindex_vector_result -eq 0 && $vindex_vector_raw_fallback_result -eq 0 && $compact_conflict_result -eq 0 && $blob_compact_conflict_result -eq 0 && $blob_alter_compact_result -eq 0 && $array_blob_interop_result -eq 0 && $map_blob_interop_result -eq 0 && $data_evolution_result -eq 0 && $data_evolution_deletion_vector_result -eq 0 && $data_evolution_py_write_result -eq 0 && $java_variant_write_py_read_result -eq 0 && $py_variant_write_java_read_result -eq 0 && $vector_append_table_result -eq 0 && $vector_dedicated_java_write_result -eq 0 && $vector_dedicated_py_write_result -eq 0 && $multi_vector_dedicated_java_write_result -eq 0 && $multi_vector_dedicated_py_write_result -eq 0 && $row_format_result -eq 0 ]]; then
if [[ $java_write_result -eq 0 && $python_read_result -eq 0 && $python_write_result -eq 0 && $java_read_result -eq 0 && $pk_dv_result -eq 0 && $btree_index_result -eq 0 && $btree_raw_fallback_result -eq 0 && $bitmap_index_result -eq 0 && $compressed_global_index_result -eq 0 && $compressed_text_result -eq 0 && $native_fulltext_result -eq 0 && $lumina_vector_result -eq 0 && $lumina_vector_btree_result -eq 0 && $vindex_vector_result -eq 0 && $vindex_vector_raw_fallback_result -eq 0 && $compact_conflict_result -eq 0 && $blob_compact_conflict_result -eq 0 && $blob_alter_compact_result -eq 0 && $array_blob_interop_result -eq 0 && $map_blob_interop_result -eq 0 && $shared_shredding_map_result -eq 0 && $data_evolution_result -eq 0 && $data_evolution_deletion_vector_result -eq 0 && $data_evolution_py_write_result -eq 0 && $java_variant_write_py_read_result -eq 0 && $py_variant_write_java_read_result -eq 0 && $vector_append_table_result -eq 0 && $vector_dedicated_java_write_result -eq 0 && $vector_dedicated_py_write_result -eq 0 && $multi_vector_dedicated_java_write_result -eq 0 && $multi_vector_dedicated_py_write_result -eq 0 && $row_format_result -eq 0 ]]; then
echo -e "${GREEN}🎉 All tests passed! Java-Python interoperability verified.${NC}"
return 0
else
Expand Down
Loading
Loading