diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index 6ad271bb5a89..4f192192d646 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -63,6 +63,125 @@ skip manifests before opening them. Each extra file belongs exclusively to one manifest. It is retained and cleaned up together with that manifest during snapshot, tag, or changelog deletion. +### Manifest Sidecar + +With `manifest.sidecar.write` enabled, a manifest writer can create a binary +`.avro.sidecar` sidecar. Its name is stored in the manifest-list +record's `_EXTRA_FILES`; the existing Avro schemas and `_VERSION` identifiers are unchanged. +Readers identify the sidecar by the `.avro.sidecar` suffix among these explicit +references, not by probing for a derived file name. Other extra-file references are preserved. + +With `manifest.sidecar.read` enabled and a partition, row-ID or bucket filter available, readers can use +the sidecar to select complete Avro blocks before reading manifest entries. Both options +default to `false`. Old manifests, null or empty extra-file lists, and lists containing only +other extra-file types use the normal manifest read path. Missing, unsupported, corrupt, +or over-budget containers also fall back to that path. Each block's partition, row-ID and bucket +coverage is independently usable; an unavailable dimension cannot exclude a block. +Cancellation and interruption errors propagate instead of triggering a full-manifest fallback. + +Version 1 uses the following layout. Container integers and payload integers +are fixed-width big endian. Encoding IDs are unsigned bytes with separate namespaces. + +```text +magic : 8 bytes // ASCII PAIMSCAR +formatVersion : int // 1 +manifestNameHash : 32 bytes // SHA-256 of the UTF-8 basename +manifestLength : long +manifestEntryCount : long // ADD + DELETE +avroHeaderLength : int +avroHeader : bytes // original schema, codec and sync marker +partitionCount : int +partitionDictionary[] + partitionByteLength : int + partitionBytes : bytes // existing manifest BinaryRow serialization +blockCount : int +blocks[] // original physical order + offset : long + length : long // complete encoded block, including sync marker + recordCount : long + partitionEncoding : byte + partitionPayloadLength : int + partitionPayload : bytes + rowIdEncoding : byte + rowIdPayloadLength : int + rowIdPayload : bytes + bucketEncoding : byte + bucketPayloadLength : int + bucketPayload : bytes +checksum : 32 bytes // SHA-256 of all preceding bytes +``` + +The block ID is its position. Its first entry ordinal is the sum of preceding record +counts and is not stored. Each complete partition tuple appears once in the dictionary, +including all its fields and nulls. The scan's partition type interprets the existing +serialized tuple. Partition predicates are evaluated once per dictionary entry. + +| Dimension | Encoding | Payload | +| --- | --- | --- | +| Any | `0` | Unavailable; payload length must be zero. | +| Partition | `1` | Positive `partitionIdCount: int` followed by sorted unique dictionary IDs (`int`). | +| Row ID | `1` | Positive `rangeCount: int` followed by sorted disjoint inclusive `(start: long, end: long)` pairs. Coverage may conservatively include gaps. | +| Bucket | `1` | Positive `pairCount: int` followed by sorted unique `(bucket: int, totalBuckets: int)` pairs. | +| Any | Other nonzero ID | Skip exactly the bounded payload length; treat only this dimension as unavailable. | + +Payload lengths exclude the encoding and length fields. Invalid lengths, known-payload +framing, dictionary references, interval order, checksums or physical coverage invalidate +the container. Byte spans must cover the entire original manifest after its header; +record counts must sum to the manifest entry count. Readers continue validating blocks +and known payloads even when a predicate has already rejected a block. + +Bucket encoding 1 contains a positive `pairCount: int` followed by that many +`(bucket: int, totalBuckets: int)` pairs. Pairs are sorted by bucket, then totalBuckets, +and deduplicated. They preserve bucket-count changes between writes; the bucket number +alone is not sufficient for point lookup after rescaling. A valid pair satisfies +`0 <= bucket < totalBuckets`. Missing, invalid, negative/synthetic or over-budget bucket +metadata makes that block's bucket coverage unavailable (encoding 0, length 0). Partition +and row-ID coverage remain independently usable; no mutual-exclusion restriction is imposed. + +Readers test bucket-only queries using the existing bucket-selection logic, including +the total-bucket count. Java uses conservative partition-independent bounds for +`ManifestBucketFilter`; arbitrary partition-dependent callbacks remain at the entry +filter stage. An unavailable bucket payload cannot exclude a block. Malformed payload lengths, +pair counts, ordering or values invalidate the container rather than excluding a block. + +All entries contribute, including ADD, DELETE and every file format/column group. +Row-ID ranges are never expanded into individual values. If an exact union exceeds its +available byte budget, it becomes the inclusive `[min,max]` envelope with encoding 1. Processing +continues through the end of the block to extend those bounds and detect unknown row IDs. +An unknown or invalid row-ID range makes only that block's row-ID payload unavailable. +Partition budget exhaustion independently makes that block's partition payload unavailable. +The dictionary can consequently be incomplete for the manifest: a dictionary miss never +excludes a block with unavailable partition coverage. Later blocks can still use existing IDs. + +`manifest.sidecar.max-bytes` bounds the whole serialized container, including the +partition dictionary and all three payload types. It accepts memory sizes such as +`16 mb` and, when unset, defaults to twice the configured `manifest.target-file-size` +(16 MiB with the default 8 MiB manifest target). An explicit sidecar size overrides this default. +The Avro header is also capped at 1 MiB and the directory at 131072 blocks. +Writers discard optional row-ID payloads, bucket payloads, then partition payloads/dictionary if necessary, +to fit the complete directory. If the directory itself cannot fit, no sidecar is published. +No emitted sidecar omits block descriptors. These are encoded-size bounds; construction +also incurs bounded object/buffer overhead. Query concurrency multiplies per-reader costs. + +For conjunctive filters a block is retained only if each dimension is either unavailable +or matches. Matching skips absent filters and short-circuits after a dimension rejects a +block, while known payloads remain validated. Matches in different dimensions can come +from different entries in the block, so entry filtering and deletion merging remain +necessary. Block min/max is derived from the first/last interval before testing the +individual intervals. + +Readers still consume and validate the whole bounded sidecar. A partition-only query +therefore reads row-ID payload bytes too; payload lengths save decoding work for unknown +encodings, not storage I/O. Selected compressed blocks are read by byte range with adjacent +spans coalesced. Existing immutable manifests are not backfilled by enabling the write option. + +Java selections covering every block can reuse the full-manifest cache; partial selections +bypass it. PyPaimon explain scans disable sidecar pruning to preserve complete entry counters. + +Selected blocks still pass through entry filtering and ADD/DELETE reconciliation. Snapshot, +tag, changelog, orphan-file and failed-commit cleanup retain or remove the sidecar through +its extra-file reference together with the owning manifest. + ## Manifest Data manifests record **ADD** (`0`) and **DELETE** (`1`) entries. Readers reconcile these entries diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 7b0665c50296..12ef2cdd03a2 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -510,6 +510,27 @@ public InlineElement getDescription() { + "in the previous file. This must not exceed " + "'variant.shredding.minFieldCardinalityRatio'."); + public static final ConfigOption MANIFEST_SIDECAR_WRITE = + key("manifest.sidecar.write") + .booleanType() + .defaultValue(false) + .withDescription( + "Write sidecars with independent partition, row-id and bucket coverage for newly created manifests."); + + public static final ConfigOption MANIFEST_SIDECAR_READ = + key("manifest.sidecar.read") + .booleanType() + .defaultValue(false) + .withDescription( + "Read optional manifest sidecars for partition, row-id or bucket filters after coarse pruning. Missing or invalid sidecars fall back to manifest reads."); + + public static final ConfigOption MANIFEST_SIDECAR_MAX_BYTES = + key("manifest.sidecar.max-bytes") + .memoryType() + .noDefaultValue() + .withDescription( + "Maximum serialized manifest sidecar size, including header and checksum. Defaults to twice manifest.target-file-size. Optional payloads are dropped before omitting a sidecar whose complete block directory cannot fit."); + public static final ConfigOption MANIFEST_COMPRESSION = key("manifest.compression") .stringType() @@ -3210,6 +3231,11 @@ public MemorySize manifestTargetSize() { return options.get(MANIFEST_TARGET_FILE_SIZE); } + public MemorySize manifestSidecarMaxSize() { + return options.getOptional(MANIFEST_SIDECAR_MAX_BYTES) + .orElseGet(() -> manifestTargetSize().multiply(2)); + } + public MemorySize manifestFullCompactionThresholdSize() { return options.get(MANIFEST_FULL_COMPACTION_FILE_SIZE); } diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java index 7399e057783c..41ded1344874 100644 --- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java @@ -204,14 +204,15 @@ public ChangelogManager changelogManager() { @Override public ManifestFile.Factory manifestFileFactory() { return new ManifestFile.Factory( - fileIO, - schemaManager, - partitionType, - FileFormat.manifestFormat(options), - options.manifestCompression(), - pathFactory(), - options.manifestTargetSize().getBytes(), - readManifestCache); + fileIO, + schemaManager, + partitionType, + FileFormat.manifestFormat(options), + options.manifestCompression(), + pathFactory(), + options.manifestTargetSize().getBytes(), + readManifestCache) + .withSidecarOptions(options.toConfiguration()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java b/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java index 536be456eef0..3e564a60f9e1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java @@ -78,6 +78,22 @@ public boolean test(BinaryRow partition, int bucket, int totalBucket) { || totalAwareBucketFilter.test(partition, bucket, totalBucket); } + /** Conservatively checks an indexed pair without inventing a partition for custom filters. */ + public boolean mayContain(int bucket, int totalBuckets) { + if (onlyReadRealBuckets && bucket < 0) { + return false; + } + if (specifiedBucket != null && bucket != specifiedBucket) { + return false; + } + if (bucketFilter != null && !bucketFilter.test(bucket)) { + return false; + } + return !(totalAwareBucketFilter instanceof ManifestBucketFilter) + || ((ManifestBucketFilter) totalAwareBucketFilter) + .mayContain(bucket, bucket, totalBuckets); + } + /** Conservatively tests whether a manifest's bucket metadata can contain a matching entry. */ public boolean mayContain(ManifestFileMeta manifest) { Integer minBucket = manifest.minBucket(); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java index b1863358779e..79500c97a60a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java @@ -70,6 +70,19 @@ public final class ManifestAvroReader implements AutoCloseable { } } + @Nullable + public byte[] headerBytes() { + return blockReader.headerBytes(); + } + + public long blockOffset() { + return blockReader.blockOffset(); + } + + public long blockLength() { + return blockReader.blockLength(); + } + /** Returns whether another raw Avro block is available. */ public boolean hasNext() throws IOException { return blockReader.hasNextBlock(); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java index 4dabe54f3f2b..182989326a5d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java @@ -43,6 +43,7 @@ import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; @@ -68,6 +69,7 @@ public final class ManifestAvroWriter implements AutoCloseable { private final String compression; private final PathFactory pathFactory; private final long targetFileSize; + private final ManifestSidecar.Settings sidecarSettings; private final List results = new ArrayList<>(); private final List completedPaths = new ArrayList<>(); @@ -83,7 +85,8 @@ public final class ManifestAvroWriter implements AutoCloseable { ObjectSerializer serializer, String compression, PathFactory pathFactory, - long targetFileSize) { + long targetFileSize, + ManifestSidecar.Settings sidecarSettings) { this.fileIO = fileIO; this.schemaManager = schemaManager; this.partitionType = partitionType; @@ -92,6 +95,7 @@ public final class ManifestAvroWriter implements AutoCloseable { this.compression = compression; this.pathFactory = pathFactory; this.targetFileSize = targetFileSize; + this.sidecarSettings = sidecarSettings; } public void write(ManifestEntry entry) throws IOException { @@ -218,6 +222,9 @@ private void closeCurrentWriter() throws IOException { currentWriter.close(); ManifestFileMeta result = currentWriter.result(); completedPaths.add(currentWriter.path); + if (currentWriter.sidecarCreated) { + completedPaths.add(ManifestSidecar.path(currentWriter.path)); + } results.add(result); currentWriter = null; } @@ -413,6 +420,7 @@ private final class FileWriter { private @Nullable RowIdStats rowIdStats = new RowIdStats(); private boolean closed; private boolean aborted; + private boolean sidecarCreated; private FileWriter(Path path) { this.path = path; @@ -488,7 +496,7 @@ private void collectStats(ManifestEntry entry) { maxLevel = Math.max(maxLevel, entry.level()); if (rowIdStats != null) { Long firstRowId = entry.file().firstRowId(); - if (firstRowId == null) { + if (!validRowIdRange(firstRowId, entry.file().rowCount())) { rowIdStats = null; } else { rowIdStats.collect(firstRowId, entry.file().rowCount()); @@ -515,7 +523,7 @@ private void collectStats(EncodedEntry entry) { minLevel = Math.min(minLevel, entry.level); maxLevel = Math.max(maxLevel, entry.level); if (rowIdStats != null) { - if (!entry.hasRowId) { + if (!entry.hasRowId || !validRowIdRange(entry.firstRowId, entry.rowCount)) { rowIdStats = null; } else { rowIdStats.collect(entry.firstRowId, entry.rowCount); @@ -697,6 +705,14 @@ private Throwable abortCollecting(@Nullable Throwable primaryFailure, boolean de ExceptionUtils.firstOrSuppressed(cleanupFailure, primaryFailure); } } + if (sidecarCreated) { + try { + fileIO.deleteQuietly(ManifestSidecar.path(path)); + } catch (Throwable cleanupFailure) { + primaryFailure = + ExceptionUtils.firstOrSuppressed(cleanupFailure, primaryFailure); + } + } return primaryFailure; } @@ -711,6 +727,7 @@ private void close() throws IOException { outputBytes = out.getPos(); out.close(); out = null; + writeSidecar(); } catch (IOException | RuntimeException | Error failure) { abortCollecting(failure, true); throw failure; @@ -719,6 +736,27 @@ private void close() throws IOException { } } + private void writeSidecar() throws IOException { + if (!sidecarSettings.write) { + return; + } + byte[] bytes = + ManifestSidecar.build( + fileIO, + path, + outputBytes, + Math.addExact(numAddedFiles, numDeletedFiles), + sidecarSettings); + if (bytes != null) { + // Publish result() only after both immutable objects have closed. No rename. + try (PositionOutputStream sidecarOut = + fileIO.newOutputStream(ManifestSidecar.path(path), false)) { + sidecarCreated = true; + sidecarOut.write(bytes); + } + } + } + private ManifestFileMeta result() { if (!closed || outputBytes == null) { throw new IllegalStateException( @@ -740,10 +778,16 @@ private ManifestFileMeta result() { rowIdStats == null ? null : rowIdStats.minRowId, rowIdStats == null ? null : rowIdStats.maxRowId, totalBucketsKnown ? totalBuckets : null, - null); + sidecarCreated + ? Collections.singletonList(ManifestSidecar.path(path).getName()) + : null); } } + private static boolean validRowIdRange(@Nullable Long first, long count) { + return first != null && first >= 0 && count > 0 && count - 1 <= Long.MAX_VALUE - first; + } + private static class RowIdStats { private long minRowId = Long.MAX_VALUE; diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 0dc99a047076..cd96bd611ab1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -18,6 +18,7 @@ package org.apache.paimon.manifest; +import org.apache.paimon.CoreOptions; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.FileFormat; @@ -27,6 +28,7 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.ProjectedManifestEntry.Projection; import org.apache.paimon.operation.metrics.CacheMetrics; +import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.types.RowType; @@ -36,6 +38,7 @@ import org.apache.paimon.utils.Filter; import org.apache.paimon.utils.ObjectsFile; import org.apache.paimon.utils.PathFactory; +import org.apache.paimon.utils.RowRangeIndex; import org.apache.paimon.utils.SegmentsCache; import javax.annotation.Nullable; @@ -59,6 +62,7 @@ public class ManifestFile extends ObjectsFile { private final RowType partitionType; private final AvroFileFormat avroFileFormat; private final long suggestedFileSize; + private final ManifestSidecar.Settings sidecarSettings; private ManifestFile( FileIO fileIO, @@ -69,7 +73,8 @@ private ManifestFile( String compression, PathFactory pathFactory, long suggestedFileSize, - @Nullable SegmentsCache cache) { + @Nullable SegmentsCache cache, + ManifestSidecar.Settings sidecarSettings) { super( fileIO, serializer, @@ -85,6 +90,7 @@ private ManifestFile( this.partitionType = partitionType; this.avroFileFormat = avroFileFormat; this.suggestedFileSize = suggestedFileSize; + this.sidecarSettings = sidecarSettings; } @Override @@ -136,9 +142,33 @@ public List read( Filter readFilter, Filter readTFilter, Function convertor) { + return read( + fileName, + fileSize, + partitionFilter, + bucketFilter, + readFilter, + readTFilter, + convertor, + null); + } + + public List read( + String fileName, + @Nullable Long fileSize, + @Nullable PartitionPredicate partitionFilter, + @Nullable BucketFilter bucketFilter, + Filter readFilter, + Filter readTFilter, + Function convertor, + @Nullable ManifestSidecar.Selection selected) { + if (selected != null && selected.blocks().isEmpty()) { + return java.util.Collections.emptyList(); + } try { Path path = pathFactory.toPath(fileName); - if (cache != null) { + // A partial manifest must never enter the cache under the full manifest's key. + if (cache != null && (selected == null || selected.isFullManifest())) { ManifestEntryFilters filters = new ManifestEntryFilters( partitionFilter, bucketFilter, readFilter, readTFilter); @@ -151,7 +181,8 @@ public List read( path, ManifestEntry.MANIFEST_ROW_TYPE, partitionFilter, - bucketFilter); + bucketFilter, + selected); return readFromIterator(iterator, serializer, readFilter, readTFilter, convertor); } catch (IOException e) { throw new UncheckedIOException(e); @@ -205,8 +236,21 @@ private static CloseableIterator createManifestIterator( @Nullable PartitionPredicate partitionFilter, @Nullable BucketFilter bucketFilter) throws IOException { + return createManifestIterator( + fileIO, path, projectedType, partitionFilter, bucketFilter, null); + } + + private static CloseableIterator createManifestIterator( + FileIO fileIO, + Path path, + RowType projectedType, + @Nullable PartitionPredicate partitionFilter, + @Nullable BucketFilter bucketFilter, + @Nullable ManifestSidecar.Selection selected) + throws IOException { try { - ManifestAvroReader reader = new ManifestAvroReader(fileIO.newInputStream(path)); + ManifestAvroReader reader = + new ManifestAvroReader(ManifestSidecar.openManifest(fileIO, path, selected)); return reader.read(projectedType, partitionFilter, bucketFilter); } catch (IOException e) { FileUtils.checkExists(fileIO, path); @@ -301,7 +345,8 @@ public ManifestAvroWriter createAvroWriter() { serializer, compression, pathFactory, - suggestedFileSize); + suggestedFileSize, + sidecarSettings); } /** Creates an Avro manifest writer for one explicit path. */ @@ -314,7 +359,8 @@ public ManifestAvroWriter createAvroWriter(Path manifestPath) { serializer, compression, singlePathFactory(manifestPath), - Long.MAX_VALUE); + Long.MAX_VALUE, + sidecarSettings); } private PathFactory singlePathFactory(Path manifestPath) { @@ -339,6 +385,45 @@ public Path toPath(String fileName) { }; } + @Nullable + public ManifestSidecar.Selection selectBlocks( + ManifestFileMeta manifest, @Nullable RowRangeIndex query) { + return selectBlocks(manifest, query, null, null); + } + + @Nullable + public ManifestSidecar.Selection selectBlocks( + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter, + @Nullable BucketFilter bucketFilter) { + return !sidecarSettings.read + || (query == null && partitionFilter == null && bucketFilter == null) + ? null + : ManifestSidecar.read( + fileIO, + pathFactory.toPath(manifest.fileName()), + manifest, + query, + partitionFilter, + partitionType, + bucketFilter, + sidecarSettings); + } + + public boolean mayContainRowIds(ManifestFileMeta manifest, @Nullable RowRangeIndex query) { + ManifestSidecar.Selection selected = selectBlocks(manifest, query); + return selected == null || !selected.blocks().isEmpty(); + } + + /** Deletes an unreferenced manifest and its explicitly referenced extra files. */ + public void delete(ManifestFileMeta manifest) { + delete(manifest.fileName()); + if (manifest.extraFiles() != null) { + manifest.extraFiles().forEach(this::delete); + } + } + /** Creator of {@link ManifestFile}. */ public static class Factory { @@ -349,6 +434,8 @@ public static class Factory { private final String compression; private final FileStorePathFactory pathFactory; private final long suggestedFileSize; + private ManifestSidecar.Settings sidecarSettings = + new ManifestSidecar.Settings(new Options()); @Nullable private final SegmentsCache cache; public Factory( @@ -370,6 +457,17 @@ public Factory( this.cache = cache; } + public Factory withSidecarOptions(Options options) { + // Disabled sidecars must not constrain the manifest target size. + sidecarSettings = + new ManifestSidecar.Settings( + options.get(CoreOptions.MANIFEST_SIDECAR_READ) + || options.get(CoreOptions.MANIFEST_SIDECAR_WRITE) + ? options + : new Options()); + return this; + } + public boolean isCacheEnabled() { return cache != null; } @@ -384,7 +482,8 @@ public ManifestFile create() { compression, pathFactory.manifestFileFactory(), suggestedFileSize, - cache); + cache, + sidecarSettings); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java index 30f74a305ade..b00135c8edfa 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java @@ -96,7 +96,9 @@ private ManifestFileMeta fromDataRow(InternalRow row) { row.isNullAt(9) ? null : row.getInt(9), row.isNullAt(10) ? null : row.getLong(10), row.isNullAt(11) ? null : row.getLong(11), - row.isNullAt(12) ? null : row.getInt(12), - row.isNullAt(13) ? null : fromStringArrayData(row.getArray(13))); + row.getFieldCount() <= 12 || row.isNullAt(12) ? null : row.getInt(12), + row.getFieldCount() <= 13 || row.isNullAt(13) + ? null + : fromStringArrayData(row.getArray(13))); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java new file mode 100644 index 000000000000..0fe56837d350 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -0,0 +1,886 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.options.Options; +import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.RowRangeIndex; +import org.apache.paimon.utils.SerializationUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.UncheckedIOException; +import java.net.SocketTimeoutException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedByInterruptException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.CancellationException; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Independently usable partition, row-id and bucket coverage for each manifest block. */ +public final class ManifestSidecar { + public static final String SUFFIX = ".avro.sidecar"; + private static final Logger LOG = LoggerFactory.getLogger(ManifestSidecar.class); + private static final long MAGIC = 0x5041494d53434152L; + private static final int FORMAT_VERSION = 1; + private static final int HEADER_BYTES = 60; + private static final int BLOCK_BYTES = 39; + private static final int MAX_BLOCKS = 131072; + private static final byte[] EMPTY = new byte[0]; + private static final int DIGEST_BYTES = 32; + private static final int MAX_AVRO_HEADER = 1024 * 1024; + private static final int READ_BUFFER_BYTES = 1024 * 1024; + + private ManifestSidecar() {} + + public static Path path(Path manifest) { + return new Path(manifest.toString() + SUFFIX); + } + + @Nullable + public static String fileName(ManifestFileMeta manifest) { + if (manifest.extraFiles() != null) { + for (String extraFile : manifest.extraFiles()) { + if (extraFile.endsWith(SUFFIX)) { + return extraFile; + } + } + } + return null; + } + + /** Independent read/write switches and construction/serialization bounds. */ + public static final class Settings { + public final boolean write; + public final boolean read; + public final int maxBytes; + + public Settings(Options options) { + write = options.get(CoreOptions.MANIFEST_SIDECAR_WRITE); + read = options.get(CoreOptions.MANIFEST_SIDECAR_READ); + long bytes = new CoreOptions(options).manifestSidecarMaxSize().getBytes(); + // Keep room for the extra byte used to detect an over-budget input. + checkArgument( + bytes >= 128 && bytes < Integer.MAX_VALUE, + "manifest.sidecar.max-bytes must be in [128, 2147483646] bytes"); + maxBytes = (int) bytes; + } + } + + /** Original file offset/length and zero-based manifest entry ordinal, not table row id. */ + public static final class Block { + public final long offset; + public final long length; + public final long firstRecord; + public final long recordCount; + + public Block(long offset, long length, long firstRecord, long recordCount) { + this.offset = offset; + this.length = length; + this.firstRecord = firstRecord; + this.recordCount = recordCount; + } + } + + /** Selected blocks in original file order. Empty means the manifest can be excluded. */ + public static final class Selection { + private final byte[] header; + private final List blocks; + private final boolean fullManifest; + + private Selection(byte[] header, List blocks, boolean fullManifest) { + this.header = header; + this.blocks = Collections.unmodifiableList(blocks); + this.fullManifest = fullManifest; + } + + public List blocks() { + return blocks; + } + + boolean isFullManifest() { + return fullManifest; + } + } + + /** + * Builds complete block descriptors even when either optional dimension becomes unavailable. + */ + public static final class Builder { + private final Settings settings; + private final byte[] header; + private final TreeMap ranges = new TreeMap<>(); + private final Map dictionary = new LinkedHashMap<>(); + private final TreeSet partitionIds = new TreeSet<>(); + private final TreeSet bucketPairs = new TreeSet<>(); + private final List blocks = new ArrayList<>(); + private boolean complete; + private long nextOffset; + private long nextRecord; + private Block current; + private long entriesInBlock; + private boolean rowAvailable; + private boolean partitionAvailable; + private boolean bucketAvailable; + private boolean coarse; + private long min; + private long max; + private int dictionaryBytes; + private int optionalBytes; + + public Builder(Settings settings, @Nullable byte[] header) { + this.settings = settings; + this.header = header; + complete = + header != null + && header.length <= MAX_AVRO_HEADER + && HEADER_BYTES + DIGEST_BYTES + 12L + header.length + <= settings.maxBytes; + nextOffset = header == null ? 0 : header.length; + } + + public boolean complete() { + return complete; + } + + public void beginBlock(long offset, long length, long records) throws IOException { + if (!complete) { + return; + } + require(current == null && offset == nextOffset && length > 0 && records > 0); + // Optional payloads can be discarded later, but descriptors must never be truncated. + if (blocks.size() == MAX_BLOCKS + || HEADER_BYTES + + DIGEST_BYTES + + 12L + + header.length + + (blocks.size() + 1L) * BLOCK_BYTES + > settings.maxBytes) { + complete = false; + blocks.clear(); + dictionary.clear(); + return; + } + current = new Block(offset, length, nextRecord, records); + entriesInBlock = 0; + rowAvailable = true; + partitionAvailable = true; + bucketAvailable = true; + coarse = false; + min = Long.MAX_VALUE; + max = -1; + ranges.clear(); + partitionIds.clear(); + bucketPairs.clear(); + } + + public void add(@Nullable Long first, long count) { + add(first, count, null); + } + + public void add(@Nullable Long first, long count, @Nullable byte[] partition) { + add(first, count, partition, null, null); + } + + public void add( + @Nullable Long first, + long count, + @Nullable byte[] partition, + @Nullable Integer bucket, + @Nullable Integer totalBuckets) { + if (!complete) { + return; + } + if (current == null) { + throw new IllegalStateException("No current Avro block"); + } + entriesInBlock++; + addPartition(partition); + addBucket(bucket, totalBuckets); + if (!rowAvailable) { + return; + } + if (first == null || first < 0 || count <= 0 || count - 1 > Long.MAX_VALUE - first) { + rowAvailable = false; + ranges.clear(); + return; + } + long end = first + (count - 1); + min = Math.min(min, first); + max = Math.max(max, end); + // Keep checking subsequent entries, including missing row IDs, after coarsening. + if (coarse) { + return; + } + long start = first; + Map.Entry before = ranges.floorEntry(start); + if (before != null && before.getValue() >= start - 1) { + start = before.getKey(); + end = Math.max(end, before.getValue()); + ranges.remove(before.getKey()); + } + Map.Entry next; + while ((next = ranges.ceilingEntry(start)) != null + && (next.getKey() <= end || next.getKey() - end == 1)) { + end = Math.max(end, next.getValue()); + ranges.remove(next.getKey()); + } + if (4L + 16L * (ranges.size() + 1L) > settings.maxBytes - optionalBytes) { + coarse = true; + ranges.clear(); + } else { + ranges.put(start, end); + } + } + + private void addBucket(@Nullable Integer bucket, @Nullable Integer totalBuckets) { + if (!bucketAvailable) { + return; + } + if (bucket == null + || totalBuckets == null + || bucket < 0 + || totalBuckets <= 0 + || bucket >= totalBuckets) { + bucketAvailable = false; + bucketPairs.clear(); + return; + } + long pair = ((long) bucket << 32) | totalBuckets; + if (!bucketPairs.contains(pair) + && 4L + 8L * (bucketPairs.size() + 1L) > settings.maxBytes - optionalBytes) { + bucketAvailable = false; + bucketPairs.clear(); + } else { + bucketPairs.add(pair); + } + } + + private void addPartition(@Nullable byte[] bytes) { + if (!partitionAvailable) { + return; + } + if (bytes == null) { + partitionAvailable = false; + partitionIds.clear(); + return; + } + Integer id = dictionary.get(ByteBuffer.wrap(bytes)); + if (id == null) { + if (bytes.length + 4L > settings.maxBytes - dictionaryBytes) { + partitionAvailable = false; + partitionIds.clear(); + return; + } + id = dictionary.size(); + dictionary.put(ByteBuffer.wrap(bytes.clone()), id); + dictionaryBytes += 4 + bytes.length; + } + partitionIds.add(id); + } + + public void endBlock() throws IOException { + if (!complete) { + return; + } + require(current != null && entriesInBlock == current.recordCount); + byte[] rowPayload = EMPTY; + byte[] partitionPayload = EMPTY; + if (rowAvailable) { + if (coarse || 4L + 16L * ranges.size() > settings.maxBytes - optionalBytes) { + ranges.clear(); + ranges.put(min, max); + } + if (4L + 16L * ranges.size() <= settings.maxBytes - optionalBytes) { + ByteBuffer out = ByteBuffer.allocate(4 + 16 * ranges.size()); + out.putInt(ranges.size()); + for (Map.Entry range : ranges.entrySet()) { + out.putLong(range.getKey()).putLong(range.getValue()); + } + rowPayload = out.array(); + optionalBytes += rowPayload.length; + } + } + if (partitionAvailable + && 4L + 4L * partitionIds.size() <= settings.maxBytes - optionalBytes) { + ByteBuffer out = ByteBuffer.allocate(4 + 4 * partitionIds.size()); + out.putInt(partitionIds.size()); + for (int id : partitionIds) { + out.putInt(id); + } + partitionPayload = out.array(); + optionalBytes += partitionPayload.length; + } + byte[] bucketPayload = EMPTY; + if (bucketAvailable + && 4L + 8L * bucketPairs.size() <= settings.maxBytes - optionalBytes) { + ByteBuffer out = ByteBuffer.allocate(4 + 8 * bucketPairs.size()); + out.putInt(bucketPairs.size()); + for (long pair : bucketPairs) { + out.putInt((int) (pair >>> 32)).putInt((int) pair); + } + bucketPayload = out.array(); + optionalBytes += bucketPayload.length; + } + blocks.add(new IndexedBlock(current, partitionPayload, rowPayload, bucketPayload)); + nextOffset = Math.addExact(current.offset, current.length); + nextRecord = Math.addExact(current.firstRecord, current.recordCount); + ranges.clear(); + partitionIds.clear(); + bucketPairs.clear(); + current = null; + } + + @Nullable + public byte[] serialize(String name, long fileSize, long entryCount) throws IOException { + if (!complete) { + return null; + } + require(current == null && nextOffset == fileSize && nextRecord == entryCount); + long size = + HEADER_BYTES + + DIGEST_BYTES + + 12L + + header.length + + dictionaryBytes + + blocks.size() * (long) BLOCK_BYTES + + optionalBytes; + // Give directory growth priority over optional coverage. Never remove a descriptor. + for (IndexedBlock block : blocks) { + if (size <= settings.maxBytes) { + break; + } + size -= block.rowIds.length; + optionalBytes -= block.rowIds.length; + block.rowIds = EMPTY; + } + for (IndexedBlock block : blocks) { + if (size <= settings.maxBytes) { + break; + } + size -= block.buckets.length; + optionalBytes -= block.buckets.length; + block.buckets = EMPTY; + } + if (size > settings.maxBytes) { + size -= dictionaryBytes; + dictionaryBytes = 0; + dictionary.clear(); + for (IndexedBlock block : blocks) { + size -= block.partitions.length; + optionalBytes -= block.partitions.length; + block.partitions = EMPTY; + } + } + require(size <= settings.maxBytes); + ByteArrayOutputStream buffer = new ByteArrayOutputStream((int) size); + DataOutputStream out = new DataOutputStream(buffer); + out.writeLong(MAGIC); + out.writeInt(FORMAT_VERSION); + out.write(digest(name.getBytes(StandardCharsets.UTF_8))); + out.writeLong(fileSize); + out.writeLong(entryCount); + out.writeInt(header.length); + out.write(header); + out.writeInt(dictionary.size()); + for (ByteBuffer bytes : dictionary.keySet()) { + out.writeInt(bytes.remaining()); + out.write(bytes.array()); + } + out.writeInt(blocks.size()); + for (IndexedBlock block : blocks) { + out.writeLong(block.block.offset); + out.writeLong(block.block.length); + out.writeLong(block.block.recordCount); + writePayload(out, block.partitions); + writePayload(out, block.rowIds); + writePayload(out, block.buckets); + } + out.write(digest(buffer.toByteArray())); + return buffer.toByteArray(); + } + + private static void writePayload(DataOutputStream out, byte[] payload) throws IOException { + out.writeByte(payload.length == 0 ? 0 : 1); + out.writeInt(payload.length); + out.write(payload); + } + } + + private static final class IndexedBlock { + private final Block block; + private byte[] partitions; + private byte[] rowIds; + private byte[] buckets; + + private IndexedBlock(Block block, byte[] partitions, byte[] rowIds, byte[] buckets) { + this.block = block; + this.partitions = partitions; + this.rowIds = rowIds; + this.buckets = buckets; + } + } + + /** Rebuild from the final physical blocks, including raw-copy and encoded rewrite paths. */ + @Nullable + public static byte[] build(FileIO io, Path path, long size, long records, Settings settings) + throws IOException { + try (ManifestAvroReader reader = new ManifestAvroReader(io.newInputStream(path))) { + Builder builder = new Builder(settings, reader.headerBytes()); + ProjectedManifestEntry.Projection projection = + ProjectedManifestEntry.BLOCK_INDEX_PROJECTION; + ProjectedManifestEntry entry = projection.createEntry(); + while (builder.complete() && reader.hasNext()) { + ManifestAvroReader.RawBlock block = reader.next(); + builder.beginBlock(reader.blockOffset(), reader.blockLength(), block.recordCount()); + ManifestAvroReader.RowIterator rows = block.toRows(projection.projectedType()); + while (builder.complete() && rows.hasNext()) { + entry.replace(rows.next()); + builder.add( + entry.file().firstRowId(), + entry.file().rowCount(), + entry.partitionBytes(), + entry.bucket(), + entry.totalBuckets()); + } + builder.endBlock(); + } + return builder.serialize(path.getName(), size, records); + } + } + + public static Selection select( + byte[] data, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + Settings settings) + throws IOException { + return select(data, manifest, query, null, null, settings); + } + + /** Validates framing and known payloads before applying independently available dimensions. */ + public static Selection select( + byte[] data, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter, + @Nullable RowType partitionType, + Settings settings) + throws IOException { + return select(data, manifest, query, partitionFilter, partitionType, null, settings); + } + + public static Selection select( + byte[] data, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter, + @Nullable RowType partitionType, + @Nullable BucketFilter bucketFilter, + Settings settings) + throws IOException { + require(data.length >= 128 && data.length <= settings.maxBytes); + int limit = data.length - DIGEST_BYTES; + require( + MessageDigest.isEqual( + digest(Arrays.copyOf(data, limit)), + Arrays.copyOfRange(data, limit, data.length))); + ByteBuffer in = ByteBuffer.wrap(data, 0, limit).slice(); + require(in.getLong() == MAGIC); + require(in.getInt() == FORMAT_VERSION); + byte[] hash = new byte[DIGEST_BYTES]; + in.get(hash); + require( + MessageDigest.isEqual( + hash, digest(manifest.fileName().getBytes(StandardCharsets.UTF_8)))); + require(in.getLong() == manifest.fileSize()); + long entries = Math.addExact(manifest.numAddedFiles(), manifest.numDeletedFiles()); + require(in.getLong() == entries); + int headerLength = in.getInt(); + require( + headerLength >= 21 + && headerLength <= MAX_AVRO_HEADER + && headerLength <= in.remaining() - 8); + byte[] header = new byte[headerLength]; + in.get(header); + require(header[0] == 'O' && header[1] == 'b' && header[2] == 'j' && header[3] == 1); + int partitions = in.getInt(); + require(partitions >= 0 && partitions <= in.remaining() / 16); + boolean[] matches = partitionFilter == null ? null : new boolean[partitions]; + Set unique = new java.util.HashSet<>(); + for (int id = 0; id < partitions; id++) { + require(in.remaining() >= 4); + int length = in.getInt(); + require(length >= 12 && length <= in.remaining()); + ByteBuffer encoded = in.slice(); + encoded.limit(length); + int arity = encoded.getInt(0); + require(arity >= 0 && 4L + ((arity + 71L) / 64) * 8 + arity * 8L <= length); + require(partitionType == null || arity == partitionType.getFieldCount()); + require(unique.add(encoded.asReadOnlyBuffer())); + if (partitionFilter != null) { + byte[] bytes = new byte[length]; + encoded.get(bytes); + BinaryRow partition = SerializationUtils.deserializeBinaryRow(bytes); + matches[id] = partitionFilter.test(partition); + } + in.position(in.position() + length); + } + require(in.remaining() >= 4); + int count = in.getInt(); + require(count >= 0 && count <= MAX_BLOCKS && count <= in.remaining() / BLOCK_BYTES); + long nextOffset = headerLength; + long firstRecord = 0; + List selected = new ArrayList<>(); + for (int i = 0; i < count; i++) { + require(in.remaining() >= BLOCK_BYTES); + long offset = in.getLong(); + long length = in.getLong(); + long records = in.getLong(); + require(offset == nextOffset && length > 0 && length <= manifest.fileSize() - offset); + require(records > 0 && records <= entries - firstRecord); + int partitionEncoding = Byte.toUnsignedInt(in.get()); + ByteBuffer partitionPayload = payload(in); + boolean partitionHit = true; + if (partitionEncoding == 0) { + require(!partitionPayload.hasRemaining()); + } else if (partitionEncoding == 1) { + require(partitionPayload.remaining() >= 4); + int ids = partitionPayload.getInt(); + require(ids > 0 && ids <= partitions && partitionPayload.remaining() == 4L * ids); + partitionHit = partitionFilter == null; + int previous = -1; + for (int j = 0; j < ids; j++) { + int id = partitionPayload.getInt(); + require(id > previous && id < partitions); + previous = id; + if (!partitionHit) { + partitionHit = matches[id]; + } + } + } + require(in.remaining() >= 5); + int rowEncoding = Byte.toUnsignedInt(in.get()); + ByteBuffer rowPayload = payload(in); + boolean rowHit = true; + if (rowEncoding == 0) { + require(!rowPayload.hasRemaining()); + } else if (rowEncoding == 1) { + require(rowPayload.remaining() >= 4); + int ranges = rowPayload.getInt(); + require(ranges > 0 && rowPayload.remaining() == 16L * ranges); + long min = rowPayload.getLong(); + long firstEnd = rowPayload.getLong(); + require(min >= 0 && firstEnd >= min); + boolean candidate = false; + if (partitionHit && query != null) { + long max = ranges == 1 ? firstEnd : rowPayload.getLong(rowPayload.limit() - 8); + require(max >= firstEnd); + candidate = query.intersects(min, max); + rowHit = candidate && (ranges == 1 || query.intersects(min, firstEnd)); + } + long previous = firstEnd; + for (int rangeIndex = 1; rangeIndex < ranges; rangeIndex++) { + long start = rowPayload.getLong(); + long end = rowPayload.getLong(); + require(start >= 0 && end >= start && start > previous); + previous = end; + if (candidate && !rowHit) { + rowHit = query.intersects(start, end); + } + } + } + require(in.remaining() >= 5); + int bucketEncoding = Byte.toUnsignedInt(in.get()); + ByteBuffer bucketPayload = payload(in); + boolean bucketHit = true; + if (bucketEncoding == 0) { + require(!bucketPayload.hasRemaining()); + } else if (bucketEncoding == 1) { + require(bucketPayload.remaining() >= 4); + int pairs = bucketPayload.getInt(); + require(pairs > 0 && bucketPayload.remaining() == 8L * pairs); + bucketHit = !partitionHit || !rowHit || bucketFilter == null; + long previous = -1; + for (int j = 0; j < pairs; j++) { + int bucket = bucketPayload.getInt(); + int totalBuckets = bucketPayload.getInt(); + require(bucket >= 0 && totalBuckets > bucket); + long pair = ((long) bucket << 32) | totalBuckets; + require(pair > previous); + previous = pair; + if (!bucketHit) { + bucketHit = bucketFilter.mayContain(bucket, totalBuckets); + } + } + } + if (partitionHit && rowHit && bucketHit) { + selected.add(new Block(offset, length, firstRecord, records)); + } + nextOffset = offset + length; + firstRecord += records; + } + require(!in.hasRemaining() && nextOffset == manifest.fileSize() && firstRecord == entries); + return new Selection(header, selected, selected.size() == count); + } + + private static ByteBuffer payload(ByteBuffer in) throws IOException { + require(in.remaining() >= 4); + int length = in.getInt(); + require(length >= 0 && length <= in.remaining()); + ByteBuffer result = in.slice(); + result.limit(length); + in.position(in.position() + length); + return result; + } + + /** Bounded, bulk sidecar reads. Null means read the original manifest. */ + @Nullable + public static Selection read( + FileIO io, + Path path, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + Settings settings) { + return read(io, path, manifest, query, null, null, settings); + } + + @Nullable + public static Selection read( + FileIO io, + Path path, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter, + @Nullable RowType partitionType, + Settings settings) { + return read(io, path, manifest, query, partitionFilter, partitionType, null, settings); + } + + @Nullable + public static Selection read( + FileIO io, + Path path, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter, + @Nullable RowType partitionType, + @Nullable BucketFilter bucketFilter, + Settings settings) { + String sidecarFileName = fileName(manifest); + if (sidecarFileName == null) { + return null; + } + try { + byte[] data; + try (InputStream in = io.newInputStream(new Path(path.getParent(), sidecarFileName))) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[Math.min(READ_BUFFER_BYTES, settings.maxBytes + 1)]; + int n; + while ((n = + in.read( + buffer, + 0, + Math.min( + buffer.length, settings.maxBytes + 1 - out.size()))) + != -1) { + out.write(buffer, 0, n); + require(out.size() <= settings.maxBytes); + } + data = out.toByteArray(); + } + return select( + data, manifest, query, partitionFilter, partitionType, bucketFilter, settings); + } catch (CancellationException failure) { + throw failure; + } catch (IOException | RuntimeException failure) { + List pending = new ArrayList<>(); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + pending.add(failure); + for (int position = 0; position < pending.size(); position++) { + Throwable cause = pending.get(position); + if (!visited.add(cause)) { + continue; + } + if (cause instanceof Error) { + throw (Error) cause; + } + if (cause instanceof CancellationException) { + throw (CancellationException) cause; + } + if (cause instanceof InterruptedException + || cause instanceof ClosedByInterruptException + || (cause instanceof InterruptedIOException + && !(cause instanceof SocketTimeoutException))) { + Thread.currentThread().interrupt(); + throw interrupted(failure); + } + if (cause.getCause() != null) { + pending.add(cause.getCause()); + } + Collections.addAll(pending, cause.getSuppressed()); + } + if (Thread.currentThread().isInterrupted()) { + throw interrupted(failure); + } + LOG.debug("Cannot use manifest sidecar for {}; reading manifest", path, failure); + return null; + } + } + + private static UncheckedIOException interrupted(Throwable failure) { + InterruptedIOException interrupted = + new InterruptedIOException("Interrupted reading manifest sidecar"); + interrupted.initCause(failure); + return new UncheckedIOException(interrupted); + } + + static InputStream openManifest(FileIO io, Path path, @Nullable Selection selected) + throws IOException { + SeekableInputStream input = io.newInputStream(path); + return selected == null ? input : new SelectedBlockInput(input, selected); + } + + /** An OCF stream comprising the original header and selected complete compressed blocks. */ + private static final class SelectedBlockInput extends InputStream { + private final SeekableInputStream input; + private final Selection selected; + private int headerPosition; + private int blockPosition; + private long remaining; + private byte[] buffer; + private int bufferPosition; + private int bufferLimit; + + private SelectedBlockInput(SeekableInputStream input, Selection selected) { + this.input = input; + this.selected = selected; + } + + @Override + public int read() throws IOException { + if (headerPosition < selected.header.length) { + return selected.header[headerPosition++] & 255; + } + return fillBuffer() ? buffer[bufferPosition++] & 255 : -1; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + if (length == 0) { + return 0; + } + if (headerPosition < selected.header.length) { + int n = Math.min(length, selected.header.length - headerPosition); + System.arraycopy(selected.header, headerPosition, bytes, offset, n); + headerPosition += n; + return n; + } + if (!fillBuffer()) { + return -1; + } + int copied = Math.min(length, bufferLimit - bufferPosition); + System.arraycopy(buffer, bufferPosition, bytes, offset, copied); + bufferPosition += copied; + return copied; + } + + private boolean fillBuffer() throws IOException { + if (bufferPosition < bufferLimit) { + return true; + } + if (remaining == 0) { + if (blockPosition == selected.blocks.size()) { + return false; + } + Block block = selected.blocks.get(blockPosition++); + long end = block.offset + block.length; + while (blockPosition < selected.blocks.size() + && selected.blocks.get(blockPosition).offset == end) { + end += selected.blocks.get(blockPosition++).length; + } + input.seek(block.offset); + remaining = end - block.offset; + } + int requested = (int) Math.min(READ_BUFFER_BYTES, remaining); + if (buffer == null || buffer.length < requested) { + buffer = new byte[requested]; + } + bufferPosition = 0; + bufferLimit = 0; + while (bufferLimit < requested) { + int count = input.read(buffer, bufferLimit, requested - bufferLimit); + if (count < 0) { + throw new EOFException("Truncated manifest block"); + } + bufferLimit += count; + } + remaining -= requested; + return true; + } + + @Override + public void close() throws IOException { + input.close(); + } + } + + private static void require(boolean valid) throws IOException { + if (!valid) { + throw new IOException( + "Invalid, unsupported, mismatched or over-budget manifest sidecar"); + } + } + + private static byte[] digest(byte[] bytes) { + try { + return MessageDigest.getInstance("SHA-256").digest(bytes); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java index 77486d4cdd53..defd81bcd3e2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java @@ -27,6 +27,7 @@ import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -49,6 +50,7 @@ public final class ProjectedManifestEntry implements ManifestEntry { private static final Projection FULL_PROJECTION = Projection.create(MANIFEST_ROW_TYPE); public static final Projection DELETE_ENTRY_PROJECTION = createDeleteEntryProjection(); public static final Projection ROW_RANGE_PROJECTION = createRowRangeProjection(); + public static final Projection BLOCK_INDEX_PROJECTION = createBlockIndexProjection(); public static final Projection ENTRY_LAYOUT_PROJECTION = createEntryLayoutProjection(); private final Projection projection; @@ -133,6 +135,13 @@ private static Projection createRowRangeProjection() { DataFileMeta.FIRST_ROW_ID))))); } + private static Projection createBlockIndexProjection() { + List fields = new ArrayList<>(ROW_RANGE_PROJECTION.projectedType().getFields()); + fields.add(MANIFEST_ROW_TYPE.getField(ManifestEntry.BUCKET)); + fields.add(MANIFEST_ROW_TYPE.getField(ManifestEntry.TOTAL_BUCKETS)); + return Projection.create(new RowType(false, fields)); + } + private static Projection createEntryLayoutProjection() { RowType manifestType = MANIFEST_ROW_TYPE; return Projection.create( diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java index 82923ef9d827..e40d27e789b8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java @@ -29,6 +29,7 @@ import org.apache.paimon.manifest.ManifestEntrySerializer; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.manifest.ManifestSidecar; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.manifest.SimpleFileEntry; import org.apache.paimon.operation.metrics.ScanMetrics; @@ -498,27 +499,35 @@ private List readManifest( @Nullable Filter additionalFilter, @Nullable Filter additionalTFilter) { + ManifestFile manifestFile = manifestFileFactory.create(); + BucketFilter bucketFilter = createBucketFilter(); + ManifestSidecar.Selection selected = + manifestFile.selectBlocks( + manifest, rowRangeIndex, manifestsReader.partitionFilter(), bucketFilter); + if (selected != null && selected.blocks().isEmpty()) { + return Collections.emptyList(); + } Filter entryRowFilter = createEntryRowFilter(); Function finalConverter = dropStats ? e -> converter.apply(dropStats(e)) : converter; List entries = - manifestFileFactory - .create() + manifestFile .withCacheMetrics( scanMetrics != null ? scanMetrics.getCacheMetrics() : null) .read( manifest.fileName(), manifest.fileSize(), manifestsReader.partitionFilter(), - createBucketFilter(), + bucketFilter, entryRowFilter.and(additionalFilter), entry -> (additionalTFilter == null || additionalTFilter.test(entry)) && (manifestEntryFilter == null || manifestEntryFilter.test(entry)) && filterByStats(entry), - finalConverter); + finalConverter, + selected); LOG.info("Read {} manifest entries from {}", entries.size(), manifest.fileName()); return entries; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java b/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java index 9689f272e2eb..eec6090a3635 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java @@ -26,7 +26,6 @@ import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestFile; -import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.ManifestList; import org.apache.paimon.stats.StatsFileHandler; import org.apache.paimon.utils.FileStorePathFactory; @@ -100,17 +99,17 @@ public Set manifestSkippingSet(List skippingSnapshots) { // base manifests if (manifestList.exists(skippingSnapshot.baseManifestList())) { skippingSet.add(skippingSnapshot.baseManifestList()); - manifestList.read(skippingSnapshot.baseManifestList()).stream() - .map(ManifestFileMeta::fileName) - .forEach(skippingSet::add); + manifestList + .read(skippingSnapshot.baseManifestList()) + .forEach(manifest -> addManifestToSkippingSet(skippingSet, manifest)); } // delta manifests if (manifestList.exists(skippingSnapshot.deltaManifestList())) { skippingSet.add(skippingSnapshot.deltaManifestList()); - manifestList.read(skippingSnapshot.deltaManifestList()).stream() - .map(ManifestFileMeta::fileName) - .forEach(skippingSet::add); + manifestList + .read(skippingSnapshot.deltaManifestList()) + .forEach(manifest -> addManifestToSkippingSet(skippingSet, manifest)); } // index manifests diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java index 6494411a8c4b..c9b6f0c47012 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java @@ -489,9 +489,9 @@ private Set manifestSkippingSet(Snapshot skippingSnapshot) { // data manifests skippingSet.add(skippingSnapshot.baseManifestList()); skippingSet.add(skippingSnapshot.deltaManifestList()); - manifestList.readDataManifests(skippingSnapshot).stream() - .map(ManifestFileMeta::fileName) - .forEach(skippingSet::add); + manifestList + .readDataManifests(skippingSnapshot) + .forEach(manifest -> addManifestToSkippingSet(skippingSet, manifest)); // index manifests String indexManifest = skippingSnapshot.indexManifest(); @@ -511,6 +511,14 @@ private Set manifestSkippingSet(Snapshot skippingSnapshot) { return skippingSet; } + protected static void addManifestToSkippingSet( + Set skippingSet, ManifestFileMeta manifest) { + skippingSet.add(manifest.fileName()); + if (manifest.extraFiles() != null) { + skippingSet.addAll(manifest.extraFiles()); + } + } + private boolean tryDeleteEmptyDirectory(Path path) { try { fileIO.delete(path, false); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java index e3f8c7af7671..fb6e7aec89ec 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java @@ -79,7 +79,7 @@ public static List merge( // exception occurs, clean up and rethrow for (ManifestFileMeta manifest : newFilesForAbort) { try { - manifestFile.delete(manifest.fileName()); + manifestFile.delete(manifest); } catch (Throwable cleanupFailure) { primaryFailure = ExceptionUtils.firstOrSuppressed(cleanupFailure, primaryFailure); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java index a24b5c4c6e9b..735a706937cb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java @@ -52,14 +52,14 @@ public void cleanUpReuseTmpManifests( String newIndexManifest) { if (deltaManifestList != null) { for (ManifestFileMeta manifest : manifestList.read(deltaManifestList.getKey())) { - manifestFile.delete(manifest.fileName()); + manifestFile.delete(manifest); } manifestList.delete(deltaManifestList.getKey()); } if (changelogManifestList != null) { for (ManifestFileMeta manifest : manifestList.read(changelogManifestList.getKey())) { - manifestFile.delete(manifest.fileName()); + manifestFile.delete(manifest); } manifestList.delete(changelogManifestList.getKey()); } @@ -80,7 +80,7 @@ public void cleanUpNoReuseTmpManifests( .collect(Collectors.toSet()); for (ManifestFileMeta suspect : mergeAfterManifests) { if (!oldMetaSet.contains(suspect.fileName())) { - manifestFile.delete(suspect.fileName()); + manifestFile.delete(suspect); } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java new file mode 100644 index 000000000000..846c73d50db3 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -0,0 +1,617 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryRowWriter; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.options.MemorySize; +import org.apache.paimon.options.Options; +import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Range; +import org.apache.paimon.utils.RowRangeIndex; +import org.apache.paimon.utils.SerializationUtils; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import static org.apache.paimon.manifest.ManifestSidecarTest.meta; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +/** Independent partition/row-ID payloads and conservative resource degradation. */ +class ManifestBlockIndexTest { + private final RowType type = RowType.of(DataTypes.INT(), DataTypes.STRING()); + private final ManifestSidecar.Settings defaults = new ManifestSidecar.Settings(new Options()); + + private byte[] fixture(String field) throws IOException { + Properties p = new Properties(); + try (java.io.InputStream in = getClass().getResourceAsStream("/manifest-sidecar.txt")) { + p.load(in); + } + return Base64.getDecoder().decode(p.getProperty(field)); + } + + private byte[] partition(int p, String q) { + BinaryRow row = new BinaryRow(2); + BinaryRowWriter writer = new BinaryRowWriter(row); + writer.writeInt(0, p); + if (q == null) { + writer.setNullAt(1); + } else { + writer.writeString(1, BinaryString.fromString(q)); + } + writer.complete(); + return SerializationUtils.serializeBinaryRow(row); + } + + private RowRangeIndex query(long point) { + return RowRangeIndex.create(Collections.singletonList(new Range(point, point))); + } + + private PartitionPredicate part(int value) { + return PartitionPredicate.fromPredicate(type, new PredicateBuilder(type).equal(0, value)); + } + + @Test + void jointGoldenPreservesTuplesNullsAndDerivedOrdinals() throws Exception { + byte[] a = partition(7, "left"); + byte[] b = partition(9, null); + assertThat(a).isEqualTo(fixture("partitionA")); + assertThat(b).isEqualTo(fixture("partitionB")); + byte[] header = fixture("avroHeader"); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(defaults, header); + builder.beginBlock(header.length, 100, 3); + builder.add(0L, 10, a); + builder.add(5L, 5, a); + builder.add(20L, 5, b); + builder.endBlock(); + builder.beginBlock(header.length + 100, 200, 2); + builder.add((1L << 32) - 2, 5, b); + builder.add(8254058425445L, 1, a); + builder.endBlock(); + builder.beginBlock(header.length + 300, 100, 2); + builder.add(20L, 5, a); + builder.add(Long.MAX_VALUE, 1, b); + builder.endBlock(); + byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); + assertThat(data).isEqualTo(fixture("indexWithPartitions")); + ManifestFileMeta meta = meta("manifest-golden", header.length + 400, 7); + PartitionPredicate filter = spy(part(7)); + assertThat(ManifestSidecar.select(data, meta, query(20), filter, type, defaults).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 5L); + verify(filter, times(2)).test(any(BinaryRow.class)); + PartitionPredicate nullFilter = + PartitionPredicate.fromPredicate(type, new PredicateBuilder(type).isNull(1)); + assertThat(ManifestSidecar.select(data, meta, null, nullFilter, type, defaults).blocks()) + .hasSize(3); + assertThat(ManifestSidecar.select(data, meta, null, part(99), type, defaults).blocks()) + .isEmpty(); + // Missing partition payloads cannot be pruned by dictionary misses. + assertThat( + ManifestSidecar.select( + fixture("index"), meta, null, part(99), type, defaults) + .blocks()) + .hasSize(3); + } + + @Test + void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(512)); + ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); + byte[] header = fixture("avroHeader"); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + builder.beginBlock(header.length, 100, 1); + builder.add(null, 10, partition(7, "left")); + builder.endBlock(); + builder.beginBlock(header.length + 100, 100, 1); + builder.add(200L, 10, partition(9, String.join("", Collections.nCopies(600, "x")))); + builder.endBlock(); + builder.beginBlock(header.length + 200, 100, 1); + builder.add(300L, 10, partition(7, "left")); // an existing dictionary ID remains usable + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 300, 3); + ManifestFileMeta meta = meta("m", header.length + 300, 3); + assertThat(ManifestSidecar.select(data, meta, null, part(9), type, settings).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(1L); + assertThat(ManifestSidecar.select(data, meta, query(999), part(7), type, settings).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + assertThat(ManifestSidecar.select(data, meta, query(200), part(9), type, settings).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(1L); + } + + @Test + void coarseningContinuesThroughTheEntireBlockAndDetectsUnknownRows() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(512)); + ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); + byte[] header = fixture("avroHeader"); + for (boolean unknown : new boolean[] {false, true}) { + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + builder.beginBlock(header.length, 100, 66); + for (int i = 0; i < 64; i++) { + builder.add(100L + i * 1000L, 10, partition(7, "left")); + } + builder.add(10L, 10, partition(7, "left")); + builder.add(unknown ? null : Long.MAX_VALUE, 1, partition(7, "left")); + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 100, 66); + ManifestFileMeta meta = meta("m", header.length + 100, 66); + for (long point : new long[] {10, 100, 200, Long.MAX_VALUE}) { + assertThat(ManifestSidecar.select(data, meta, query(point), settings).blocks()) + .hasSize(1); + } + assertThat(ManifestSidecar.select(data, meta, query(0), settings).blocks()) + .hasSize(unknown ? 1 : 0); + assertThat(ManifestSidecar.select(data, meta, null, part(9), type, settings).blocks()) + .isEmpty(); + } + } + + private List positions(byte[] data) { + ByteBuffer in = ByteBuffer.wrap(data); + in.position(60); + int header = in.getInt(); + in.position(in.position() + header); + int partitions = in.getInt(); + for (int i = 0; i < partitions; i++) { + int length = in.getInt(); + in.position(in.position() + length); + } + int blocks = in.getInt(); + List result = new ArrayList<>(); + for (int i = 0; i < blocks; i++) { + int block = in.position(); + in.position(block + 24); + int partition = in.position(); + in.get(); + int length = in.getInt(); + in.position(in.position() + length); + int row = in.position(); + in.get(); + length = in.getInt(); + in.position(in.position() + length); + int bucket = in.position(); + in.get(); + length = in.getInt(); + in.position(in.position() + length); + result.add(new int[] {block, partition, row, bucket}); + } + return result; + } + + private byte[] checksum(byte[] data) throws Exception { + byte[] hash = + MessageDigest.getInstance("SHA-256").digest(Arrays.copyOf(data, data.length - 32)); + System.arraycopy(hash, 0, data, data.length - 32, 32); + return data; + } + + @Test + void partitionMissSkipsRowAndBucketMatching() throws Exception { + byte[] data = fixture("indexWithBuckets"); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + RowRangeIndex rows = spy(query(0)); + BucketFilter buckets = mock(BucketFilter.class); + assertThat( + ManifestSidecar.select(data, meta, rows, part(99), type, buckets, defaults) + .blocks()) + .isEmpty(); + verifyNoInteractions(rows, buckets); + } + + @Test + void rowMissSkipsBucketMatchingWithOrWithoutPartitionFilter() throws Exception { + byte[] data = fixture("indexWithBuckets"); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + for (PartitionPredicate partition : Arrays.asList(null, part(7))) { + BucketFilter buckets = mock(BucketFilter.class); + assertThat( + ManifestSidecar.select( + data, meta, query(15), partition, type, buckets, + defaults) + .blocks()) + .isEmpty(); + verifyNoInteractions(buckets); + } + } + + @Test + void absentPartitionFilterKeepsRowAndBucketMatching() throws Exception { + byte[] data = fixture("indexWithBuckets"); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + BucketFilter buckets = spy(BucketFilter.create(false, 1, null, null)); + assertThat( + ManifestSidecar.select(data, meta, query(20), null, type, buckets, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + verify(buckets).mayContain(1, 4); + verify(buckets).mayContain(0, 1); + verify(buckets).mayContain(3, 4); + verifyNoMoreInteractions(buckets); + } + + @Test + void absentRowOrBucketFiltersKeepRemainingDimensions() throws Exception { + byte[] data = fixture("indexWithBuckets"); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + assertThat( + ManifestSidecar.select( + data, + meta, + null, + part(7), + type, + BucketFilter.create(false, 1, null, null), + defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + assertThat( + ManifestSidecar.select(data, meta, query(20), part(7), type, null, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 5L); + assertThat(ManifestSidecar.select(data, meta, null, null, type, null, defaults).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 3L, 5L); + } + + @Test + void unknownUnsignedEncodingsSkipOnlyTheirDimensionAndMalformedPayloadsFail() throws Exception { + byte[] good = fixture("indexWithBuckets"); + int[] first = positions(good).get(0); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + byte[] data = good.clone(); + data[first[1]] = (byte) 200; + assertThat( + ManifestSidecar.select( + checksum(data), meta, query(0), part(99), type, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + data = good.clone(); + data[first[2]] = (byte) 201; + assertThat( + ManifestSidecar.select( + checksum(data), meta, query(16), part(7), type, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + data = good.clone(); + data[first[3]] = (byte) 202; + // Unknown encodings skip their payload without decoding even an invalid pair count. + ByteBuffer.wrap(data).putInt(first[3] + 5, 0); + checksum(data); + BucketFilter noBucket = BucketFilter.create(false, 99, null, null); + assertThat( + ManifestSidecar.select( + data, meta, query(20), part(7), type, noBucket, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + assertThat( + ManifestSidecar.select( + data, meta, query(999), part(7), type, noBucket, defaults) + .blocks()) + .isEmpty(); + assertThat( + ManifestSidecar.select( + data, meta, query(20), part(99), type, noBucket, defaults) + .blocks()) + .isEmpty(); + for (int position : new int[] {first[1], first[2], first[3]}) { + byte[] bad = good.clone(); + bad[position] = 0; // encoding 0 cannot have payload bytes + checksum(bad); + assertThatThrownBy(() -> ManifestSidecar.select(bad, meta, query(0), defaults)) + .isInstanceOf(IOException.class); + byte[] invalidLength = good.clone(); + invalidLength[position] = (byte) 255; + ByteBuffer.wrap(invalidLength).putInt(position + 1, -1); + checksum(invalidLength); + assertThatThrownBy( + () -> ManifestSidecar.select(invalidLength, meta, query(0), defaults)) + .isInstanceOf(IOException.class); + } + // A checksummed directory with missing bytes/entries must still be rejected. + byte[] bad = good.clone(); + ByteBuffer.wrap(bad).putLong(first[0] + 16, 2); + checksum(bad); + assertThatThrownBy(() -> ManifestSidecar.select(bad, meta, query(0), defaults)) + .isInstanceOf(IOException.class); + byte[] badRange = good.clone(); + // Row payload begins after its encoding and length, then the range-count integer. + ByteBuffer.wrap(badRange).putLong(first[2] + 9 + 16, 9L); + checksum(badRange); + assertThatThrownBy( + () -> + ManifestSidecar.select( + badRange, meta, query(999), part(99), type, defaults)) + .isInstanceOf(IOException.class); + byte[] badId = good.clone(); + ByteBuffer.wrap(badId).putInt(first[1] + 9, 999); + checksum(badId); + assertThatThrownBy(() -> ManifestSidecar.select(badId, meta, query(999), defaults)) + .isInstanceOf(IOException.class); + } + + @Test + void bucketPayloadGoldenAndTotalBucketsArePreserved() throws Exception { + byte[] header = fixture("avroHeader"); + byte[] a = partition(7, "left"); + byte[] b = partition(9, null); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(defaults, header); + builder.beginBlock(header.length, 100, 3); + builder.add(0L, 10, a, 1, 4); + builder.add(5L, 5, a, 1, 4); + builder.add(20L, 5, b, 1, 8); + builder.endBlock(); + builder.beginBlock(header.length + 100, 200, 2); + builder.add((1L << 32) - 2, 5, b, 2, 4); + builder.add(8254058425445L, 1, a, 2, 8); + builder.endBlock(); + builder.beginBlock(header.length + 300, 100, 2); + builder.add(20L, 5, a, 0, 1); + builder.add(Long.MAX_VALUE, 1, b, 3, 4); + builder.endBlock(); + byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); + assertThat(data).isEqualTo(fixture("indexWithBuckets")); + ManifestFileMeta meta = meta("manifest-golden", header.length + 400, 7); + BucketFilter bucket = BucketFilter.create(false, 1, null, null); + assertThat(ManifestSidecar.select(data, meta, null, null, type, bucket, defaults).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + // Existing row-id and partition coverage remains independently usable. + assertThat( + ManifestSidecar.select( + data, + meta, + query(0), + part(7), + type, + BucketFilter.create(false, 2, null, null), + defaults) + .blocks()) + .isEmpty(); + ManifestBucketFilter totalAware = + new ManifestBucketFilter() { + @Override + public boolean test(BinaryRow partition, Integer bucket, Integer total) { + throw new AssertionError("Block lookup must not invent a partition"); + } + + @Override + public boolean mayContain(int min, int max, int total) { + return min == 2 && max == 2 && total == 8; + } + }; + BucketFilter filter = BucketFilter.create(false, null, null, totalAware); + assertThat(ManifestSidecar.select(data, meta, null, null, type, filter, defaults).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(3L); + BucketFilter partitionAware = + BucketFilter.create( + false, + null, + null, + (partition, bucketId, total) -> { + throw new AssertionError("Needs actual entry partition"); + }); + assertThat( + ManifestSidecar.select( + data, meta, null, null, type, partitionAware, defaults) + .blocks()) + .hasSize(3); + for (String unavailable : new String[] {"index", "indexWithPartitions"}) { + assertThat( + ManifestSidecar.select( + fixture(unavailable), + meta, + null, + null, + type, + BucketFilter.create(false, 99, null, null), + defaults) + .blocks()) + .hasSize(3); + } + } + + @Test + void unknownInvalidOrOverBudgetBucketPayloadIsUnavailable() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(512)); + ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); + byte[] header = fixture("avroHeader"); + for (Integer[] pair : + Arrays.asList( + new Integer[] {null, null}, + new Integer[] {-1, 4}, + new Integer[] {4, 4}, + new Integer[] {0, 0}, + new Integer[] {2, 8})) { + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + int extraPairs = pair[0] != null && pair[0] == 2 ? 65 : 0; + builder.beginBlock(header.length, 100, 2 + extraPairs); + builder.add(100L, 10, partition(7, "left"), 1, 4); + builder.add(200L, 10, partition(7, "left"), pair[0], pair[1]); + for (int i = 0; i < extraPairs; i++) { + builder.add(200L, 10, partition(7, "left"), i, 100); + } + builder.endBlock(); + builder.beginBlock(header.length + 100, 100, 1); + builder.add(300L, 10, partition(7, "left"), 1, 4); + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 200, 3 + extraPairs); + int bucket = positions(data).get(0)[3]; + assertThat(data[bucket]).isZero(); + assertThat(ByteBuffer.wrap(data).getInt(bucket + 1)).isZero(); + ManifestFileMeta meta = meta("m", header.length + 200, 3 + extraPairs); + assertThat( + ManifestSidecar.select( + data, + meta, + null, + null, + type, + BucketFilter.create(false, 99, null, null), + settings) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + assertThat( + ManifestSidecar.select( + data, + meta, + query(999), + null, + type, + BucketFilter.create(false, 99, null, null), + settings) + .blocks()) + .isEmpty(); + } + } + + @Test + void malformedBucketPayloadInvalidatesTheContainer() throws Exception { + byte[] good = fixture("indexWithBuckets"); + int payload = positions(good).get(0)[3] + 1; + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + for (int[] mutation : + new int[][] { + {payload, -2}, + {payload, Integer.MAX_VALUE}, + {payload, 0}, + {payload + 4, 0}, + {payload + 8, -1}, + {payload + 12, 1}, + {payload + 16, 0} + }) { + byte[] bad = good.clone(); + ByteBuffer.wrap(bad).putInt(mutation[0], mutation[1]); + checksum(bad); + assertThatThrownBy( + () -> + ManifestSidecar.select( + bad, meta, query(999), part(99), type, defaults)) + .isInstanceOf(IOException.class); + } + } + + @Test + void payloadsCanExceedFormerLimitsWithinByteBudget() throws Exception { + byte[] header = fixture("avroHeader"); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(defaults, header); + int blocks = 33; + int entriesPerBlock = 4097; + int entries = blocks * entriesPerBlock; + int blockBytes = 1024 * 1024; + for (int block = 0; block < blocks; block++) { + builder.beginBlock( + header.length + (long) block * blockBytes, blockBytes, entriesPerBlock); + for (int i = 0; i < entriesPerBlock; i++) { + int entry = block * entriesPerBlock + i; + builder.add(entry * 2L, 1, partition(entry, null), i, entriesPerBlock + 1); + } + builder.endBlock(); + } + long fileSize = header.length + (long) blocks * blockBytes; + byte[] data = builder.serialize("m", fileSize, entries); + assertThat(data.length).isLessThanOrEqualTo(defaults.maxBytes); + ManifestFileMeta meta = meta("m", fileSize, entries); + long last = (entries - 1L) * 2; + assertThat(ManifestSidecar.select(data, meta, query(last), defaults).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly((blocks - 1L) * entriesPerBlock); + assertThat(ManifestSidecar.select(data, meta, query(last - 1), defaults).blocks()) + .isEmpty(); + assertThat(ManifestSidecar.select(data, meta, null, part(entries), type, defaults).blocks()) + .isEmpty(); + assertThat( + ManifestSidecar.select( + data, + meta, + null, + null, + type, + BucketFilter.create(false, entriesPerBlock, null, null), + defaults) + .blocks()) + .isEmpty(); + } + + @Test + void tightByteBudgetKeepsAllDescriptorsOrOmitsTheWholeFile() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(280)); + ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); + byte[] header = fixture("avroHeader"); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + for (int i = 0; i < 3; i++) { + builder.beginBlock(header.length + 100L * i, 100, 1); + builder.add(i * 100L, 10, partition(7, "left")); + builder.endBlock(); + } + byte[] data = builder.serialize("m", header.length + 300, 3); + assertThat(data.length).isLessThanOrEqualTo(280); + assertThat( + ManifestSidecar.select( + data, + meta("m", header.length + 300, 3), + query(999), + part(99), + type, + settings) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 1L, 2L); + builder.beginBlock(header.length + 300, 100, 1); + builder.add(300L, 1, partition(7, "left")); + builder.endBlock(); + assertThat(builder.serialize("m", header.length + 400, 4)).isNull(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java index 46ff2067d29f..0b2dc24baf09 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java @@ -18,6 +18,7 @@ package org.apache.paimon.manifest; +import org.apache.paimon.data.GenericRow; import org.apache.paimon.utils.ObjectSerializer; import org.apache.paimon.utils.ObjectSerializerTestBase; @@ -53,7 +54,8 @@ void testExtraFiles() throws IOException { Arrays.asList( null, Collections.emptyList(), - Arrays.asList("extra-1", "extra-2"))) { + Arrays.asList("extra-1", "extra-2"), + Arrays.asList("partition-index", "manifest" + ManifestSidecar.SUFFIX))) { ManifestFileMeta meta = new ManifestFileMeta( original.fileName(), @@ -85,6 +87,19 @@ void testExtraFiles() throws IOException { } } + @Test + void testOldRowWithoutExtraFilesField() { + ManifestFileMeta meta = object(); + ManifestFileMetaSerializer serializer = new ManifestFileMetaSerializer(); + GenericRow current = (GenericRow) serializer.toRow(meta); + GenericRow legacy = new GenericRow(current.getFieldCount() - 1); + for (int field = 0; field < legacy.getFieldCount(); field++) { + legacy.setField(field, current.getField(field)); + } + assertThat(serializer.fromRow(legacy)).isEqualTo(meta); + assertThat(serializer.fromRow(legacy).extraFiles()).isNull(); + } + @Override protected ObjectSerializer serializer() { return new ManifestFileMetaSerializer(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index c85d82bf9c01..a1f4b4778ef5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.manifest; import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.FileFormat; @@ -28,13 +29,19 @@ import org.apache.paimon.fs.FileIOFinder; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.SeekableInputStreamWrapper; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMetaWriteColsLegacySerializer; +import org.apache.paimon.operation.AppendOnlyFileStoreScan; +import org.apache.paimon.operation.ManifestsReader; +import org.apache.paimon.operation.commit.CommitCleaner; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.schema.FileSystemSchemaManager; +import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.StatsTestUtils; import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; @@ -42,6 +49,9 @@ import org.apache.paimon.utils.FailingFileIO; import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.Filter; +import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.Range; +import org.apache.paimon.utils.RowRangeIndex; import org.apache.paimon.utils.SegmentsCache; import org.junit.jupiter.api.RepeatedTest; @@ -63,13 +73,18 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.apache.paimon.TestKeyValueGenerator.DEFAULT_PART_TYPE; +import static org.apache.paimon.manifest.ManifestIndexTestUtils.withExtraFiles; import static org.apache.paimon.stats.StatsTestUtils.convertWithoutSchemaEvolution; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** Tests for {@link ManifestFile}. */ public class ManifestFileTest { @@ -1260,16 +1275,609 @@ private static int indexOf(byte[] bytes, byte[] target, int from, int limit) { return -1; } + @Test + void testRowIdSidecarRollingRawRewriteAndDelete() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); + ManifestFile manifests = createManifestFile(tempDir.toString(), 1, options); + List entries = new ArrayList<>(); + for (int i = 0; i < 2200; i++) { + ManifestEntry source = gen.next(); + entries.add( + ManifestEntry.create( + i % 2 == 0 ? FileKind.ADD : FileKind.DELETE, + source.partition(), + source.bucket(), + source.totalBuckets(), + source.file().newFirstRowId(i * 100000000L))); + } + List metas = manifests.write(entries); + assertThat(metas.size()).isGreaterThan(1); + + for (ManifestFileMeta meta : metas) { + assertThat(ManifestSidecar.fileName(meta)) + .isEqualTo(meta.fileName() + ManifestSidecar.SUFFIX); + List actual = manifests.read(meta.fileName()); + for (ManifestEntry entry : + Arrays.asList(actual.get(0), actual.get(actual.size() - 1))) { + assertThat( + manifests.mayContainRowIds( + meta, + RowRangeIndex.create( + Collections.singletonList( + new Range( + entry.file().firstRowId(), + entry.file().firstRowId()))))) + .isTrue(); + } + long gap = actual.get(0).file().firstRowId() + actual.get(0).file().rowCount(); + assertThat( + manifests.mayContainRowIds( + meta, + RowRangeIndex.create( + Collections.singletonList(new Range(gap, gap))))) + .isFalse(); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest") + .resolve(meta.fileName() + ManifestSidecar.SUFFIX))) + .isTrue(); + } + ManifestFileMeta source = metas.get(0); + ManifestAvroWriter rewrite = manifests.createAvroWriter(); + try (ManifestAvroReader reader = + manifests.scanAvroBlocks(source.fileName(), source.fileSize())) { + rewrite.writeEncodedManifest(reader, source); + } + rewrite.close(); + ManifestFileMeta rewritten = rewrite.result().get(0); + assertThat(ManifestSidecar.fileName(rewritten)) + .isEqualTo(rewritten.fileName() + ManifestSidecar.SUFFIX); + assertThat(manifests.read(rewritten.fileName())) + .isEqualTo(manifests.read(source.fileName())); + long outside = metas.get(metas.size() - 1).maxRowId(); + assertThat( + manifests.mayContainRowIds( + rewritten, + RowRangeIndex.create( + Collections.singletonList(new Range(outside, outside))))) + .isFalse(); + rewrite.abort(); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest") + .resolve(rewritten.fileName() + ManifestSidecar.SUFFIX))) + .isFalse(); + for (ManifestFileMeta meta : metas) { + manifests.delete(meta); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest") + .resolve(meta.fileName() + ManifestSidecar.SUFFIX))) + .isFalse(); + } + } + + @Test + void testDisabledSidecarsDoNotConstrainManifestTargetSize() { + for (String target : new String[] {"1 bytes", "1 gb"}) { + Options options = new Options(); + options.setString(CoreOptions.MANIFEST_TARGET_FILE_SIZE.key(), target); + ManifestFile manifestFile = + createManifestFileFactory( + tempDir.toString(), + Long.MAX_VALUE, + options, + new RecordingFileIO()) + .create(); + List entries = Collections.singletonList(gen.next()); + ManifestFileMeta meta = manifestFile.write(entries).get(0); + assertThat(ManifestSidecar.fileName(meta)).isNull(); + assertThat(manifestFile.read(meta.fileName())).containsExactlyElementsOf(entries); + } + } + + @Test + void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); + RecordingFileIO fileIO = new RecordingFileIO(); + ManifestFile.Factory factory = + createManifestFileFactory( + tempDir.toString(), + Long.MAX_VALUE, + options, + fileIO, + new SegmentsCache<>(1024, MemorySize.ofMebiBytes(16), Long.MAX_VALUE)); + ManifestFile manifests = factory.create(); + List entries = new ArrayList<>(); + for (int i = 0; i < 4000; i++) { + ManifestEntry entry = gen.next(); + entries.add( + ManifestEntry.create( + FileKind.ADD, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + entry.file().newFirstRowId(i * 1000000L))); + } + ManifestFileMeta meta = manifests.write(entries).get(0); + RowRangeIndex query = + RowRangeIndex.create( + Arrays.asList( + new Range(1000000000L, 1000000000L), + new Range(3000000000L, 3000000000L))); + + ManifestSidecar.Selection selected = manifests.selectBlocks(meta, query); + assertThat(selected.blocks()).hasSize(2); + + fileIO.reset(); + List actual = + factory.create() + .read( + meta.fileName(), + meta.fileSize(), + null, + null, + row -> true, + entry -> true, + java.util.function.Function.identity(), + selected); + List expected = new ArrayList<>(); + for (ManifestSidecar.Block block : selected.blocks()) { + expected.addAll( + entries.subList( + (int) block.firstRecord, + (int) (block.firstRecord + block.recordCount))); + } + assertThat(actual).containsExactlyElementsOf(expected); + assertThat(actual).contains(entries.get(1000), entries.get(3000)); + assertThat(fileIO.bytes.get()).isLessThan(meta.fileSize() / 4); + assertThat(fileIO.seeks) + .containsExactlyElementsOf( + selected.blocks().stream() + .map(block -> block.offset) + .collect(Collectors.toList())); + assertThat(fileIO.opened) + .containsExactly(new Path(tempDir.toString(), "manifest/" + meta.fileName())); + ManifestSidecar.Selection allBlocks = + manifests.selectBlocks( + meta, + RowRangeIndex.create( + Collections.singletonList(new Range(0, Long.MAX_VALUE)))); + fileIO.reset(); + assertThat( + factory.create() + .read( + meta.fileName(), + meta.fileSize(), + null, + null, + row -> true, + entry -> true, + java.util.function.Function.identity(), + allBlocks)) + .containsExactlyInAnyOrderElementsOf(entries); + // The preceding partial read must not populate the full-manifest cache. + assertThat(fileIO.opened) + .containsExactly(new Path(tempDir.toString(), "manifest/" + meta.fileName())); + + fileIO.reset(); + assertThat( + factory.create() + .read( + meta.fileName(), + meta.fileSize(), + null, + null, + row -> true, + entry -> true, + java.util.function.Function.identity(), + allBlocks)) + .containsExactlyInAnyOrderElementsOf(entries); + assertThat(manifests.read(meta.fileName())).containsExactlyInAnyOrderElementsOf(entries); + assertThat(fileIO.opened).isEmpty(); + } + + @Test + void testScannerPreservesDeletesAndColumnGroups() { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); + RecordingFileIO fileIO = new RecordingFileIO(); + ManifestFile.Factory factory = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO); + ManifestFile manifests = factory.create(); + ManifestEntry entry = gen.next(); + ManifestEntry add = + ManifestEntry.create( + FileKind.ADD, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + entry.file().newFirstRowId(100L)); + ManifestEntry delete = + ManifestEntry.create( + FileKind.DELETE, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + add.file()); + ManifestEntry other = gen.next(); + ManifestEntry live = + ManifestEntry.create( + FileKind.ADD, + other.partition(), + other.bucket(), + other.totalBuckets(), + other.file().newFirstRowId(100L)); + List metas = new ArrayList<>(); + metas.addAll(manifests.write(Arrays.asList(add, live))); + metas.addAll(manifests.write(Collections.singletonList(delete))); + metas.addAll( + manifests.write( + Collections.singletonList( + ManifestEntry.create( + FileKind.ADD, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + entry.file().newFirstRowId(0L))))); + AppendOnlyFileStoreScan scan = + new AppendOnlyFileStoreScan( + mock(ManifestsReader.class), + null, + null, + null, + mock(TableSchema.class), + factory, + 2, + false, + false, + false); + scan.withRowRanges(Collections.singletonList(new Range(100, 100))); + fileIO.reset(); + List result = new ArrayList<>(); + scan.readManifestEntries(metas, false).forEachRemaining(result::add); + assertThat(result).containsExactly(live); + assertThat( + fileIO.opened.stream() + .filter(path -> !path.getName().endsWith(ManifestSidecar.SUFFIX)) + .map(Path::getName)) + .containsExactlyInAnyOrder(metas.get(0).fileName(), metas.get(1).fileName()); + } + + @Test + void testSidecarWriteFailureAbortsAllRollingOutputs() { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + AtomicInteger indexes = new AtomicInteger(); + FileIO failing = + new LocalFileIO() { + @Override + public PositionOutputStream newOutputStream(Path path, boolean overwrite) + throws IOException { + if (path.getName().endsWith(ManifestSidecar.SUFFIX) + && indexes.incrementAndGet() == 2) { + throw new IOException("sidecar write failed"); + } + return super.newOutputStream(path, overwrite); + } + }; + ManifestFile manifests = + createManifestFileFactory(tempDir.toString(), 1, options, failing).create(); + List entries = new ArrayList<>(); + for (int i = 0; i < 2200; i++) { + ManifestEntry entry = gen.next(); + entries.add( + ManifestEntry.create( + FileKind.ADD, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + entry.file().newFirstRowId(0L))); + } + assertThatThrownBy(() -> manifests.write(entries)) + .hasRootCauseMessage("sidecar write failed"); + try (java.util.stream.Stream files = + java.nio.file.Files.list(tempDir.resolve("manifest"))) { + assertThat(files).isEmpty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Test + void testBucketOnlyPlanningAndRawRewriteUseNullableBucketPayload() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); + RecordingFileIO io = new RecordingFileIO(); + ManifestFile.Factory factory = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io); + ManifestFile manifests = factory.create(); + List entries = new ArrayList<>(); + for (int i = 0; i < 4000; i++) { + ManifestEntry entry = gen.next(); + entries.add( + ManifestEntry.create( + FileKind.ADD, entry.partition(), i / 1000, 4, entry.file())); + } + ManifestFileMeta meta = manifests.write(entries).get(0); + AppendOnlyFileStoreScan scan = + new AppendOnlyFileStoreScan( + mock(ManifestsReader.class), + null, + null, + null, + mock(TableSchema.class), + factory, + 2, + false, + false, + false); + scan.withBucket(1); + io.reset(); + assertThat(scan.readManifest(meta)).containsExactlyElementsOf(entries.subList(1000, 2000)); + assertThat(io.bytes.get()).isLessThan(meta.fileSize()); + assertThat(io.seeks).isNotEmpty(); + ManifestAvroWriter writer = manifests.createAvroWriter(); + try (ManifestAvroReader reader = + manifests.scanAvroBlocks(meta.fileName(), meta.fileSize())) { + writer.writeEncodedManifest(reader, meta); + } + writer.close(); + assertThat(scan.readManifest(writer.result().get(0))) + .containsExactlyElementsOf(entries.subList(1000, 2000)); + + ManifestEntry added = entries.get(1000); + ManifestEntry deleted = + ManifestEntry.create(FileKind.DELETE, added.partition(), 1, 4, added.file()); + List changes = new ArrayList<>(); + changes.addAll(manifests.write(Collections.singletonList(added))); + changes.addAll(manifests.write(Collections.singletonList(deleted))); + assertThat(scan.readManifestEntries(changes, false)).isExhausted(); + } + + @Test + void testPartitionOnlyPlanningUsesBlocksWithoutRowIds() { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); + RecordingFileIO io = new RecordingFileIO(); + ManifestFile.Factory factory = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io); + BinaryRow first = gen.next().partition(); + BinaryRow second = gen.next().partition(); + while (second.equals(first)) { + second = gen.next().partition(); + } + List entries = new ArrayList<>(); + for (int i = 0; i < 4000; i++) { + ManifestEntry entry = gen.next(); + entries.add( + ManifestEntry.create( + FileKind.ADD, + i < 1000 ? first : second, + entry.bucket(), + entry.totalBuckets(), + entry.file())); + } + ManifestFileMeta meta = factory.create().write(entries).get(0); + ManifestsReader lists = mock(ManifestsReader.class); + when(lists.partitionFilter()) + .thenReturn( + PartitionPredicate.fromMultiple( + DEFAULT_PART_TYPE, Collections.singletonList(first))); + AppendOnlyFileStoreScan scan = + new AppendOnlyFileStoreScan( + lists, + null, + null, + null, + mock(TableSchema.class), + factory, + 2, + false, + false, + false); + io.reset(); + List actual = scan.readManifest(meta); + assertThat(actual).containsExactlyElementsOf(entries.subList(0, 1000)); + assertThat(io.bytes.get()).isLessThan(meta.fileSize()); + assertThat(io.opened).hasSize(2); + } + + @Test + void testUnknownRowIdKeepsPartitionIndexAndNoQueryDoesNotReadSidecar() { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); + RecordingFileIO fileIO = new RecordingFileIO(); + ManifestFile manifests = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO) + .create(); + ManifestFileMeta meta = manifests.write(Collections.singletonList(gen.next())).get(0); + assertThat(ManifestSidecar.fileName(meta)).isNotNull(); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest") + .resolve(meta.fileName() + ManifestSidecar.SUFFIX))) + .isTrue(); + + fileIO.reset(); + assertThat(manifests.mayContainRowIds(meta, null)).isTrue(); + assertThat(fileIO.opened).isEmpty(); + } + + @Test + void testExplicitIndexReferenceAndNullDoesNotProbe() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); + RecordingFileIO io = new RecordingFileIO(); + ManifestFile manifests = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io).create(); + ManifestEntry original = gen.next(); + ManifestEntry entry = + ManifestEntry.create( + FileKind.ADD, + original.partition(), + original.bucket(), + original.totalBuckets(), + original.file().newFirstRowId(100L)); + ManifestFileMeta written = manifests.write(Collections.singletonList(entry)).get(0); + assertThat(ManifestSidecar.fileName(written)).isNotNull(); + RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(0, 0))); + io.reset(); + for (List extraFiles : + Arrays.asList( + null, + Collections.emptyList(), + Collections.singletonList("other-partition-index"))) { + ManifestFileMeta unindexed = withExtraFiles(written, extraFiles); + assertThat(manifests.selectBlocks(unindexed, query)).isNull(); + assertThat(io.opened).isEmpty(); + } + // An existing suffix-named object must not be inferred as a reference. + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest") + .resolve(ManifestSidecar.fileName(written)))) + .isTrue(); + String explicitName = "custom-index-name" + ManifestSidecar.SUFFIX; + java.nio.file.Files.move( + tempDir.resolve("manifest").resolve(ManifestSidecar.fileName(written)), + tempDir.resolve("manifest").resolve(explicitName)); + String otherName = "other-partition-index"; + java.nio.file.Path otherPath = tempDir.resolve("manifest").resolve(otherName); + java.nio.file.Files.write(otherPath, new byte[] {1, 2, 3}); + ManifestFileMeta indexed = withExtraFiles(written, Arrays.asList(otherName, explicitName)); + assertThat(manifests.selectBlocks(indexed, query).blocks()).isEmpty(); + assertThat(io.opened) + .containsExactly(new Path(tempDir.toString(), "manifest/" + explicitName)); + manifests.delete(indexed); + assertThat(java.nio.file.Files.exists(otherPath)).isFalse(); + assertThat(java.nio.file.Files.exists(tempDir.resolve("manifest").resolve(explicitName))) + .isFalse(); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest").resolve(written.fileName()))) + .isFalse(); + } + + @Test + void testCommitCleanerDeletesExplicitIndexReferences() throws Exception { + ManifestFile manifests = createManifestFile(tempDir.toString(), Long.MAX_VALUE); + ManifestList lists = mock(ManifestList.class); + CommitCleaner cleaner = new CommitCleaner(lists, manifests, mock(IndexManifestFile.class)); + for (int mode = 0; mode < 2; mode++) { + ManifestFileMeta meta = manifests.write(Collections.singletonList(gen.next())).get(0); + String indexName = "commit-index-" + mode + ManifestSidecar.SUFFIX; + Path indexPath = new Path(tempDir.toString(), "manifest/" + indexName); + LocalFileIO.create().newOutputStream(indexPath, false).close(); + String otherName = "commit-other-" + mode; + Path otherPath = new Path(tempDir.toString(), "manifest/" + otherName); + LocalFileIO.create().newOutputStream(otherPath, false).close(); + ManifestFileMeta indexed = withExtraFiles(meta, Arrays.asList(otherName, indexName)); + if (mode == 0) { + when(lists.read("delta-list")).thenReturn(Collections.singletonList(indexed)); + cleaner.cleanUpReuseTmpManifests(Pair.of("delta-list", 1L), null, null, null); + } else { + cleaner.cleanUpNoReuseTmpManifests( + null, Collections.emptyList(), Collections.singletonList(indexed)); + } + assertThat(LocalFileIO.create().exists(indexPath)).isFalse(); + assertThat(LocalFileIO.create().exists(otherPath)).isFalse(); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest").resolve(meta.fileName()))) + .isFalse(); + } + } + + /** Observes actual file access without adding counters to production readers. */ + private static final class RecordingFileIO extends LocalFileIO { + private final List opened = Collections.synchronizedList(new ArrayList<>()); + private final List seeks = Collections.synchronizedList(new ArrayList<>()); + private final AtomicLong bytes = new AtomicLong(); + + private void reset() { + opened.clear(); + seeks.clear(); + bytes.set(0); + } + + @Override + public SeekableInputStream newInputStream(Path path) throws IOException { + opened.add(path); + return new SeekableInputStreamWrapper(super.newInputStream(path)) { + @Override + public void seek(long desired) throws IOException { + seeks.add(desired); + super.seek(desired); + } + + @Override + public int read() throws IOException { + int value = super.read(); + if (value >= 0) { + bytes.incrementAndGet(); + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int n = super.read(buffer, offset, length); + if (n > 0) { + bytes.addAndGet(n); + } + return n; + } + }; + } + } + private ManifestFile createManifestFile(String pathStr) { return createManifestFile(pathStr, ThreadLocalRandom.current().nextInt(8192) + 1024); } private ManifestFile createManifestFile(String pathStr, long suggestedFileSize) { - return createManifestFile(pathStr, suggestedFileSize, null); + return createManifestFile(pathStr, suggestedFileSize, new Options()); } private ManifestFile createManifestFile( String pathStr, long suggestedFileSize, @Nullable SegmentsCache cache) { + return createManifestFileFactory( + pathStr, + suggestedFileSize, + new Options(), + FileIOFinder.find(new Path(pathStr)), + cache) + .create(); + } + + private ManifestFile createManifestFile( + String pathStr, long suggestedFileSize, Options options) { + return createManifestFileFactory( + pathStr, suggestedFileSize, options, FileIOFinder.find(new Path(pathStr))) + .create(); + } + + private ManifestFile.Factory createManifestFileFactory( + String pathStr, long suggestedFileSize, Options options, FileIO fileIO) { + return createManifestFileFactory(pathStr, suggestedFileSize, options, fileIO, null); + } + + private ManifestFile.Factory createManifestFileFactory( + String pathStr, + long suggestedFileSize, + Options options, + FileIO fileIO, + @Nullable SegmentsCache cache) { Path path = new Path(pathStr); FileStorePathFactory pathFactory = new FileStorePathFactory( @@ -1288,7 +1896,6 @@ private ManifestFile createManifestFile( null, false, null); - FileIO fileIO = FileIOFinder.find(path); return new ManifestFile.Factory( fileIO, new FileSystemSchemaManager(fileIO, path), @@ -1298,7 +1905,7 @@ private ManifestFile createManifestFile( pathFactory, suggestedFileSize, cache) - .create(); + .withSidecarOptions(options); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java new file mode 100644 index 000000000000..32a94c9a5bf7 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.FileStore; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.utils.JsonSerdeUtil; +import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.SnapshotManager; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Synthetic index references for manifest serialization and lifecycle tests. */ +public final class ManifestIndexTestUtils { + private ManifestIndexTestUtils() {} + + public static ManifestFileMeta withIndexFileName(ManifestFileMeta meta, String indexFileName) { + return withExtraFiles( + meta, indexFileName == null ? null : Collections.singletonList(indexFileName)); + } + + public static ManifestFileMeta withExtraFiles(ManifestFileMeta meta, List extraFiles) { + return new ManifestFileMeta( + meta.fileName(), + meta.fileSize(), + meta.numAddedFiles(), + meta.numDeletedFiles(), + meta.partitionStats(), + meta.schemaId(), + meta.minBucket(), + meta.maxBucket(), + meta.minLevel(), + meta.maxLevel(), + meta.minRowId(), + meta.maxRowId(), + meta.totalBuckets(), + extraFiles); + } + + /** Replaces only synthetic snapshot fixtures, using newly written manifest lists. */ + public static void registerIndexReferences(FileStore store, long snapshotId) + throws IOException { + SnapshotManager manager = store.snapshotManager(); + FileIO io = manager.fileIO(); + Path snapshotPath = manager.snapshotPath(snapshotId); + ObjectNode snapshot = + (ObjectNode) + JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.readTree( + io.readFileUtf8(snapshotPath)); + ManifestList lists = store.manifestListFactory().create(); + for (String field : + new String[] {"baseManifestList", "deltaManifestList", "changelogManifestList"}) { + JsonNode value = snapshot.get(field); + if (value == null || value.isNull()) { + continue; + } + List indexed = new ArrayList<>(); + for (ManifestFileMeta meta : lists.read(value.asText())) { + // Deliberately use a name which cannot be derived by appending the sidecar suffix. + String name = "index-for-" + meta.fileName() + ManifestSidecar.SUFFIX; + Path index = store.pathFactory().toManifestFilePath(name); + if (!io.exists(index)) { + // GC treats index bytes as opaque; unsupported/partial files are still owned. + io.newOutputStream(index, false).close(); + } + indexed.add(withIndexFileName(meta, name)); + } + Pair replacement = lists.write(indexed); + snapshot.put(field, replacement.getLeft()); + snapshot.put(field + "Size", replacement.getRight()); + } + io.overwriteFileUtf8( + snapshotPath, JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.writeValueAsString(snapshot)); + manager.invalidateCache(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java new file mode 100644 index 000000000000..067a59ce4c8a --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -0,0 +1,747 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.fs.ByteArraySeekableStream; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.MemorySize; +import org.apache.paimon.options.Options; +import org.apache.paimon.utils.IOUtils; +import org.apache.paimon.utils.Range; +import org.apache.paimon.utils.RowRangeIndex; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CancellationException; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Cross-language format, physical block positions, completeness and allocation bounds. */ +class ManifestSidecarTest { + @TempDir java.nio.file.Path temp; + private final ManifestSidecar.Settings settings = new ManifestSidecar.Settings(new Options()); + + static ManifestFileMeta meta(String name, long size, long entries) { + ManifestFileMeta meta = mock(ManifestFileMeta.class); + when(meta.fileName()).thenReturn(name); + when(meta.fileSize()).thenReturn(size); + when(meta.extraFiles()) + .thenReturn(Collections.singletonList(name + ManifestSidecar.SUFFIX)); + when(meta.numAddedFiles()).thenReturn(entries); + return meta; + } + + private Properties fixture() throws IOException { + Properties properties = new Properties(); + try (java.io.InputStream input = getClass().getResourceAsStream("/manifest-sidecar.txt")) { + properties.load(input); + } + return properties; + } + + private byte[] header() throws IOException { + return Base64.getDecoder().decode(fixture().getProperty("avroHeader")); + } + + private byte[] golden() throws IOException { + return Base64.getDecoder().decode(fixture().getProperty("index")); + } + + private ManifestFileMeta goldenMeta() throws IOException { + return meta("manifest-golden", header().length + 400, 7); + } + + @Test + void crossLanguageFormatAndBlockOrdinals() throws Exception { + byte[] header = header(); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + builder.beginBlock(header.length, 100, 3); + builder.add(0L, 10); + builder.add(5L, 5); + builder.add(20L, 5); + builder.endBlock(); + builder.beginBlock(header.length + 100, 200, 2); + builder.add((1L << 32) - 2, 5); + builder.add(8254058425445L, 1); + builder.endBlock(); + builder.beginBlock(header.length + 300, 100, 2); + builder.add(20L, 5); + builder.add(Long.MAX_VALUE, 1); + builder.endBlock(); + byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); + assertThat(data).isEqualTo(golden()); + ManifestFileMeta meta = goldenMeta(); + for (long point : + new long[] { + 0, + 9, + 20, + 24, + (1L << 32) - 2, + 1L << 32, + (1L << 32) + 2, + 8254058425445L, + Long.MAX_VALUE + }) { + assertThat(select(data, meta, point).blocks()).as("row %s", point).isNotEmpty(); + } + for (long point : + new long[] { + 10, 19, 25, (1L << 32) - 3, (1L << 32) + 3, 8254058425444L, Long.MAX_VALUE - 1 + }) { + assertThat(select(data, meta, point).blocks()).as("row %s", point).isEmpty(); + } + ManifestSidecar.Selection selected = select(data, meta, 20); + assertThat(selected.blocks()).extracting(b -> b.firstRecord).containsExactly(0L, 5L); + assertThat(selected.blocks()) + .extracting(b -> b.offset) + .containsExactly((long) header.length, header.length + 300L); + assertThat(selected.blocks()).extracting(b -> b.length).containsExactly(100L, 100L); + + ManifestSidecar.Selection gap = select(data, meta, 16); + + assertThat(gap.blocks()).isEmpty(); + RowRangeIndex query = + RowRangeIndex.create(Arrays.asList(new Range(10, 19), new Range(25, 40))); + assertThat(ManifestSidecar.select(data, meta, query, settings).blocks()).isEmpty(); + assertThat(query.ranges()).containsExactly(new Range(10, 19), new Range(25, 40)); + } + + @Test + void settingsUseMemorySizesAndFollowTheManifestTarget() { + Options options = new Options(); + assertThat(new ManifestSidecar.Settings(options).maxBytes).isEqualTo(16 * 1024 * 1024); + options.setString(CoreOptions.MANIFEST_TARGET_FILE_SIZE.key(), "12 mb"); + assertThat(new ManifestSidecar.Settings(options).maxBytes).isEqualTo(24 * 1024 * 1024); + options.setString(CoreOptions.MANIFEST_TARGET_FILE_SIZE.key(), "64 mb"); + assertThat(new ManifestSidecar.Settings(options).maxBytes).isEqualTo(128 * 1024 * 1024); + + options.setString(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES.key(), "512 kb"); + options.setString(CoreOptions.MANIFEST_TARGET_FILE_SIZE.key(), "1 gb"); + assertThat(new ManifestSidecar.Settings(options).maxBytes).isEqualTo(512 * 1024); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(Integer.MAX_VALUE - 1L)); + assertThat(new ManifestSidecar.Settings(options).maxBytes).isEqualTo(Integer.MAX_VALUE - 1); + for (String value : new String[] {"127 bytes", "2147483647 bytes", "2 gb"}) { + options.setString(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES.key(), value); + assertThatThrownBy(() -> new ManifestSidecar.Settings(options)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("manifest.sidecar.max-bytes"); + } + } + + @Test + void minMaxSkipsExactIntersectionChecksAndHandlesOneInterval() throws Exception { + byte[] header = header(); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + builder.beginBlock(header.length, 100, 2); + builder.add(0L, 10); + builder.add(20L, 10); + builder.endBlock(); + builder.beginBlock(header.length + 100, 100, 2); + builder.add(100L, 10); + builder.add(200L, 10); + builder.endBlock(); + builder.beginBlock(header.length + 200, 100, 1); + builder.add(1L << 32, 10); + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 300, 5); + ManifestFileMeta meta = meta("m", header.length + 300, 5); + RowRangeIndex outside = + spy(RowRangeIndex.create(Collections.singletonList(new Range(50, 59)))); + ManifestSidecar.Selection none = ManifestSidecar.select(data, meta, outside, settings); + assertThat(none.blocks()).isEmpty(); + + // Only the three envelopes are tested; no individual interval intersection is evaluated. + verify(outside, times(3)).intersects(anyLong(), anyLong()); + verify(outside).intersects(0, 29); + verify(outside).intersects(100, 209); + verify(outside).intersects(1L << 32, (1L << 32) + 9); + + RowRangeIndex one = + spy( + RowRangeIndex.create( + Collections.singletonList( + new Range((1L << 32) + 9, (1L << 32) + 9)))); + ManifestSidecar.Selection hit = ManifestSidecar.select(data, meta, one, settings); + assertThat(hit.blocks()).extracting(b -> b.firstRecord).containsExactly(4L); + + // A one-interval block needs no second intersection check after its envelope matches. + verify(one, times(3)).intersects(anyLong(), anyLong()); + } + + @Test + void singleIntervalHandlesBoundariesAndAbsentQueries() throws Exception { + byte[] header = header(); + for (Range range : + Arrays.asList( + new Range(0, 0), + new Range(42, 51), + new Range(Long.MAX_VALUE, Long.MAX_VALUE))) { + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + builder.beginBlock(header.length, 100, 1); + builder.add(range.from, range.to - range.from + 1); + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 100, 1); + ManifestFileMeta meta = meta("m", header.length + 100, 1); + assertThat(ManifestSidecar.select(data, meta, null, settings).blocks()).hasSize(1); + assertThat( + ManifestSidecar.select( + data, + meta, + RowRangeIndex.create(Collections.emptyList()), + settings) + .blocks()) + .isEmpty(); + for (long point : new long[] {range.from, range.to}) { + RowRangeIndex query = + spy( + RowRangeIndex.create( + Collections.singletonList(new Range(point, point)))); + assertThat(ManifestSidecar.select(data, meta, query, settings).blocks()).hasSize(1); + verify(query).intersects(range.from, range.to); + } + long missing = range.from > 0 ? range.from - 1 : range.to + 1; + assertThat(select(data, meta, missing).blocks()).isEmpty(); + } + } + + @Test + void malformedIntervalsStillFallbackAfterMinMaxRejectionOrAnEarlyHit() throws Exception { + int firstBlockIntervals = 60 + 4 + header().length + 4 + 4 + 24 + 5 + 5 + 4; + ManifestFileMeta meta = goldenMeta(); + for (long[] mutation : new long[][] {{0, -1}, {8, -1}, {8, 30}, {16, 9}, {24, 19}}) { + byte[] data = golden(); + ByteBuffer.wrap(data).putLong(firstBlockIntervals + (int) mutation[0], mutation[1]); + byte[] hash = + MessageDigest.getInstance("SHA-256") + .digest(Arrays.copyOf(data, data.length - 32)); + System.arraycopy(hash, 0, data, data.length - 32, 32); + Files.write(temp.resolve("manifest-golden" + ManifestSidecar.SUFFIX), data); + for (RowRangeIndex query : + Arrays.asList( + RowRangeIndex.create(Collections.singletonList(new Range(30, 30))), + RowRangeIndex.create(Collections.singletonList(new Range(0, 0))), + RowRangeIndex.create(Collections.emptyList()), + null)) { + assertThat( + ManifestSidecar.read( + LocalFileIO.create(), + new Path(temp.toString(), "manifest-golden"), + meta, + query, + settings)) + .isNull(); + } + } + } + + @Test + void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception { + byte[] header = header(); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + builder.beginBlock(header.length, 100, 2); + builder.add(0L, Long.MAX_VALUE); + builder.add(Long.MAX_VALUE, 1); + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 100, 2); + assertThat(data.length).isLessThan(512); + assertThat(select(data, meta("m", header.length + 100, 2), Long.MAX_VALUE).blocks()) + .hasSize(1); + for (Long first : Arrays.asList(null, -1L, Long.MAX_VALUE)) { + builder = new ManifestSidecar.Builder(settings, header); + builder.beginBlock(header.length, 100, 1); + builder.add(first, 2); + builder.endBlock(); + assertThat( + select( + builder.serialize("m", header.length + 100, 1), + meta("m", header.length + 100, 1), + 100) + .blocks()) + .hasSize(1); + } + for (long count : new long[] {0, -1}) { + builder = new ManifestSidecar.Builder(settings, header); + builder.beginBlock(header.length, 100, 1); + builder.add(0L, count); + builder.endBlock(); + assertThat( + select( + builder.serialize("m", header.length + 100, 1), + meta("m", header.length + 100, 1), + 100) + .blocks()) + .hasSize(1); + } + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(512)); + builder = new ManifestSidecar.Builder(new ManifestSidecar.Settings(options), header); + builder.beginBlock(header.length, 100, 64); + for (int i = 0; i < 64; i++) { + builder.add(i * 10L, 1); + } + builder.endBlock(); + assertThat( + select( + builder.serialize("m", header.length + 100, 64), + meta("m", header.length + 100, 64), + 5) + .blocks()) + .hasSize(1); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(128)); + builder = new ManifestSidecar.Builder(new ManifestSidecar.Settings(options), header); + assertThat(builder.serialize("m", 1, 2)).isNull(); + } + + @Test + void corruptMissingIncompleteAndMismatchedIndexesFallback() throws Exception { + Path manifest = new Path(temp.toString(), "manifest-golden"); + java.nio.file.Path index = temp.resolve("manifest-golden" + ManifestSidecar.SUFFIX); + ManifestFileMeta meta = goldenMeta(); + RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(11, 11))); + + assertThat(ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query, settings)) + .isNull(); + byte[] good = golden(); + for (int position : new int[] {0, 9, 11, 15, 16, 55, 63, 67, 75, good.length - 1}) { + byte[] bad = good.clone(); + bad[position] ^= 2; + Files.write(index, bad); + assertThat(ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query, settings)) + .isNull(); + } + // A valid checksum cannot make an unsupported container version readable. + for (int version : new int[] {0, 2, 99}) { + byte[] bad = good.clone(); + ByteBuffer.wrap(bad).putInt(8, version); + byte[] hash = + MessageDigest.getInstance("SHA-256") + .digest(Arrays.copyOf(bad, bad.length - 32)); + System.arraycopy(hash, 0, bad, bad.length - 32, 32); + assertThatThrownBy(() -> ManifestSidecar.select(bad, meta, query, settings)) + .isInstanceOf(IOException.class); + } + Files.write(index, Arrays.copyOf(good, good.length - 1)); + assertThat(ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query, settings)) + .isNull(); + Files.write(index, good); + assertThat( + ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query, settings) + .blocks()) + .isEmpty(); + assertThatThrownBy( + () -> + ManifestSidecar.select( + good, meta("other", meta.fileSize(), 7), query, settings)) + .isInstanceOf(IOException.class); + } + + @Test + void ioTimeoutFallsBackButInterruptionAndFatalErrorsPropagate() { + ManifestFileMeta manifest = meta("m", 1, 1); + RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(1, 1))); + Path path = new Path(temp.toString(), "m"); + + LocalFileIO timedOut = + new LocalFileIO() { + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) + throws IOException { + throw new java.net.SocketTimeoutException("timeout"); + } + }; + assertThat(ManifestSidecar.read(timedOut, path, manifest, query, settings)).isNull(); + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + LocalFileIO interrupted = + new LocalFileIO() { + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) + throws IOException { + throw new java.io.InterruptedIOException("stop"); + } + }; + try { + assertThatThrownBy( + () -> + ManifestSidecar.read( + interrupted, path, manifest, query, settings)) + .isInstanceOf(java.io.UncheckedIOException.class); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + LocalFileIO failed = + new LocalFileIO() { + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) { + throw new AssertionError("fatal"); + } + }; + assertThatThrownBy(() -> ManifestSidecar.read(failed, path, manifest, query, settings)) + .isInstanceOf(AssertionError.class); + } + + @Test + void suppressedCancellationInterruptionAndFatalErrorsPropagate() { + ManifestFileMeta manifest = meta("m", 1, 1); + RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(1, 1))); + for (Throwable closeFailure : + Arrays.asList( + new CancellationException("cancelled"), + new java.io.InterruptedIOException("interrupted"), + new AssertionError("fatal"))) { + AtomicBoolean closed = new AtomicBoolean(); + LocalFileIO fileIO = + new LocalFileIO() { + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) { + return new ByteArraySeekableStream(new byte[0]) { + @Override + public int read(byte[] bytes, int offset, int length) + throws IOException { + throw new IOException("read failed"); + } + + @Override + public void close() throws IOException { + super.close(); + closed.set(true); + if (closeFailure instanceof IOException) { + throw (IOException) closeFailure; + } + if (closeFailure instanceof Error) { + throw (Error) closeFailure; + } + throw (RuntimeException) closeFailure; + } + }; + } + }; + try { + assertThatThrownBy( + () -> + ManifestSidecar.read( + fileIO, + new Path(temp.toString(), "m"), + manifest, + query, + settings)) + .isInstanceOf( + closeFailure instanceof java.io.InterruptedIOException + ? java.io.UncheckedIOException.class + : closeFailure.getClass()); + assertThat(Thread.currentThread().isInterrupted()) + .isEqualTo(closeFailure instanceof java.io.InterruptedIOException); + assertThat(closed).isTrue(); + } finally { + Thread.interrupted(); + } + } + } + + @Test + void ordinaryExceptionCyclesFallBack() { + IOException first = new IOException("first"); + IOException second = new IOException("second"); + first.initCause(second); + second.addSuppressed(first); + LocalFileIO fileIO = + new LocalFileIO() { + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) + throws IOException { + throw first; + } + }; + assertThat( + ManifestSidecar.read( + fileIO, + new Path(temp.toString(), "m"), + meta("m", 1, 1), + RowRangeIndex.create(Collections.singletonList(new Range(1, 1))), + settings)) + .isNull(); + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + } + + @Test + void indexReadsUseBoundedBulkRequests() throws Exception { + byte[] header = header(); + for (int blockCount : new int[] {5000, 25000}) { + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + for (int blockNumber = 0; blockNumber < blockCount; blockNumber++) { + builder.beginBlock(header.length + blockNumber * 100L, 100, 1); + builder.add((long) blockNumber, 1); + builder.endBlock(); + } + long size = header.length + blockCount * 100L; + byte[] data = builder.serialize("manifest-large", size, blockCount); + ManifestFileMeta meta = meta("manifest-large", size, blockCount); + CountingInput stream = new CountingInput(data, Integer.MAX_VALUE); + Path path = new Path(temp.toString(), meta.fileName()); + FileIO io = mock(FileIO.class); + when(io.newInputStream(ManifestSidecar.path(path))).thenReturn(stream); + ManifestSidecar.Selection actual = + ManifestSidecar.read( + io, + path, + meta, + RowRangeIndex.create(Collections.singletonList(new Range(0, 0))), + settings); + assertThat(actual.blocks()).hasSize(1); + assertThat(actual.blocks().get(0).offset).isEqualTo(header.length); + assertThat(stream.readLengths).hasSize((data.length + (1 << 20) - 1) / (1 << 20)); + assertThat(stream.requests).allMatch(request -> request <= 1 << 20); + assertThat(stream.closed).isTrue(); + } + } + + @Test + void indexShortReadsAndExactBudget() throws Exception { + byte[] data = golden(); + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(data.length)); + Path path = new Path(temp.toString(), "manifest-golden"); + for (int maxRead : new int[] {Integer.MAX_VALUE, 7}) { + CountingInput stream = new CountingInput(data, maxRead); + FileIO io = mock(FileIO.class); + when(io.newInputStream(ManifestSidecar.path(path))).thenReturn(stream); + ManifestSidecar.Selection actual = + ManifestSidecar.read( + io, + path, + goldenMeta(), + RowRangeIndex.create(Collections.singletonList(new Range(20, 20))), + new ManifestSidecar.Settings(options)); + assertThat(actual.blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 5L); + assertThat(stream.closed).isTrue(); + } + } + + @Test + void indexOverBudgetStopsAfterOneExtraByte() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(128)); + Path path = new Path(temp.toString(), "manifest-golden"); + CountingInput stream = new CountingInput(golden(), Integer.MAX_VALUE); + FileIO io = mock(FileIO.class); + when(io.newInputStream(ManifestSidecar.path(path))).thenReturn(stream); + assertThat( + ManifestSidecar.read( + io, + path, + goldenMeta(), + RowRangeIndex.create(Collections.singletonList(new Range(20, 20))), + new ManifestSidecar.Settings(options))) + .isNull(); + assertThat(stream.readLengths).containsExactly(129); + assertThat(stream.closed).isTrue(); + } + + @Test + void adjacentBlocksShareReadsForSingleByteConsumers() throws Exception { + byte[] header = header(); + byte[] body = new byte[400]; + for (int position = 0; position < body.length; position++) { + body[position] = (byte) position; + } + ManifestSidecar.Selection selected = + ManifestSidecar.select( + golden(), + goldenMeta(), + RowRangeIndex.create( + Arrays.asList( + new Range(0, 0), + new Range(8254058425445L, 8254058425445L))), + settings); + byte[] manifest = Arrays.copyOf(header, header.length + body.length); + System.arraycopy(body, 0, manifest, header.length, body.length); + CountingInput stream = new CountingInput(manifest, Integer.MAX_VALUE); + Path path = new Path(temp.toString(), "manifest-golden"); + FileIO io = mock(FileIO.class); + when(io.newInputStream(path)).thenReturn(stream); + ByteArrayOutputStream actual = new ByteArrayOutputStream(); + try (InputStream input = ManifestSidecar.openManifest(io, path, selected)) { + int value; + while ((value = input.read()) != -1) { + actual.write(value); + } + assertThat(input.read(new byte[1], 0, 0)).isZero(); + } + assertThat(actual.toByteArray()).isEqualTo(Arrays.copyOf(manifest, header.length + 300)); + assertThat(stream.readLengths).containsExactly(300); + assertThat(stream.seeks).containsExactly((long) header.length); + assertThat(stream.closed).isTrue(); + } + + @Test + void blockReadsSkipGapsAndEmptySelections() throws Exception { + byte[] header = header(); + byte[] manifest = Arrays.copyOf(header, header.length + 400); + Arrays.fill(manifest, header.length + 100, header.length + 300, (byte) 7); + Path path = new Path(temp.toString(), "manifest-golden"); + for (long point : new long[] {20, 16}) { + CountingInput stream = new CountingInput(manifest, Integer.MAX_VALUE); + FileIO io = mock(FileIO.class); + when(io.newInputStream(path)).thenReturn(stream); + byte[] actual; + try (InputStream input = + ManifestSidecar.openManifest(io, path, select(golden(), goldenMeta(), point))) { + actual = IOUtils.readFully(input, false); + } + if (point == 20) { + assertThat(actual).isEqualTo(Arrays.copyOf(header, header.length + 200)); + assertThat(stream.readLengths).containsExactly(100, 100); + assertThat(stream.seeks) + .containsExactly((long) header.length, header.length + 300L); + } else { + assertThat(actual).isEqualTo(header); + assertThat(stream.readLengths).isEmpty(); + assertThat(stream.seeks).isEmpty(); + } + assertThat(stream.closed).isTrue(); + } + } + + @Test + void largeBlockSpansUseBoundedReads() throws Exception { + byte[] header = header(); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + long offset = header.length; + for (int length : new int[] {512 * 1024, 512 * 1024, 257}) { + builder.beginBlock(offset, length, 1); + builder.add(20L, 1); + builder.endBlock(); + offset += length; + } + byte[] data = builder.serialize("manifest-large", offset, 3); + byte[] manifest = Arrays.copyOf(header, (int) offset); + CountingInput stream = new CountingInput(manifest, Integer.MAX_VALUE); + FileIO io = mock(FileIO.class); + Path path = new Path(temp.toString(), "manifest-large"); + when(io.newInputStream(path)).thenReturn(stream); + try (InputStream input = + ManifestSidecar.openManifest( + io, path, select(data, meta("manifest-large", offset, 3), 20))) { + assertThat(IOUtils.readFully(input, false)).isEqualTo(manifest); + } + assertThat(stream.readLengths).containsExactly(1 << 20, 257); + assertThat(stream.seeks).containsExactly((long) header.length); + assertThat(stream.closed).isTrue(); + } + + @Test + void blockShortReadsAndTruncation() throws Exception { + byte[] header = header(); + Path path = new Path(temp.toString(), "manifest-golden"); + ManifestSidecar.Selection selected = + ManifestSidecar.select( + golden(), + goldenMeta(), + RowRangeIndex.create( + Collections.singletonList(new Range(0, Long.MAX_VALUE))), + settings); + for (int bodyLength : new int[] {400, 399}) { + byte[] manifest = Arrays.copyOf(header, header.length + bodyLength); + CountingInput stream = new CountingInput(manifest, 7); + FileIO io = mock(FileIO.class); + when(io.newInputStream(path)).thenReturn(stream); + try (InputStream input = ManifestSidecar.openManifest(io, path, selected)) { + if (bodyLength == 400) { + assertThat(IOUtils.readFully(input, false)).isEqualTo(manifest); + } else { + assertThatThrownBy(() -> IOUtils.readFully(input, false)) + .isInstanceOf(EOFException.class); + } + } + assertThat(stream.closed).isTrue(); + } + } + + private static class CountingInput extends ByteArraySeekableStream { + private final int maxRead; + private final List requests = new ArrayList<>(); + private final List readLengths = new ArrayList<>(); + private final List seeks = new ArrayList<>(); + private boolean closed; + + private CountingInput(byte[] data, int maxRead) { + super(data); + this.maxRead = maxRead; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + requests.add(length); + int count = super.read(bytes, offset, Math.min(length, maxRead)); + if (count > 0) { + readLengths.add(count); + } + return count; + } + + @Override + public void seek(long position) throws IOException { + seeks.add(position); + super.seek(position); + } + + @Override + public void close() throws IOException { + closed = true; + super.close(); + } + } + + private ManifestSidecar.Selection select(byte[] data, ManifestFileMeta meta, long point) + throws IOException { + return ManifestSidecar.select( + data, + meta, + RowRangeIndex.create(Collections.singletonList(new Range(point, point))), + settings); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java index 24a6e0eaea46..957e4b73177c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java @@ -38,8 +38,11 @@ import org.apache.paimon.manifest.FileSource; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.manifest.ManifestIndexTestUtils; +import org.apache.paimon.manifest.ManifestSidecar; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.options.ExpireConfig; +import org.apache.paimon.options.MemorySize; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; @@ -854,6 +857,75 @@ public void testExpirePlansManifestsConcurrentlyWithSkippingSet() throws Excepti store.assertCleaned(); } + @Test + void testSidecarsFollowSnapshotAndTagRetention() throws Exception { + store.options().toConfiguration().set(CoreOptions.MANIFEST_MERGE_MIN_COUNT, 2); + store.options() + .toConfiguration() + .set(CoreOptions.MANIFEST_TARGET_FILE_SIZE, MemorySize.parse("8 mb")); + List allData = new ArrayList<>(); + List snapshotPositions = new ArrayList<>(); + commit(8, allData, snapshotPositions); + int latest = requireNonNull(snapshotManager.latestSnapshotId()).intValue(); + Set manifests = new HashSet<>(); + Set retainedManifests = new HashSet<>(); + for (int snapshotId = 1; snapshotId <= latest; snapshotId++) { + rewriteSnapshotTime(snapshotId, 0); + ManifestIndexTestUtils.registerIndexReferences(store, snapshotId); + snapshotManager.invalidateCache(); + for (ManifestFileMeta meta : + store.manifestListFactory() + .create() + .readDataManifests(snapshotManager.snapshot(snapshotId))) { + Path manifest = store.pathFactory().toManifestFilePath(meta.fileName()); + manifests.add(manifest); + if (snapshotId == 3 || snapshotId == latest) { + retainedManifests.add(manifest); + } + } + } + store.newTagManager() + .createTag( + snapshotManager.snapshot(3), + "keep-sidecars", + store.options().tagDefaultTimeRetained(), + Collections.emptyList(), + false); + Set expiredManifests = new HashSet<>(manifests); + expiredManifests.removeAll(retainedManifests); + assertThat(expiredManifests).isNotEmpty(); + ExpireSnapshotsImpl expire = + (ExpireSnapshotsImpl) store.newExpire(expireAllButLatestConfig()); + expire.setCurrentTimeMillis(() -> 1000L); + expire.expire(); + for (Path manifest : manifests) { + boolean retained = retainedManifests.contains(manifest); + assertThat(fileIO.exists(manifest)).as("manifest %s", manifest).isEqualTo(retained); + assertThat( + fileIO.exists( + new Path( + manifest.getParent(), + "index-for-" + + manifest.getName() + + ManifestSidecar.SUFFIX))) + .as("sidecar for %s", manifest) + .isEqualTo(retained); + } + for (ManifestFileMeta meta : + store.manifestListFactory() + .create() + .readDataManifests( + store.newTagManager() + .getOrThrow("keep-sidecars") + .trimToSnapshot())) { + assertThat( + fileIO.exists( + store.pathFactory() + .toManifestFilePath(ManifestSidecar.fileName(meta)))) + .isTrue(); + } + } + @Test public void testExpireWithTagsAndConcurrentPlanningKeepsTaggedSnapshotsReadable() throws Exception { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java index 9e3bf05b69ae..49e22ee551e1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java @@ -32,7 +32,9 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.manifest.ManifestIndexTestUtils; import org.apache.paimon.manifest.ManifestList; +import org.apache.paimon.manifest.ManifestSidecar; import org.apache.paimon.mergetree.compact.ConcatRecordReader; import org.apache.paimon.options.Options; import org.apache.paimon.reader.ReaderSupplier; @@ -158,6 +160,39 @@ public void testNormallyRemoving() throws Throwable { normallyRemoving(tablePath); } + @Test + void testOrphanCleanupProtectsReferencedSidecars() throws Exception { + commit(Collections.singletonList(TestPojo.next())); + ManifestIndexTestUtils.registerIndexReferences( + table.store(), table.snapshotManager().latestSnapshotId()); + table.snapshotManager().invalidateCache(); + table.createTag("sidecar-tag", table.snapshotManager().latestSnapshotId()); + List sidecars = new ArrayList<>(); + List unreferenced = new ArrayList<>(); + for (ManifestFileMeta meta : + table.store() + .manifestListFactory() + .create() + .readDataManifests(table.snapshotManager().latestSnapshot())) { + Path sidecar = new Path(manifestDir, ManifestSidecar.fileName(meta)); + sidecars.add(sidecar); + Path guessed = new Path(manifestDir, meta.fileName() + ManifestSidecar.SUFFIX); + fileIO.newOutputStream(guessed, false).close(); + unreferenced.add(guessed); + } + Path orphan = new Path(manifestDir, "manifest-orphan" + ManifestSidecar.SUFFIX); + fileIO.newOutputStream(orphan, false).close(); + new LocalOrphanFilesClean(table, System.currentTimeMillis() + 2000).clean(); + assertThat(fileIO.exists(orphan)).isFalse(); + assertThat(sidecars).isNotEmpty(); + for (Path sidecar : sidecars) { + assertThat(fileIO.exists(sidecar)).isTrue(); + } + for (Path guessed : unreferenced) { + assertThat(fileIO.exists(guessed)).isFalse(); + } + } + @Test public void testKeepManagedBlobPack() throws Exception { commit(Collections.singletonList(TestPojo.next())); diff --git a/paimon-core/src/test/resources/manifest-sidecar.txt b/paimon-core/src/test/resources/manifest-sidecar.txt new file mode 100644 index 000000000000..f8a914f451d8 --- /dev/null +++ b/paimon-core/src/test/resources/manifest-sidecar.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +avroHeader=T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAA +partitionA=AAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhA== +partitionB=AAAAAgACAAAAAAAACQAAAAAAAAAAAAAAAAAAAA== +index=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGAAAAAAAAAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAAAAAAABAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZQAAAAAAAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAAAAAAABAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////wAAAAAAvl+JhwByNVfzaEZmgXTZEMYB85Mh4bR1hf61USimKec= +indexWithPartitions=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAAAAAAAAAAAACdAAAAAAAAAMgAAAAAAAAAAgEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAP////4AAAABAAAAAgAAB4HMOGxlAAAHgcw4bGUAAAAAAAAAAAAAAAFlAAAAAAAAAGQAAAAAAAAAAgEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAABQAAAAAAAAAGH//////////f/////////8AAAAAACeaHd6e41hdWskZ6DqWibQKE2TYT04EVjoN1YBmh9VF +indexWithBuckets=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgBAAAAFAAAAAIAAAABAAAABAAAAAEAAAAIAAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZQEAAAAUAAAAAgAAAAIAAAAEAAAAAgAAAAgAAAAAAAABZQAAAAAAAABkAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3//////////AQAAABQAAAACAAAAAAAAAAEAAAADAAAABGCAy0ixcqKywlCau1+E/XLEorySmCmL1zl5q9/dwdBu diff --git a/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java index 43a68c54a44d..382f55f93d60 100644 --- a/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java +++ b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java @@ -22,27 +22,107 @@ import org.apache.avro.io.DatumReader; import org.apache.avro.io.Decoder; +import java.io.ByteArrayOutputStream; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; /** Package bridge exposing Avro's compressed blocks without reflection. */ public final class RawBlockReader extends DataFileStream { + private final CountingInput input; + private final byte[] headerBytes; + private long blockOffset; + private long blockLength; + private boolean pending; + public RawBlockReader(InputStream input) throws IOException { + this(new CountingInput(input)); + } + + private RawBlockReader(CountingInput input) throws IOException { super(input, new NoOpDatumReader()); + this.input = input; + long length = position(); + this.headerBytes = + length <= CountingInput.MAX_HEADER + ? Arrays.copyOf(input.prefix.toByteArray(), (int) length) + : null; + input.prefix = null; + } + + public byte[] headerBytes() { + return headerBytes == null ? null : headerBytes.clone(); + } + + public long blockOffset() { + return blockOffset; + } + + public long blockLength() { + return blockLength; } - public boolean hasNextRawBlock() { - return super.hasNextBlock(); + private long position() throws IOException { + // This is the same read-ahead adjustment used by DataFileReader.blockFinished(). + return input.position - vin.inputStream().available(); + } + + public boolean hasNextRawBlock() throws IOException { + if (!pending) { + blockOffset = position(); + pending = super.hasNextBlock(); + } + return pending; } public RawBlock nextRawBlock(RawBlock reuse) throws IOException { + if (!hasNextRawBlock()) { + throw new java.util.NoSuchElementException(); + } DataBlock raw = super.nextRawBlock(reuse == null ? null : reuse.dataBlock()); + blockLength = position() - blockOffset; + pending = false; return reuse == null ? new RawBlock(raw, resolveCodec(), getSchema()) : reuse.replace(raw, resolveCodec(), getSchema()); } + private static final class CountingInput extends FilterInputStream { + private static final int MAX_HEADER = 1024 * 1024; + private long position; + private ByteArrayOutputStream prefix = new ByteArrayOutputStream(); + + private CountingInput(InputStream input) { + super(input); + } + + @Override + public int read() throws IOException { + int value = in.read(); + if (value >= 0) { + position++; + if (prefix != null && prefix.size() < MAX_HEADER) { + prefix.write(value); + } + } + return value; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int n = in.read(bytes, offset, length); + if (n > 0) { + position += n; + if (prefix != null && prefix.size() < MAX_HEADER) { + prefix.write(bytes, offset, Math.min(n, MAX_HEADER - prefix.size())); + } + } + return n; + } + } + private static final class NoOpDatumReader implements DatumReader { @Override diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java index c4359afcda43..eeca62bb18c8 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java @@ -54,6 +54,19 @@ public AvroBlockReader(InputStream input) throws IOException { } } + @Nullable + public byte[] headerBytes() { + return reader.headerBytes(); + } + + public long blockOffset() { + return reader.blockOffset(); + } + + public long blockLength() { + return reader.blockLength(); + } + /** Creates a record decoder from the writer schema stored in the Avro file header. */ public AvroRecordDecoder createRecordDecoder() { return new AvroRecordDecoder(reader.getSchema()); diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index 903697a94ff2..d6c5193e09ff 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -297,6 +297,28 @@ class CoreOptions: .with_description("The parallelism for scanning manifest files.") ) + MANIFEST_SIDECAR_WRITE: ConfigOption[bool] = ( + ConfigOptions.key("manifest.sidecar.write") + .boolean_type() + .default_value(False) + ) + + MANIFEST_SIDECAR_READ: ConfigOption[bool] = ( + ConfigOptions.key("manifest.sidecar.read") + .boolean_type() + .default_value(False) + ) + + MANIFEST_SIDECAR_MAX_BYTES: ConfigOption[MemorySize] = ( + ConfigOptions.key("manifest.sidecar.max-bytes") + .memory_type() + .no_default_value() + .with_description( + "Maximum serialized manifest sidecar size, including header and checksum. " + "Defaults to twice manifest.target-file-size." + ) + ) + MANIFEST_COMPRESSION: ConfigOption[str] = ( ConfigOptions.key("manifest.compression") .string_type() @@ -1222,6 +1244,10 @@ def manifest_target_size(self, default=None): default = MemorySize.of_bytes(default) if isinstance(default, int) else MemorySize.parse(default) return self.options.get(CoreOptions.MANIFEST_TARGET_FILE_SIZE, default).get_bytes() + def manifest_sidecar_max_size(self): + size = self.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES) + return size.get_bytes() if size is not None else 2 * self.manifest_target_size() + def manifest_merge_skip_on_write_only(self, default=None): return self.options.get(CoreOptions.MANIFEST_MERGE_SKIP_ON_WRITE_ONLY, default) diff --git a/paimon-python/pypaimon/manifest/manifest_file_manager.py b/paimon-python/pypaimon/manifest/manifest_file_manager.py index a3ac5cfe7434..bd2cb09cfccd 100644 --- a/paimon-python/pypaimon/manifest/manifest_file_manager.py +++ b/paimon-python/pypaimon/manifest/manifest_file_manager.py @@ -19,6 +19,10 @@ from io import BytesIO from typing import Callable, List, Optional +from pypaimon.manifest.manifest_sidecar import ( + Settings, SUFFIX, Query, build_from_entries, read_sidecar, read_selected_bytes, +) + import fastavro from datetime import datetime @@ -57,13 +61,27 @@ def read_entries_parallel(self, manifest_files: List[ManifestFileMeta], manifest early_entry_filter: Optional[Callable[[int, int], bool]] = None, early_record_filter: Optional[Callable[[dict], bool]] = None, partition_filter=None, + row_ranges=None, ) -> List[ManifestEntry]: - def _process_single_manifest(manifest_file: ManifestFileMeta) -> List[ManifestEntry]: - return self.read(manifest_file.file_name, manifest_entry_filter, drop_stats, - early_entry_filter=early_entry_filter, - early_record_filter=early_record_filter, - partition_filter=partition_filter) + settings = Settings.from_options(self.table.options) + query = Query(row_ranges) if settings.read and row_ranges is not None else None + + def _process_single_manifest(manifest_file: ManifestFileMeta): + path = f"{self.manifest_path}/{manifest_file.file_name}" + selected = None + if settings.read and (query is not None or partition_filter is not None + or early_entry_filter is not None): + selected = read_sidecar(self.file_io, path, manifest_file, query, settings, + partition_filter, self.partition_keys_fields, early_entry_filter) + if selected is not None and not selected.blocks: + return [] + return self.read( + manifest_file.file_name, manifest_entry_filter, drop_stats, + early_entry_filter=early_entry_filter, + early_record_filter=early_record_filter, + partition_filter=partition_filter, + selected_blocks=selected) def _entry_identifier(e: ManifestEntry) -> tuple: return ( @@ -97,6 +115,7 @@ def read(self, manifest_file_name: str, manifest_entry_filter=None, drop_stats=T early_entry_filter: Optional[Callable[[int, int], bool]] = None, early_record_filter: Optional[Callable[[dict], bool]] = None, partition_filter=None, + selected_blocks=None, ) -> List[ManifestEntry]: """ early_entry_filter: ``(bucket, total_buckets) -> bool``, skip before deserializing _FILE. @@ -110,8 +129,11 @@ def read(self, manifest_file_name: str, manifest_entry_filter=None, drop_stats=T manifest_file_path = f"{self.manifest_path}/{manifest_file_name}" entries = [] - with self.file_io.new_input_stream(manifest_file_path) as input_stream: - avro_bytes = input_stream.read() + if selected_blocks is not None: + avro_bytes = read_selected_bytes(self.file_io, manifest_file_path, selected_blocks) + else: + with self.file_io.new_input_stream(manifest_file_path) as input_stream: + avro_bytes = input_stream.read() buffer = BytesIO(avro_bytes) reader = fastavro.reader(buffer) @@ -244,7 +266,7 @@ def write(self, file_name, entries: List[ManifestEntry]): fastavro.writer( buf, MANIFEST_ENTRY_SCHEMA, self._to_avro_records(entries), codec=self._codec) - self._flush(file_name, buf.getvalue()) + return self._flush(file_name, buf.getvalue(), entries) def rolling_write(self, entries: List[ManifestEntry], suggested_file_size: int, @@ -268,10 +290,9 @@ def rolling_write(self, entries: List[ManifestEntry], writer.flush() avro_bytes = buf.getvalue() file_name = f"{name_prefix}-{len(result)}" - self._flush(file_name, avro_bytes) - written_files.append(file_name) - result.append(self._build_meta( - file_name, entries[chunk_start:i + 1], len(avro_bytes))) + meta = self._flush(file_name, avro_bytes, entries[chunk_start:i + 1]) + written_files.append(meta) + result.append(meta) chunk_start = i + 1 buf = BytesIO() writer = Writer( @@ -282,13 +303,12 @@ def rolling_write(self, entries: List[ManifestEntry], writer.flush() avro_bytes = buf.getvalue() file_name = f"{name_prefix}-{len(result)}" - self._flush(file_name, avro_bytes) - written_files.append(file_name) - result.append(self._build_meta( - file_name, entries[chunk_start:], len(avro_bytes))) - except Exception: - for fname in written_files: - self.file_io.delete_quietly(f"{self.manifest_path}/{fname}") + meta = self._flush(file_name, avro_bytes, entries[chunk_start:]) + written_files.append(meta) + result.append(meta) + except BaseException: + for meta in written_files: + self.delete(meta) raise return result @@ -335,17 +355,37 @@ def _to_avro_record(entry: ManifestEntry) -> dict: def _to_avro_records(self, entries: List[ManifestEntry]) -> List[dict]: return [self._to_avro_record(e) for e in entries] - def _flush(self, file_name: str, avro_bytes: bytes): + def delete(self, manifest: ManifestFileMeta): + self.file_io.delete_quietly(f"{self.manifest_path}/{manifest.file_name}") + for extra_file in manifest.extra_files or []: + self.file_io.delete_quietly(f"{self.manifest_path}/{extra_file}") + + def _flush(self, file_name: str, avro_bytes: bytes, entries) -> ManifestFileMeta: manifest_path = f"{self.manifest_path}/{file_name}" + sidecar_file_name = None try: with self.file_io.new_output_stream(manifest_path) as output_stream: output_stream.write(avro_bytes) - except Exception as e: + settings = Settings.from_options(self.table.options) + if settings.write: + data = build_from_entries(avro_bytes, entries, file_name, settings) + if data is not None: + sidecar_file_name = file_name + SUFFIX + with self.file_io.new_output_stream(f"{self.manifest_path}/{sidecar_file_name}") as output_stream: + output_stream.write(data) + # Publish the reference only after both objects close successfully. + return self._build_meta(file_name, entries, len(avro_bytes), + [sidecar_file_name] if sidecar_file_name is not None else None) + except BaseException as e: self.file_io.delete_quietly(manifest_path) + if sidecar_file_name is not None: + self.file_io.delete_quietly(f"{self.manifest_path}/{sidecar_file_name}") + if not isinstance(e, Exception) or isinstance(e, InterruptedError): + raise raise RuntimeError(f"Failed to write manifest file: {e}") from e def _build_meta(self, file_name: str, entries: List[ManifestEntry], - file_size: int = None) -> ManifestFileMeta: + file_size: int = None, extra_files: Optional[List[str]] = None) -> ManifestFileMeta: added_file_count = 0 deleted_file_count = 0 schema_id = None @@ -370,7 +410,9 @@ def _build_meta(self, file_name: str, entries: List[ManifestEntry], min_row_id = None max_row_id = None for entry in entries: - if entry.file.first_row_id is None: + if (entry.file.first_row_id is None or entry.file.first_row_id < 0 + or entry.file.row_count <= 0 + or entry.file.row_count - 1 > (1 << 63) - 1 - entry.file.first_row_id): min_row_id = None max_row_id = None break @@ -402,4 +444,5 @@ def _build_meta(self, file_name: str, entries: List[ManifestEntry], schema_id=schema_id, min_row_id=min_row_id, max_row_id=max_row_id, + extra_files=extra_files, ) diff --git a/paimon-python/pypaimon/manifest/manifest_file_merger.py b/paimon-python/pypaimon/manifest/manifest_file_merger.py index 821b12aef5e9..2f14f44e7736 100644 --- a/paimon-python/pypaimon/manifest/manifest_file_merger.py +++ b/paimon-python/pypaimon/manifest/manifest_file_merger.py @@ -93,8 +93,4 @@ def _merge_candidates(self, candidates: List[ManifestFileMeta], def _delete_manifests(self, manifests: List[ManifestFileMeta]): for manifest in manifests: - manifest_path = "{}/{}".format( - self.manifest_file_manager.manifest_path, - manifest.file_name, - ) - self.manifest_file_manager.file_io.delete_quietly(manifest_path) + self.manifest_file_manager.delete(manifest) diff --git a/paimon-python/pypaimon/manifest/manifest_sidecar.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py new file mode 100644 index 000000000000..66541d4270ea --- /dev/null +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -0,0 +1,497 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Independent partition, row-id and bucket coverage for each Avro block.""" + +import hashlib +import logging +import struct +from bisect import bisect_left +from concurrent.futures import CancelledError +from dataclasses import dataclass +from io import BytesIO +from typing import Tuple + +from pyarrow import ArrowCancelled + +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.utils.range import Range +from pypaimon.table.row.generic_row import GenericRowSerializer, GenericRowDeserializer + +LOG = logging.getLogger(__name__) +SUFFIX = '.avro.sidecar' +MAGIC = b'PAIMSCAR' +FORMAT_VERSION = 1 +MAX_ROW_ID = (1 << 63) - 1 +MAX_AVRO_HEADER = 1024 * 1024 +READ_BUFFER_BYTES = 1024 * 1024 +HEADER = struct.Struct('>8sI32sqq') +BLOCK = struct.Struct('>qqq') +MAX_BLOCKS = 131072 +BLOCK_BYTES = 39 +PAIR = struct.Struct('>qq') +LONG = struct.Struct('>q') +_PROPAGATED_ERRORS = (InterruptedError, CancelledError, ArrowCancelled, MemoryError, RecursionError) + + +@dataclass +class Settings: + write: bool = False + read: bool = False + max_bytes: int = 2 * CoreOptions.MANIFEST_TARGET_FILE_SIZE.default_value().get_bytes() + + def __post_init__(self): + if not 128 <= self.max_bytes < (1 << 31) - 1: + raise ValueError('manifest.sidecar.max-bytes must be in [128, 2147483646] bytes') + + @classmethod + def from_options(cls, options): + write = options.options.get(CoreOptions.MANIFEST_SIDECAR_WRITE) + read = options.options.get(CoreOptions.MANIFEST_SIDECAR_READ) + # Disabled sidecars must not constrain the manifest target size. + if not write and not read: + return cls() + return cls( + write, read, options.manifest_sidecar_max_size()) + + +@dataclass(frozen=True) +class Block: + offset: int + length: int + first_record: int + record_count: int + + +@dataclass(frozen=True) +class Selection: + header: bytes + blocks: Tuple[Block, ...] + + +class Query: + def __init__(self, ranges): + normalized = Range.sort_and_merge_overlap(list(ranges), True) + self.starts = [r.from_ for r in normalized] + self.ends = [r.to for r in normalized] + + def intersects(self, first, last): + candidate = bisect_left(self.ends, first) + return candidate < len(self.starts) and self.starts[candidate] <= last + + +class Builder: + def __init__(self, settings, header): + self.settings = settings + self.header = header + self.complete = (header is not None and len(header) <= MAX_AVRO_HEADER + and HEADER.size + 44 + len(header) <= settings.max_bytes) + self.dictionary = {} + self.dictionary_bytes = 0 + self.optional_bytes = 0 + self.blocks = [] + self.ranges = [] + self.partition_ids = set() + self.bucket_pairs = set() + self.next_offset = len(header) if header is not None else 0 + self.next_record = 0 + self.current = None + + def begin_block(self, offset, length, records): + if not self.complete: + return + _require(self.current is None and offset == self.next_offset and length > 0 and records > 0) + if (len(self.blocks) == MAX_BLOCKS or HEADER.size + 44 + len(self.header) + + (len(self.blocks) + 1) * BLOCK_BYTES > self.settings.max_bytes): + self.complete = False + self.blocks.clear() + self.dictionary.clear() + return + self.current = Block(offset, length, self.next_record, records) + self.entries_in_block = 0 + self.row_available = self.partition_available = True + self.bucket_available = True + self.coarse = False + self.min = MAX_ROW_ID + self.max = -1 + self.ranges.clear() + self.partition_ids.clear() + self.bucket_pairs.clear() + + def add(self, first, count, partition=None, bucket=None, total_buckets=None): + if not self.complete: + return + _require(self.current is not None) + self.entries_in_block += 1 + self._add_partition(partition) + self._add_bucket(bucket, total_buckets) + if not self.row_available: + return + if (first is None or first < 0 or count <= 0 + or first > MAX_ROW_ID or count - 1 > MAX_ROW_ID - first): + self.row_available = False + self.ranges.clear() + return + end = first + count - 1 + self.min, self.max = min(self.min, first), max(self.max, end) + # Even after coarsening, inspect every entry to extend bounds or mark coverage unknown. + if self.coarse: + return + left = bisect_left(self.ranges, (first, -1)) + if left and self.ranges[left - 1][1] >= first - 1: + left -= 1 + right = left + while right < len(self.ranges) and self.ranges[right][0] <= end + 1: + first = min(first, self.ranges[right][0]) + end = max(end, self.ranges[right][1]) + right += 1 + if (4 + 16 * (len(self.ranges) - (right - left) + 1) + > self.settings.max_bytes - self.optional_bytes): + self.coarse = True + self.ranges.clear() + else: + self.ranges[left:right] = [(first, end)] + + def _add_bucket(self, bucket, total_buckets): + if not self.bucket_available: + return + if (bucket is None or total_buckets is None or bucket < 0 or total_buckets <= bucket + or total_buckets > (1 << 31) - 1): + self.bucket_available = False + self.bucket_pairs.clear() + return + pair = (bucket, total_buckets) + if (pair not in self.bucket_pairs + and 4 + 8 * (len(self.bucket_pairs) + 1) > self.settings.max_bytes - self.optional_bytes): + self.bucket_available = False + self.bucket_pairs.clear() + else: + self.bucket_pairs.add(pair) + + def _add_partition(self, partition): + if not self.partition_available: + return + if partition is None: + self.partition_available = False + self.partition_ids.clear() + return + partition = bytes(partition) + id_ = self.dictionary.get(partition) + if id_ is None: + if len(partition) + 4 > self.settings.max_bytes - self.dictionary_bytes: + self.partition_available = False + self.partition_ids.clear() + return + id_ = len(self.dictionary) + self.dictionary[partition] = id_ + self.dictionary_bytes += 4 + len(partition) + self.partition_ids.add(id_) + + def end_block(self): + if not self.complete: + return + block = self.current + _require(block is not None and self.entries_in_block == block.record_count) + row_payload = partition_payload = b'' + if self.row_available: + if (self.coarse + or 4 + 16 * len(self.ranges) > self.settings.max_bytes - self.optional_bytes): + self.ranges = [(self.min, self.max)] + if 4 + 16 * len(self.ranges) <= self.settings.max_bytes - self.optional_bytes: + row_payload = struct.pack('>I', len(self.ranges)) + b''.join(PAIR.pack(*r) for r in self.ranges) + self.optional_bytes += len(row_payload) + if (self.partition_available + and 4 + 4 * len(self.partition_ids) <= self.settings.max_bytes - self.optional_bytes): + partition_payload = struct.pack('>I', len(self.partition_ids)) + partition_payload += b''.join(struct.pack('>I', id_) for id_ in sorted(self.partition_ids)) + self.optional_bytes += len(partition_payload) + bucket_payload = b'' + if (self.bucket_available + and 4 + 8 * len(self.bucket_pairs) <= self.settings.max_bytes - self.optional_bytes): + bucket_payload = struct.pack('>I', len(self.bucket_pairs)) + bucket_payload += b''.join(struct.pack('>ii', *pair) for pair in sorted(self.bucket_pairs)) + self.optional_bytes += len(bucket_payload) + self.blocks.append([block, partition_payload, row_payload, bucket_payload]) + self.next_offset = block.offset + block.length + self.next_record = block.first_record + block.record_count + self.ranges.clear() + self.partition_ids.clear() + self.bucket_pairs.clear() + self.current = None + + def serialize(self, name, file_size, entry_count): + if not self.complete: + return None + _require(self.current is None and self.next_offset == file_size and self.next_record == entry_count) + size = (HEADER.size + 44 + len(self.header) + self.dictionary_bytes + + len(self.blocks) * BLOCK_BYTES + self.optional_bytes) + for item in self.blocks: + if size <= self.settings.max_bytes: + break + size -= len(item[2]) + self.optional_bytes -= len(item[2]) + item[2] = b'' + for item in self.blocks: + if size <= self.settings.max_bytes: + break + size -= len(item[3]) + self.optional_bytes -= len(item[3]) + item[3] = b'' + if size > self.settings.max_bytes: + size -= self.dictionary_bytes + self.dictionary_bytes = 0 + self.dictionary.clear() + for item in self.blocks: + size -= len(item[1]) + self.optional_bytes -= len(item[1]) + item[1] = b'' + _require(size <= self.settings.max_bytes) + data = bytearray(HEADER.pack( + MAGIC, FORMAT_VERSION, hashlib.sha256(name.encode('utf-8')).digest(), file_size, entry_count)) + data.extend(struct.pack('>I', len(self.header))) + data.extend(self.header) + data.extend(struct.pack('>I', len(self.dictionary))) + for partition in self.dictionary: + data.extend(struct.pack('>I', len(partition))) + data.extend(partition) + data.extend(struct.pack('>I', len(self.blocks))) + for block, partitions, row_ids, buckets in self.blocks: + data.extend(BLOCK.pack(block.offset, block.length, block.record_count)) + for payload in (partitions, row_ids, buckets): + data.extend(struct.pack('>BI', 1 if payload else 0, len(payload))) + data.extend(payload) + return bytes(data) + hashlib.sha256(data).digest() + + +def build_from_entries(avro_bytes, entries, name, settings): + import fastavro + blocks = iter(fastavro.block_reader(BytesIO(avro_bytes))) + first_block = next(blocks, None) + header = avro_bytes[:first_block.offset] if first_block else avro_bytes + builder = Builder(settings, header) + position = 0 + block = first_block + while block is not None and builder.complete: + builder.begin_block(block.offset, block.size, block.num_records) + end = position + block.num_records + _require(end <= len(entries)) + for i in range(position, end): + entry = entries[i] + builder.add(entry.file.first_row_id, entry.file.row_count, GenericRowSerializer.to_bytes(entry.partition), + entry.bucket, entry.total_buckets) + if not builder.complete: + break + builder.end_block() + position = end + block = next(blocks, None) if builder.complete else None + return builder.serialize(name, len(avro_bytes), len(entries)) + + +def _require(condition): + if not condition: + raise ValueError('Invalid, unsupported, mismatched or over-budget manifest sidecar') + + +def select(data, manifest, query, settings, partition_filter=None, partition_fields=None, bucket_filter=None): + if query is not None and not isinstance(query, Query): + query = Query(query) + _require(128 <= len(data) <= settings.max_bytes) + limit = len(data) - 32 + _require(hashlib.sha256(data[:limit]).digest() == data[limit:]) + magic, version, name_hash, size, entries = HEADER.unpack_from(data) + _require(magic == MAGIC and version == FORMAT_VERSION) + _require(name_hash == hashlib.sha256(manifest.file_name.encode('utf-8')).digest()) + _require(size == manifest.file_size and entries == manifest.num_added_files + manifest.num_deleted_files) + header_length, = struct.unpack_from('>I', data, HEADER.size) + offset = HEADER.size + 4 + _require(21 <= header_length <= MAX_AVRO_HEADER and header_length <= limit - offset - 8) + header = bytes(data[offset:offset + header_length]) + _require(header[:4] == b'Obj\x01') + offset += header_length + partitions, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(partitions <= (limit - offset) // 16) + matches = None if partition_filter is None else [] + unique = set() + for _ in range(partitions): + _require(offset + 4 <= limit) + length, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(12 <= length <= limit - offset) + partition = bytes(data[offset:offset + length]) + arity, = struct.unpack_from('>i', partition) + _require(arity >= 0 and 4 + ((arity + 71) // 64) * 8 + arity * 8 <= length) + _require(partition_fields is None or arity == len(partition_fields)) + _require(partition not in unique) + unique.add(partition) + if partition_filter is not None: + _require(partition_fields is not None) + matches.append(partition_filter.test(GenericRowDeserializer.from_bytes(partition, partition_fields))) + offset += length + _require(offset + 4 <= limit) + blocks, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(blocks <= MAX_BLOCKS and blocks <= (limit - offset) // BLOCK_BYTES) + next_offset = header_length + first_record = 0 + selected = [] + for _ in range(blocks): + _require(offset + BLOCK_BYTES <= limit) + file_offset, length, count = BLOCK.unpack_from(data, offset) + offset += BLOCK.size + _require(file_offset == next_offset and 0 < length <= size - file_offset) + _require(0 < count <= entries - first_record) + partition_encoding, payload_length = struct.unpack_from('>BI', data, offset) + offset += 5 + _require(payload_length <= limit - offset) + partition_hit = True + if partition_encoding == 0: + _require(payload_length == 0) + elif partition_encoding == 1: + _require(payload_length >= 4) + ids, = struct.unpack_from('>I', data, offset) + _require(0 < ids <= partitions and 4 + 4 * ids == payload_length) + partition_hit = partition_filter is None + previous = -1 + for j in range(ids): + id_, = struct.unpack_from('>i', data, offset + 4 + 4 * j) + _require(previous < id_ < partitions) + previous = id_ + if not partition_hit: + partition_hit = matches[id_] + offset += payload_length + _require(offset + 5 <= limit) + row_encoding, payload_length = struct.unpack_from('>BI', data, offset) + offset += 5 + _require(payload_length <= limit - offset) + row_hit = True + if row_encoding == 0: + _require(payload_length == 0) + elif row_encoding == 1: + _require(payload_length >= 4) + ranges, = struct.unpack_from('>I', data, offset) + _require(ranges > 0 and 4 + 16 * ranges == payload_length) + min_row_id, first_end = PAIR.unpack_from(data, offset + 4) + _require(min_row_id >= 0 and first_end >= min_row_id) + candidate = False + if partition_hit and query is not None: + max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, offset + payload_length - 8)[0] + _require(max_row_id >= first_end) + candidate = query.intersects(min_row_id, max_row_id) + row_hit = candidate and (ranges == 1 or query.intersects(min_row_id, first_end)) + previous = first_end + for range_position in range(1, ranges): + start, end = PAIR.unpack_from(data, offset + 4 + 16 * range_position) + _require(start >= 0 and end >= start and start > previous) + previous = end + if candidate and not row_hit: + row_hit = query.intersects(start, end) + offset += payload_length + _require(offset + 5 <= limit) + bucket_encoding, payload_length = struct.unpack_from('>BI', data, offset) + offset += 5 + _require(payload_length <= limit - offset) + bucket_hit = True + if bucket_encoding == 0: + _require(payload_length == 0) + elif bucket_encoding == 1: + _require(payload_length >= 4) + pairs, = struct.unpack_from('>I', data, offset) + _require(pairs > 0 and payload_length == 4 + 8 * pairs) + bucket_hit = not partition_hit or not row_hit or bucket_filter is None + previous = (-1, -1) + for j in range(pairs): + bucket, total_buckets = struct.unpack_from('>ii', data, offset + 4 + 8 * j) + _require(0 <= bucket < total_buckets and (bucket, total_buckets) > previous) + previous = (bucket, total_buckets) + if not bucket_hit: + bucket_hit = bucket_filter(bucket, total_buckets) + offset += payload_length + if partition_hit and row_hit and bucket_hit: + selected.append(Block(file_offset, length, first_record, count)) + next_offset = file_offset + length + first_record += count + _require(offset == limit and next_offset == size and first_record == entries) + return Selection(header, tuple(selected)) + + +def sidecar_file_name(manifest): + return next((name for name in manifest.extra_files or [] if name.endswith(SUFFIX)), None) + + +def read_sidecar(file_io, manifest_path, manifest, query, settings, partition_filter=None, partition_fields=None, + bucket_filter=None): + name = sidecar_file_name(manifest) + if name is None: + return None + sidecar_path = manifest_path.rsplit('/', 1)[0] + '/' + name + try: + with file_io.new_input_stream(sidecar_path) as stream: + data = bytearray() + while True: + chunk = stream.read(min(READ_BUFFER_BYTES, settings.max_bytes + 1 - len(data))) + if not chunk: + break + data.extend(chunk) + _require(len(data) <= settings.max_bytes) + return select(data, manifest, query, settings, partition_filter, partition_fields, bucket_filter) + except _PROPAGATED_ERRORS: + raise + except Exception as error: + pending = [error] + visited = set() + while pending: + cause = pending.pop() + if id(cause) in visited: + continue + visited.add(id(cause)) + if not isinstance(cause, Exception) or isinstance(cause, _PROPAGATED_ERRORS): + raise cause + if cause.__cause__ is not None: + pending.append(cause.__cause__) + if cause.__context__ is not None: + pending.append(cause.__context__) + LOG.debug('Cannot use manifest sidecar for %s; reading manifest: %s', manifest_path, error) + return None + + +def read_selected_bytes(file_io, manifest_path, selected): + """Read complete selected blocks with seek; adjacent blocks share one contiguous span. + + The concatenated original header and blocks form a valid Avro OCF. Partial entries + must not be stored in a cache keyed by the complete manifest. + """ + data = bytearray(selected.header) + with file_io.new_input_stream(manifest_path) as stream: + block_position = 0 + while block_position < len(selected.blocks): + block = selected.blocks[block_position] + block_position += 1 + end = block.offset + block.length + while (block_position < len(selected.blocks) + and selected.blocks[block_position].offset == end): + end += selected.blocks[block_position].length + block_position += 1 + stream.seek(block.offset) + remaining = end - block.offset + while remaining: + chunk = stream.read(min(remaining, READ_BUFFER_BYTES)) + if not chunk: + raise EOFError('Truncated manifest block') + data.extend(chunk) + remaining -= len(chunk) + return bytes(data) diff --git a/paimon-python/pypaimon/read/scanner/file_scanner.py b/paimon-python/pypaimon/read/scanner/file_scanner.py index c114fd656e1c..0c794d678dee 100755 --- a/paimon-python/pypaimon/read/scanner/file_scanner.py +++ b/paimon-python/pypaimon/read/scanner/file_scanner.py @@ -583,7 +583,7 @@ def read_manifest_entries(self, manifest_files: List[ManifestFileMeta], self.scan_stats.manifest_files_after_partition += len(manifest_files) # Force single-threaded so we can mutate stats without locking. max_workers = 1 - # Disable both early filters in explain mode (scan_stats) so all entries + # Disable early entry filters and sidecar pruning in explain mode so all entries # flow through _filter_manifest_entry for accurate funnel counting. early_row_filter = None if self.scan_stats is not None \ else _build_early_row_range_filter(row_ranges) @@ -597,6 +597,7 @@ def read_manifest_entries(self, manifest_files: List[ManifestFileMeta], early_entry_filter=self._build_early_bucket_filter(), early_record_filter=early_row_filter, partition_filter=partition_filter, + row_ranges=row_ranges if self.scan_stats is None else None, ) def _build_early_bucket_filter(self): diff --git a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py new file mode 100644 index 000000000000..688e2351b30e --- /dev/null +++ b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py @@ -0,0 +1,358 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import base64 +import hashlib +import struct +import random +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import pytest + +from pypaimon.manifest import manifest_sidecar +from pypaimon.manifest.manifest_sidecar import Builder, Query, Settings, select +from pypaimon.schema.data_types import AtomicType, DataField +from pypaimon.table.row.generic_row import GenericRow, GenericRowSerializer +from pypaimon.tests.manifest.manifest_sidecar_test import avro_header, golden, golden_meta +from pypaimon.utils.range import Range + +FIELDS = [DataField(0, 'p', AtomicType('INT')), DataField(1, 'q', AtomicType('STRING'))] + + +def partition(p, q): + return GenericRowSerializer.to_bytes(GenericRow([p, q], FIELDS)) + + +def part(p): + return SimpleNamespace(test=lambda row: row.values[0] == p) + + +def fixture(key): + path = Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources/manifest-sidecar.txt' + value = next(line.split('=', 1)[1] for line in path.read_text().splitlines() if line.startswith(key + '=')) + return base64.b64decode(value) + + +def meta(name, size, count): + return SimpleNamespace(file_name=name, file_size=size, num_added_files=count, num_deleted_files=0) + + +def test_partition_dictionary_golden_tuples_nulls_and_derived_ordinals(): + a, b, header = partition(7, 'left'), partition(9, None), avro_header() + assert a == fixture('partitionA') + assert b == fixture('partitionB') + builder = Builder(Settings(), header) + values = [(0, 100, [(0, 10, a), (5, 5, a), (20, 5, b)]), + (100, 200, [((1 << 32) - 2, 5, b), (8254058425445, 1, a)]), + (300, 100, [(20, 5, a), ((1 << 63) - 1, 1, b)])] + for offset, length, entries in values: + builder.begin_block(len(header) + offset, length, len(entries)) + for first, count, p in entries: + builder.add(first, count, p) + builder.end_block() + data = builder.serialize('manifest-golden', len(header) + 400, 7) + assert data == fixture('indexWithPartitions') + predicate = part(7) + with patch.object(predicate, 'test', wraps=predicate.test) as evaluated: + selected = select(data, golden_meta(), [Range(20, 20)], Settings(), predicate, FIELDS) + assert evaluated.call_count == 2 + assert [b.first_record for b in selected.blocks] == [0, 5] + nulls = SimpleNamespace(test=lambda row: row.values[1] is None) + assert len(select(data, golden_meta(), None, Settings(), nulls, FIELDS).blocks) == 3 + assert not select(data, golden_meta(), None, Settings(), part(99), FIELDS).blocks + assert len(select(golden(), golden_meta(), None, Settings(), part(99), FIELDS).blocks) == 3 + + +def test_unavailable_dimensions_are_independent_and_dictionary_misses_keep_unknown_blocks(): + settings, header = Settings(max_bytes=512), avro_header() + builder = Builder(settings, header) + for i, (first, p) in enumerate([(None, partition(7, 'left')), (200, partition(9, 'x' * 600)), + (300, partition(7, 'left'))]): + builder.begin_block(len(header) + 100 * i, 100, 1) + builder.add(first, 10, p) + builder.end_block() + data = builder.serialize('m', len(header) + 300, 3) + metadata = meta('m', len(header) + 300, 3) + assert [b.first_record for b in select(data, metadata, None, settings, part(9), FIELDS).blocks] == [1] + selected = select(data, metadata, [Range(999, 999)], settings, part(7), FIELDS) + assert [b.first_record for b in selected.blocks] == [0] + selected = select(data, metadata, [Range(200, 200)], settings, part(9), FIELDS) + assert [b.first_record for b in selected.blocks] == [1] + + +@pytest.mark.parametrize('unknown', [False, True]) +def test_coarse_row_ranges_keep_extending_bounds_and_detect_late_unknowns(unknown): + settings, header = Settings(max_bytes=512), avro_header() + builder = Builder(settings, header) + builder.begin_block(len(header), 100, 66) + for i in range(64): + builder.add(100 + i * 1000, 10, partition(7, 'left')) + for first, count in [(10, 10), (None if unknown else (1 << 63) - 1, 1)]: + builder.add(first, count, partition(7, 'left')) + builder.end_block() + data = builder.serialize('m', len(header) + 100, 66) + metadata = meta('m', len(header) + 100, 66) + for point in [10, 100, 200, (1 << 63) - 1]: + assert len(select(data, metadata, [Range(point, point)], settings).blocks) == 1 + assert len(select(data, metadata, [Range(0, 0)], settings).blocks) == int(unknown) + assert not select(data, metadata, None, settings, part(9), FIELDS).blocks + + +def positions(data): + offset = 64 + struct.unpack_from('>I', data, 60)[0] + count, = struct.unpack_from('>I', data, offset) + offset += 4 + for _ in range(count): + length, = struct.unpack_from('>I', data, offset) + offset += 4 + length + count, = struct.unpack_from('>I', data, offset) + offset += 4 + result = [] + for _ in range(count): + block = offset + p = offset + 24 + r = p + 5 + struct.unpack_from('>I', data, p + 1)[0] + offset = r + 5 + struct.unpack_from('>I', data, r + 1)[0] + bucket = offset + length, = struct.unpack_from('>I', data, offset + 1) + offset += 5 + length + result.append((block, p, r, bucket)) + return result + + +def checksum(data): + data[-32:] = hashlib.sha256(data[:-32]).digest() + return data + + +def test_partition_miss_skips_row_and_bucket_matching(): + rows = Query([Range(0, 0)]) + buckets = Mock(return_value=True) + with patch.object(rows, 'intersects', wraps=rows.intersects) as intersects, \ + patch.object(manifest_sidecar, 'LONG', wraps=manifest_sidecar.LONG) as bounds: + selected = select(fixture('indexWithBuckets'), golden_meta(), rows, Settings(), part(99), FIELDS, buckets) + assert not selected.blocks + intersects.assert_not_called() + bounds.unpack_from.assert_not_called() + buckets.assert_not_called() + + +@pytest.mark.parametrize('partition_filter', [None, part(7)]) +def test_row_miss_skips_bucket_matching(partition_filter): + buckets = Mock(return_value=True) + selected = select(fixture('indexWithBuckets'), golden_meta(), [Range(15, 15)], + Settings(), partition_filter, FIELDS, buckets) + assert not selected.blocks + buckets.assert_not_called() + + +def test_absent_partition_filter_keeps_row_and_bucket_matching(): + buckets = Mock(side_effect=lambda bucket, total: bucket == 1) + with patch('pypaimon.manifest.manifest_sidecar.GenericRowDeserializer.from_bytes') as decode_partition: + selected = select(fixture('indexWithBuckets'), golden_meta(), [Range(20, 20)], + Settings(), None, FIELDS, buckets) + assert [b.first_record for b in selected.blocks] == [0] + decode_partition.assert_not_called() + assert [call.args for call in buckets.call_args_list] == [(1, 4), (0, 1), (3, 4)] + + +def test_absent_row_or_bucket_filters_keep_remaining_dimensions(): + data = fixture('indexWithBuckets') + buckets = Mock(side_effect=lambda bucket, total: bucket == 1) + with patch.object(manifest_sidecar, 'LONG', wraps=manifest_sidecar.LONG) as bounds: + selected = select(data, golden_meta(), None, Settings(), part(7), FIELDS, buckets) + assert [b.first_record for b in selected.blocks] == [0] + bounds.unpack_from.assert_not_called() + assert [call.args for call in buckets.call_args_list] == [(1, 4), (2, 4), (2, 8), (0, 1), (3, 4)] + selected = select(data, golden_meta(), [Range(20, 20)], Settings(), part(7), FIELDS) + assert [b.first_record for b in selected.blocks] == [0, 5] + selected = select(data, golden_meta(), None, Settings()) + assert [b.first_record for b in selected.blocks] == [0, 3, 5] + + +def test_unsigned_unknown_encodings_skip_only_one_payload_and_validate_lengths(): + good = fixture('indexWithBuckets') + block, p, r, b = positions(good)[0] + data = bytearray(good) + data[p] = 200 + selected = select(checksum(data), golden_meta(), [Range(0, 0)], Settings(), part(99), FIELDS) + assert [b.first_record for b in selected.blocks] == [0] + data = bytearray(good) + data[r] = 201 + selected = select(checksum(data), golden_meta(), [Range(16, 16)], Settings(), part(7), FIELDS) + assert [b.first_record for b in selected.blocks] == [0] + data = bytearray(good) + data[b] = 202 + # Skip unknown payloads without decoding even an invalid pair count. + struct.pack_into('>I', data, b + 5, 0) + checksum(data) + no_bucket = lambda bucket, total: False + selected = select(data, golden_meta(), [Range(20, 20)], Settings(), part(7), FIELDS, no_bucket) + assert [block.first_record for block in selected.blocks] == [0] + assert not select(data, golden_meta(), [Range(999, 999)], Settings(), part(7), FIELDS, no_bucket).blocks + assert not select(data, golden_meta(), [Range(20, 20)], Settings(), part(99), FIELDS, no_bucket).blocks + for position in (p, r, b): + data = bytearray(good) + data[position] = 0 + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), None, Settings()) + data = bytearray(good) + data[position] = 255 + struct.pack_into('>i', data, position + 1, -1) + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), None, Settings()) + data = bytearray(good) + struct.pack_into('>q', data, block + 16, 2) + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), None, Settings()) + data = bytearray(good) + struct.pack_into('>i', data, p + 9, 999) # out-of-dictionary ID + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), None, Settings()) + data = bytearray(good) + struct.pack_into('>q', data, r + 9 + 16, 9) # overlap first interval, even in a rejected block + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), [Range(999, 999)], Settings(), part(99), FIELDS) + + +def test_optional_payload_exhaustion_never_truncates_the_block_directory(): + settings, header = Settings(max_bytes=280), avro_header() + builder = Builder(settings, header) + for i in range(3): + builder.begin_block(len(header) + 100 * i, 100, 1) + builder.add(100 * i, 10, partition(7, 'left')) + builder.end_block() + data = builder.serialize('m', len(header) + 300, 3) + assert len(data) <= 280 + selected = select(data, meta('m', len(header) + 300, 3), [Range(999, 999)], settings, part(99), FIELDS) + assert [b.first_record for b in selected.blocks] == [0, 1, 2] + assert builder.serialize('m', len(header) + 300, 3) == data + builder.begin_block(len(header) + 300, 100, 1) + builder.add(300, 1, partition(7, 'left')) + builder.end_block() + assert builder.serialize('m', len(header) + 400, 4) is None + + +def test_randomized_budget_degradation_has_no_false_negatives(): + rng = random.Random(9743) + header = avro_header() + for _ in range(60): + settings = Settings(max_bytes=rng.choice([384, 512, 1024, 8192])) + builder = Builder(settings, header) + blocks = [] + for i in range(5): + values = [(rng.choice([None, rng.randrange(100)]), rng.randrange(1, 10), rng.randrange(5)) + for _ in range(6)] + blocks.append(values) + builder.begin_block(len(header) + 100 * i, 100, len(values)) + for first, count, p in values: + builder.add(first, count, partition(p, None)) + builder.end_block() + data = builder.serialize('m', len(header) + 500, 30) + assert data is not None and len(data) <= settings.max_bytes + metadata = meta('m', len(header) + 500, 30) + for point in range(0, 110, 11): + for p in range(5): + selected = select(data, metadata, [Range(point, point)], settings, part(p), FIELDS) + ordinals = {b.first_record for b in selected.blocks} + for i, values in enumerate(blocks): + if (any(row_p == p for _, _, row_p in values) + and any(first is None or first <= point < first + count for first, count, _ in values)): + assert i * 6 in ordinals + + +def test_bucket_payload_golden_rescale_and_unavailable_payloads(): + a, b, header = partition(7, 'left'), partition(9, None), avro_header() + builder = Builder(Settings(), header) + for offset, length, values in [ + (0, 100, [(0, 10, a, 1, 4), (5, 5, a, 1, 4), (20, 5, b, 1, 8)]), + (100, 200, [((1 << 32) - 2, 5, b, 2, 4), (8254058425445, 1, a, 2, 8)]), + (300, 100, [(20, 5, a, 0, 1), ((1 << 63) - 1, 1, b, 3, 4)])]: + builder.begin_block(len(header) + offset, length, len(values)) + for value in values: + builder.add(*value) + builder.end_block() + data = builder.serialize('manifest-golden', len(header) + 400, 7) + assert data == fixture('indexWithBuckets') + selected = select(data, golden_meta(), None, Settings(), bucket_filter=lambda bucket, total: bucket == 1) + assert [b.first_record for b in selected.blocks] == [0] + selected = select(data, golden_meta(), None, Settings(), + bucket_filter=lambda bucket, total: (bucket, total) == (2, 8)) + assert [b.first_record for b in selected.blocks] == [3] + assert not select(data, golden_meta(), [Range(0, 0)], Settings(), part(7), FIELDS, + bucket_filter=lambda bucket, total: bucket == 2).blocks + for data in [golden(), fixture('indexWithPartitions')]: + assert len(select(data, golden_meta(), None, Settings(), bucket_filter=lambda bucket, total: False).blocks) == 3 + + +@pytest.mark.parametrize('pair', [(None, None), (-1, 4), (4, 4), (0, 0), (2, 8)]) +def test_bucket_budget_and_unknown_pairs_degrade_only_bucket_payload(pair): + header, settings = avro_header(), Settings(max_bytes=512) + builder = Builder(settings, header) + extra_pairs = 65 if pair == (2, 8) else 0 + builder.begin_block(len(header), 100, 2 + extra_pairs) + builder.add(100, 10, partition(7, 'left'), 1, 4) + builder.add(200, 10, partition(7, 'left'), *pair) + for i in range(extra_pairs): + builder.add(200, 10, partition(7, 'left'), i, 100) + builder.end_block() + builder.begin_block(len(header) + 100, 100, 1) + builder.add(300, 10, partition(7, 'left'), 1, 4) + builder.end_block() + data = builder.serialize('m', len(header) + 200, 3 + extra_pairs) + assert struct.unpack_from('>BI', data, positions(data)[0][3]) == (0, 0) + metadata = meta('m', len(header) + 200, 3 + extra_pairs) + selected = select(data, metadata, None, settings, bucket_filter=lambda bucket, total: False) + assert [b.first_record for b in selected.blocks] == [0] + assert not select(data, metadata, [Range(999, 999)], settings, bucket_filter=lambda bucket, total: False).blocks + + +def test_payloads_can_exceed_former_limits_within_byte_budget(): + header, settings = avro_header(), Settings() + builder = Builder(settings, header) + blocks, entries_per_block, block_bytes = 33, 4097, 1024 * 1024 + entries = blocks * entries_per_block + for block in range(blocks): + builder.begin_block(len(header) + block * block_bytes, block_bytes, entries_per_block) + for i in range(entries_per_block): + entry = block * entries_per_block + i + builder.add(entry * 2, 1, partition(entry, None), i, entries_per_block + 1) + builder.end_block() + file_size = len(header) + blocks * block_bytes + data = builder.serialize('m', file_size, entries) + assert len(data) <= settings.max_bytes + metadata = meta('m', file_size, entries) + last = (entries - 1) * 2 + selected = select(data, metadata, [Range(last, last)], settings) + assert [b.first_record for b in selected.blocks] == [(blocks - 1) * entries_per_block] + assert not select(data, metadata, [Range(last - 1, last - 1)], settings).blocks + assert not select(data, metadata, None, settings, part(entries), FIELDS).blocks + assert not select(data, metadata, None, settings, + bucket_filter=lambda bucket, total: bucket == entries_per_block).blocks + + +def test_malformed_bucket_payload_invalidates_the_container(): + good = fixture('indexWithBuckets') + payload = positions(good)[0][3] + 1 + for offset, value in [(payload, -2), (payload, (1 << 31) - 1), (payload, 0), (payload + 4, 0), + (payload + 8, -1), (payload + 12, 1), (payload + 16, 0)]: + bad = bytearray(good) + struct.pack_into('>i', bad, offset, value) + with pytest.raises(ValueError): + select(checksum(bad), golden_meta(), [Range(999, 999)], Settings(), part(99), FIELDS) diff --git a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py new file mode 100644 index 000000000000..5e2177c842b7 --- /dev/null +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -0,0 +1,767 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import base64 +import hashlib +import os +import struct +import unittest +from concurrent.futures import CancelledError +from copy import deepcopy +from io import BytesIO + +import fastavro +from pyarrow import ArrowCancelled +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options +from pypaimon.globalindex.global_index_result import GlobalIndexResult +from pypaimon.manifest import manifest_sidecar +from pypaimon.manifest.manifest_sidecar import ( + Block, Builder, Selection, Settings, SUFFIX, MAX_ROW_ID, Query, select, read_sidecar, + read_selected_bytes, sidecar_file_name, +) +from pypaimon.manifest.schema.manifest_entry import ManifestEntry +from pypaimon.manifest.manifest_list_manager import ManifestListManager +from pypaimon.manifest.manifest_file_manager import ManifestFileManager +from pypaimon.manifest.schema.manifest_file_meta import MANIFEST_FILE_META_SCHEMA +from pypaimon.read.scanner.file_scanner import FileScanner +from pypaimon.read.scan_stats import ScanStats +from pypaimon.tests.manifest import manifest_entry_identifier_test as existing +from pypaimon.schema.schema import Schema +from pypaimon.table.row.generic_row import GenericRow +from pypaimon.utils.range import Range + + +def fixture(): + path = (Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources' / + 'manifest-sidecar.txt') + return dict(line.split('=', 1) for line in path.read_text().splitlines() + if line.startswith(('index=', 'avroHeader='))) + + +def golden(): + return base64.b64decode(fixture()['index']) + + +def avro_header(): + return base64.b64decode(fixture()['avroHeader']) + + +def golden_meta(): + return SimpleNamespace(file_name='manifest-golden', file_size=len(avro_header()) + 400, + num_added_files=7, num_deleted_files=0) + + +def intersects(data, meta, ranges, settings): + return bool(select(data, meta, ranges, settings).blocks) + + +class CountingInput(BytesIO): + def __init__(self, data, max_read=None): + super().__init__(data) + self.max_read = max_read + self.reads = [] + self.requests = [] + self.seeks = [] + + def read(self, size=-1): + if size < 0: + raise AssertionError('Unbounded read') + self.requests.append(size) + position = self.tell() + data = super().read(size if self.max_read is None else min(size, self.max_read)) + if data: + self.reads.append((position, len(data))) + return data + + def seek(self, offset, whence=0): + self.seeks.append(offset) + return super().seek(offset, whence) + + +class FailingIndexInput(BytesIO): + def __init__(self, data, failure, phase, close_failure=None): + super().__init__(data) + self.failure = failure + self.phase = phase + self.close_failure = close_failure + + def read(self, size=-1): + if self.phase == 'read': + raise self.failure + return super().read(size) + + def close(self): + was_closed = self.closed + super().close() + if not was_closed and self.close_failure is not None: + raise self.close_failure + if self.phase == 'close' and not was_closed: + raise self.failure + + +class ManifestSidecarReadTest(unittest.TestCase): + def test_index_reads_use_bounded_bulk_requests(self): + header = avro_header() + for block_count in (5000, 25000): + with self.subTest(block_count=block_count): + builder = Builder(Settings(), header) + for block_number in range(block_count): + builder.begin_block(len(header) + block_number * 100, 100, 1) + builder.add(block_number, 1) + builder.end_block() + size = len(header) + block_count * 100 + data = builder.serialize('manifest-large', size, block_count) + meta = SimpleNamespace(file_name='manifest-large', file_size=size, + num_added_files=block_count, num_deleted_files=0, + extra_files=['manifest-large' + SUFFIX]) + stream = CountingInput(data) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + actual = read_sidecar(file_io, '/manifest/manifest-large', meta, + [Range(0, 0)], Settings()) + self.assertEqual(actual, select(data, meta, [Range(0, 0)], Settings())) + self.assertEqual(len(stream.reads), (len(data) + (1 << 20) - 1) // (1 << 20)) + self.assertLessEqual(max(stream.requests), 1 << 20) + self.assertTrue(stream.closed) + + def test_index_short_reads_and_exact_budget(self): + data, meta = golden(), golden_meta() + meta.extra_files = [meta.file_name + SUFFIX] + for max_read in (None, 7): + with self.subTest(max_read=max_read): + stream = CountingInput(data, max_read) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + settings = Settings(max_bytes=len(data)) + actual = read_sidecar(file_io, '/manifest/manifest-golden', meta, + [Range(20, 20)], settings) + self.assertEqual(actual, select(data, meta, [Range(20, 20)], settings)) + self.assertTrue(stream.closed) + + def test_index_over_budget_stops_after_one_extra_byte(self): + data, meta = golden(), golden_meta() + meta.extra_files = [meta.file_name + SUFFIX] + stream = CountingInput(data) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + self.assertIsNone(read_sidecar(file_io, '/manifest/manifest-golden', meta, + [Range(20, 20)], Settings(max_bytes=128))) + self.assertEqual(stream.reads, [(0, 129)]) + self.assertTrue(stream.closed) + + def test_adjacent_blocks_share_reads_without_reading_gaps(self): + header = avro_header() + body = bytes(range(200)) * 2 + for points, spans in [([0, 8254058425445], [(0, 300)]), + ([20], [(0, 100), (300, 100)]), ([16], [])]: + with self.subTest(points=points): + selected = select(golden(), golden_meta(), + [Range(point, point) for point in points], Settings()) + stream = CountingInput(header + body) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + actual = read_selected_bytes(file_io, '/manifest/manifest-golden', selected) + expected = header + b''.join(body[start:start + size] for start, size in spans) + self.assertEqual(actual, expected) + self.assertEqual(stream.reads, [(len(header) + start, size) for start, size in spans]) + self.assertEqual(stream.seeks, [len(header) + start for start, _ in spans]) + self.assertTrue(stream.closed) + + def test_large_block_spans_use_bounded_reads(self): + header = avro_header() + block_size = 512 * 1024 + body = bytes(2 * block_size + 257) + selected = Selection(header, (Block(len(header), block_size, 0, 1), + Block(len(header) + block_size, block_size, 1, 1), + Block(len(header) + 2 * block_size, 257, 2, 1))) + stream = CountingInput(header + body) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + self.assertEqual(read_selected_bytes(file_io, '/manifest/manifest-large', selected), header + body) + self.assertEqual(stream.reads, [(len(header), 1 << 20), (len(header) + (1 << 20), 257)]) + self.assertEqual(stream.seeks, [len(header)]) + self.assertTrue(stream.closed) + + def test_block_short_reads_and_truncation(self): + header = avro_header() + body = bytes(range(200)) * 2 + selected = select(golden(), golden_meta(), [Range(0, MAX_ROW_ID)], Settings()) + stream = CountingInput(header + body, 7) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + self.assertEqual(read_selected_bytes(file_io, '/manifest/manifest-golden', selected), header + body) + self.assertTrue(stream.closed) + stream = CountingInput(header + body[:-1], 7) + with self.assertRaises(EOFError): + read_selected_bytes(file_io, '/manifest/manifest-golden', selected) + self.assertTrue(stream.closed) + + +class ManifestSidecarFormatTest(unittest.TestCase): + def test_settings_use_memory_sizes_and_follow_the_manifest_target(self): + options = CoreOptions(Options({})) + self.assertEqual(Settings().max_bytes, 16 * 1024 * 1024) + self.assertEqual(Settings.from_options(options).max_bytes, 16 * 1024 * 1024) + options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, True) + options.options.set(CoreOptions.MANIFEST_TARGET_FILE_SIZE, '12 mb') + self.assertEqual(Settings.from_options(options).max_bytes, 24 * 1024 * 1024) + options.options.set(CoreOptions.MANIFEST_TARGET_FILE_SIZE, '64 mb') + self.assertEqual(Settings.from_options(options).max_bytes, 128 * 1024 * 1024) + options.options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, '512 kb') + options.options.set(CoreOptions.MANIFEST_TARGET_FILE_SIZE, '1 gb') + self.assertEqual(Settings.from_options(options).max_bytes, 512 * 1024) + options.options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, '2147483646 bytes') + self.assertEqual(Settings.from_options(options).max_bytes, (1 << 31) - 2) + for value in ('127 bytes', '2147483647 bytes', '2 gb'): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'manifest.sidecar.max-bytes'): + options.options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, value) + Settings.from_options(options) + + def test_disabled_sidecars_do_not_constrain_manifest_target_size(self): + for target in ('1 bytes', '1 gb'): + with self.subTest(target=target): + settings = Settings.from_options(CoreOptions(Options({'manifest.target-file-size': target}))) + self.assertFalse(settings.read) + self.assertFalse(settings.write) + + def test_cross_language_and_block_ordinals(self): + data, meta, header = golden(), golden_meta(), avro_header() + for point in (0, 9, 20, 24, (1 << 32) - 2, 1 << 32, (1 << 32) + 2, + 8254058425445, MAX_ROW_ID): + self.assertTrue(intersects(data, meta, [Range(point, point)], Settings())) + for point in (10, 19, 25, (1 << 32) - 3, (1 << 32) + 3, 8254058425444, MAX_ROW_ID - 1): + self.assertFalse(intersects(data, meta, [Range(point, point)], Settings())) + selected = select(data, meta, [Range(20, 20)], Settings()) + self.assertEqual([b.first_record for b in selected.blocks], [0, 5]) + self.assertEqual([b.offset for b in selected.blocks], [len(header), len(header) + 300]) + self.assertEqual([b.length for b in selected.blocks], [100, 100]) + + gap = select(data, meta, [Range(16, 16)], Settings()) + + self.assertFalse(gap.blocks) + ranges = [Range(10, 19), Range(25, 40)] + self.assertFalse(intersects(data, meta, ranges, Settings())) + self.assertEqual(ranges, [Range(10, 19), Range(25, 40)]) + b = Builder(Settings(), header) + for offset, length, values in [ + (len(header), 100, [(0, 10), (5, 5), (20, 5)]), + (len(header) + 100, 200, [((1 << 32) - 2, 5), (8254058425445, 1)]), + (len(header) + 300, 100, [(20, 5), (MAX_ROW_ID, 1)])]: + b.begin_block(offset, length, len(values)) + for first, count in values: + b.add(first, count) + b.end_block() + self.assertEqual(b.serialize(meta.file_name, meta.file_size, 7), + golden()) + + def test_minmax_skips_exact_checks_and_one_interval_is_already_exact(self): + header = avro_header() + builder = Builder(Settings(), header) + for offset, values in [(0, [(0, 10), (20, 10)]), + (100, [(100, 10), (200, 10)]), + (200, [(1 << 32, 10)])]: + builder.begin_block(len(header) + offset, 100, len(values)) + for first, count in values: + builder.add(first, count) + builder.end_block() + data = builder.serialize('m', len(header) + 300, 5) + meta = SimpleNamespace(file_name='m', file_size=len(header) + 300, + num_added_files=5, num_deleted_files=0) + for point, expected in [(50, 0), ((1 << 32) + 9, 1)]: + query = Query([Range(point, point)]) + with patch.object(query, 'intersects', wraps=query.intersects) as check: + selected = select(data, meta, query, Settings()) + + self.assertEqual(len(selected.blocks), expected) + self.assertEqual(check.call_count, 3) + check.assert_any_call(0, 29) + check.assert_any_call(100, 209) + check.assert_any_call(1 << 32, (1 << 32) + 9) + + def test_single_interval_decodes_pair_once(self): + header = avro_header() + for first, count in [(0, 1), (42, 10), (MAX_ROW_ID, 1)]: + builder = Builder(Settings(), header) + builder.begin_block(len(header), 100, 1) + builder.add(first, count) + builder.end_block() + data = builder.serialize('m', len(header) + 100, 1) + meta = SimpleNamespace(file_name='m', file_size=len(header) + 100, + num_added_files=1, num_deleted_files=0) + last = first + count - 1 + missing = first - 1 if first > 0 else last + 1 + for ranges, expected in [(None, 1), ([], 0), ([Range(first, first)], 1), + ([Range(last, last)], 1), ([Range(missing, missing)], 0)]: + with self.subTest(first=first, count=count, ranges=ranges), \ + patch.object(manifest_sidecar, 'PAIR', wraps=manifest_sidecar.PAIR) as pairs, \ + patch.object(manifest_sidecar, 'LONG', wraps=manifest_sidecar.LONG) as bounds: + selected = select(data, meta, ranges, Settings()) + self.assertEqual(len(selected.blocks), expected) + pairs.unpack_from.assert_called_once() + bounds.unpack_from.assert_not_called() + + def test_rejected_and_early_hit_blocks_still_validate_every_interval(self): + first_block_intervals = 60 + 4 + len(avro_header()) + 4 + 4 + 24 + 5 + 5 + 4 + for relative_offset, value in [(0, -1), (8, -1), (8, 30), (16, 9), (24, 19)]: + data = bytearray(golden()) + struct.pack_into('>q', data, first_block_intervals + relative_offset, value) + data[-32:] = hashlib.sha256(data[:-32]).digest() + for ranges in ([Range(30, 30)], [Range(0, 0)], None, []): + with self.subTest(offset=relative_offset, value=value, ranges=ranges), \ + self.assertRaises(ValueError): + select(data, golden_meta(), ranges, Settings()) + + def test_coverage_and_budgets(self): + header = avro_header() + for first, count in [(None, 1), (-1, 1), (10, 0), (10, -1), (MAX_ROW_ID, 2)]: + b = Builder(Settings(), header) + b.begin_block(len(header), 100, 1) + b.add(first, count) + b.end_block() + data = b.serialize('m', len(header) + 100, 1) + meta = SimpleNamespace(file_name='m', file_size=len(header) + 100, + num_added_files=1, num_deleted_files=0) + self.assertEqual(len(select(data, meta, [Range(100, 100)], Settings()).blocks), 1) + b = Builder(Settings(), header) + b.begin_block(len(header), 100, 2) + b.add(0, MAX_ROW_ID) + b.add(MAX_ROW_ID, 1) + b.end_block() + self.assertLess(len(b.serialize('m', len(header) + 100, 2)), 512) + b = Builder(Settings(max_bytes=512), header) + b.begin_block(len(header), 100, 64) + for i in range(64): + b.add(1 + i * 100, 1) + b.end_block() + data = b.serialize('m', len(header) + 100, 64) + meta = SimpleNamespace(file_name='m', file_size=len(header) + 100, + num_added_files=64, num_deleted_files=0) + self.assertEqual(len(select(data, meta, [Range(10, 10)], Settings()).blocks), 1) + b = Builder(Settings(max_bytes=128), header) + self.assertIsNone(b.serialize('m', 1, 1)) + + def test_invalid_envelopes(self): + meta, data = golden_meta(), golden() + for index in (0, 9, 11, 15, 16, 55, 63, 67, 75, len(data) - 1): + bad = bytearray(data) + bad[index] ^= 2 + with self.assertRaises(ValueError): + select(bad, meta, [Range(10, 10)], Settings()) + for version in (0, 2, 99): + bad = bytearray(data[:-32]) + struct.pack_into('>I', bad, 8, version) + bad.extend(hashlib.sha256(bad).digest()) + with self.assertRaises(ValueError): + select(bad, meta, [Range(10, 10)], Settings()) + with self.assertRaises(ValueError): + select(data[:-1], meta, [Range(10, 10)], Settings()) + meta.file_name = 'mismatch' + with self.assertRaises(ValueError): + select(data, meta, [Range(10, 10)], Settings()) + + +class ManifestSidecarScanTest(existing.ManifestEntryIdentifierTest): + def setUp(self): + super().setUp() + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, True) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, True) + + def entry(self, name, first, count=10, kind=0): + return ManifestEntry(kind, self._create_file_meta('unused').min_key, 0, 1, + replace(self._create_file_meta(name), first_row_id=first, row_count=count)) + + def write_meta(self, name, entries): + manager = self.manifest_file_manager + return manager.write(name, entries) + + def test_bucket_point_lookup_with_rescale_and_delete_entries(self): + from pypaimon.common.predicate import Predicate + from pypaimon.read.scanner.bucket_select_converter import create_bucket_selector + selector = create_bucket_selector(Predicate('equal', 0, 'id', [7]), self.table.fields[:1]) + self.assertIsNotNone(selector) + entries = [replace(self.entry('%s-%s-%s.parquet' % (total, bucket, i), None), + bucket=bucket, total_buckets=total) + for total in (4, 8) for bucket in range(total) for i in range(300)] + metadata = self.write_meta('buckets', entries) + results = [] + for enabled in (False, True): + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) + scanner = FileScanner(self.table, lambda: ([metadata], None)) + scanner._bucket_selector = selector + with patch('pypaimon.manifest.manifest_file_manager.read_selected_bytes', + wraps=read_selected_bytes) as read_blocks: + actual = scanner.read_manifest_entries([metadata]) + results.append([e.file.file_name for e in actual]) + if enabled: + read_blocks.assert_called_once() + selected = read_blocks.call_args[0][2] + self.assertLess(sum(b.record_count for b in selected.blocks), 1000) + else: + read_blocks.assert_not_called() + self.assertEqual(results[0], results[1]) + self.assertEqual(len(results[1]), 600) + chosen = next(bucket for bucket in range(4) if selector(bucket, 4)) + added = [replace(self.entry('point.' + suffix, None), bucket=chosen, total_buckets=4) + for suffix in ('parquet', 'blob')] + metas = [self.write_meta('point-add', added), + self.write_meta('point-delete', [replace(e, kind=1) for e in added])] + self.assertEqual(scanner.read_manifest_entries(metas), []) + + def test_partition_only_and_conjunctive_planning_keep_entry_and_delete_filters(self): + import pyarrow as pa + schema = Schema.from_pyarrow_schema( + pa.schema([('p', pa.int32()), ('q', pa.string()), ('value', pa.string())]), + partition_keys=['p', 'q'], + options={'manifest.sidecar.write': 'true', 'manifest.sidecar.read': 'true'}) + self.catalog.create_table('default.partition_block_index', schema, False) + self.table = self.catalog.get_table('default.partition_block_index') + self.manifest_file_manager = ManifestFileManager(self.table) + fields = self.table.partition_keys_fields + + def entry(name, first, p, kind=0): + return replace(self.entry(name, first, kind=kind), partition=GenericRow([p, None], fields)) + + # Build many blocks without any row tracking; partition-only planning must use the index. + entries = [entry('part-%d.parquet' % i, None, i // 1000) for i in range(4000)] + manifest = self.write_meta('partitioned', entries) + from pypaimon.common.predicate import Predicate + predicate = Predicate('equal', 0, 'p', [1]) + results = [] + for enabled in (False, True): + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) + scanner = FileScanner(self.table, lambda: ([manifest], None), partition_predicate=predicate) + with patch('pypaimon.manifest.manifest_file_manager.read_selected_bytes', + wraps=read_selected_bytes) as read_blocks: + actual = scanner.read_manifest_entries([manifest]) + results.append([e.file.file_name for e in actual]) + if enabled: + read_blocks.assert_called_once() + selected = read_blocks.call_args[0][2] + self.assertLess(sum(b.record_count for b in selected.blocks), 1500) + else: + read_blocks.assert_not_called() + self.assertEqual(results[0], results[1]) + self.assertEqual(len(results[1]), 1000) + + # A block can match the two dimensions through different entries; keep entry filtering. + mixed = self.write_meta('mixed', [entry('a.parquet', 100, 1), entry('b.parquet', 5, 2)]) + scanner = FileScanner(self.table, lambda: ([mixed], None), partition_predicate=predicate) + self.assertEqual(scanner.read_manifest_entries([mixed], row_ranges=[Range(5, 5)]), []) + # Both column groups and DELETE blocks must survive the same partition + row-id filter. + add = [entry('data.parquet', 100, 1), entry('data.blob', 100, 1)] + metas = [self.write_meta('adds', add), self.write_meta('deletes', [replace(e, kind=1) for e in add])] + self.assertEqual(scanner.read_manifest_entries(metas, row_ranges=[Range(105, 105)]), []) + + def test_explain_keeps_complete_entry_counts_with_sidecar_enabled(self): + import pyarrow as pa + from pypaimon.common.predicate import Predicate + + schema = Schema.from_pyarrow_schema( + pa.schema([('p', pa.int32()), ('q', pa.string()), ('value', pa.string())]), + partition_keys=['p', 'q'], + options={'manifest.sidecar.write': 'true', 'manifest.sidecar.read': 'true'}) + self.catalog.create_table('default.explain_sidecar', schema, False) + self.table = self.catalog.get_table('default.explain_sidecar') + self.manifest_file_manager = ManifestFileManager(self.table) + fields = self.table.partition_keys_fields + entries = [replace(self.entry('part-%d.parquet' % i, i * 1000), + partition=GenericRow([i // 1000, None], fields)) for i in range(4000)] + manifest = self.write_meta('explain-manifest', entries) + predicate = Predicate('equal', 0, 'p', [1]) + ranges = [Range(1000000, 1000000)] + for partition_filter, row_ranges in [(predicate, None), (None, ranges), (predicate, ranges)]: + results = [] + for enabled in (False, True): + with self.subTest(partition=partition_filter, row_ranges=row_ranges, sidecar=enabled): + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) + scanner = FileScanner(self.table, lambda: ([manifest], None), + partition_predicate=partition_filter) + scanner.scan_stats = ScanStats() + with patch('pypaimon.manifest.manifest_file_manager.read_sidecar', + wraps=read_sidecar) as read_metadata: + actual = scanner.read_manifest_entries([manifest], row_ranges=row_ranges) + read_metadata.assert_not_called() + stats = scanner.scan_stats + self.assertEqual(stats.entries_potential_total, 4000) + self.assertEqual(stats.entries_total, 4000) + self.assertEqual(stats.entries_after_partition, 1000 if partition_filter else 4000) + self.assertEqual(stats.partition_keys_before, {(p, None) for p in range(4)}) + results.append([entry.file.file_name for entry in actual]) + self.assertEqual(results[0], results[1]) + + def test_explicit_reference_and_null_does_not_probe(self): + manager = self.manifest_file_manager + written = self.write_meta('explicit', [self.entry('data.parquet', 100)]) + self.assertEqual(sidecar_file_name(written), written.file_name + SUFFIX) + index_path = Path(manager.manifest_path, sidecar_file_name(written)) + explicit_path = index_path.with_name('independent-index' + SUFFIX) + index_path.rename(explicit_path) + other_path = index_path.with_name('other-partition-index') + other_path.write_bytes(b'not a manifest sidecar') + indexed = replace(written, extra_files=[other_path.name, explicit_path.name]) + with patch.object(self.table.file_io, 'new_input_stream', + wraps=self.table.file_io.new_input_stream) as opened: + self.assertEqual(manager.read_entries_parallel([indexed], row_ranges=[Range(0, 0)]), []) + self.assertEqual([call[0][0] for call in opened.call_args_list], [str(explicit_path)]) + + for extra_files in (None, [], [other_path.name]): + with self.subTest(extra_files=extra_files): + unindexed = replace(indexed, extra_files=extra_files) + with patch.object(self.table.file_io, 'new_input_stream', + wraps=self.table.file_io.new_input_stream) as opened: + actual = manager.read_entries_parallel([unindexed], row_ranges=[Range(0, 0)]) + self.assertEqual(len(actual), 1) + self.assertEqual([call[0][0] for call in opened.call_args_list], + [str(Path(manager.manifest_path, written.file_name))]) + manager.delete(indexed) + self.assertFalse(other_path.exists()) + self.assertFalse(explicit_path.exists()) + self.assertFalse(Path(manager.manifest_path, written.file_name).exists()) + + def test_manifest_list_index_reference_compatibility(self): + indexed = self.write_meta('indexed', [self.entry('data.parquet', 100)]) + indexed = replace(indexed, extra_files=['other-index'] + indexed.extra_files) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, False) + unindexed = self.write_meta('legacy-entry', [self.entry('old.parquet', None)]) + self.assertIsNone(sidecar_file_name(unindexed)) + lists = ManifestListManager(self.table) + lists.write('references', [indexed, unindexed]) + actual = lists.read('references') + self.assertEqual([meta.extra_files for meta in actual], [indexed.extra_files, None]) + self.assertEqual([sidecar_file_name(meta) for meta in actual], [sidecar_file_name(indexed), None]) + self.assertEqual([meta.file_name for meta in actual], [indexed.file_name, unindexed.file_name]) + + data = Path(lists.manifest_path, 'references').read_bytes() + legacy_schema = deepcopy(MANIFEST_FILE_META_SCHEMA) + legacy_schema['fields'] = [field for field in legacy_schema['fields'] + if field['name'] != '_EXTRA_FILES'] + legacy_records = list(fastavro.reader(BytesIO(data), reader_schema=legacy_schema)) + self.assertTrue(all('_EXTRA_FILES' not in record for record in legacy_records)) + self.assertEqual([record['_FILE_NAME'] for record in legacy_records], + [indexed.file_name, unindexed.file_name]) + with self.table.file_io.new_output_stream(str(Path(lists.manifest_path, 'old-list'))) as stream: + fastavro.writer(stream, legacy_schema, legacy_records) + self.assertTrue(all(sidecar_file_name(meta) is None for meta in lists.read('old-list'))) + + def test_skips_blocks_inside_a_matching_manifest(self): + entries = [self.entry('file-%d.parquet' % i, i * 1000) for i in range(4000)] + meta = self.write_meta('many-blocks', entries) + outputs = [] + reader = fastavro.reader + for enabled in (False, True): + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) + scanner = FileScanner(self.table, lambda: ([meta], None)) + scanner.with_global_index_result(GlobalIndexResult.from_ranges([Range(2000005, 2000005)])) + decoded = [] + + def observed_reader(stream): + for record in reader(stream): + decoded.append(record) + yield record + + with patch('pypaimon.manifest.manifest_file_manager.fastavro.reader', side_effect=observed_reader), \ + patch('pypaimon.manifest.manifest_file_manager.read_selected_bytes', + wraps=read_selected_bytes) as selected_read: + actual, _ = scanner._create_data_evolution_split_generator() + outputs.append([e.file.file_name for e in actual]) + if enabled: + self.assertEqual(selected_read.call_count, 1) + selected = selected_read.call_args[0][2] + self.assertEqual(len(selected.blocks), 1) + self.assertLess(sum(block.length for block in selected.blocks), meta.file_size // 10) + self.assertLess(len(decoded), 200) + self.assertEqual(len(decoded), selected.blocks[0].record_count) + else: + self.assertEqual(selected_read.call_count, 0) + self.assertEqual(len(decoded), 4000) + self.assertEqual(outputs, [['file-2000.parquet']] * 2) + + def test_actual_global_index_scanner_72_to_2(self): + metas = [] + for i in range(72): + entries = [self.entry('a%d.parquet' % i, 0), self.entry('b%d.parquet' % i, 100)] + if i < 2: + entries.append(self.entry('hit.' + ('parquet' if i == 0 else 'blob'), 45)) + metas.append(self.write_meta('manifest-%d' % i, entries)) + results = [] + for enabled in (False, True): + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) + scanner = FileScanner(self.table, lambda: (metas, None)) + scanner.with_global_index_result(GlobalIndexResult.from_ranges([Range(50, 50)])) + manager = scanner.manifest_file_manager + with patch.object(manager, 'read', wraps=manager.read) as read_body, \ + patch('pypaimon.manifest.manifest_file_manager.read_sidecar', wraps=read_sidecar) as read_metadata: + entries, _ = scanner._create_data_evolution_split_generator() + results.append(sorted(e.file.file_name for e in entries)) + self.assertEqual(len(read_body.call_args_list), 2 if enabled else 72) + self.assertEqual(len(read_metadata.call_args_list), 72 if enabled else 0) + self.assertEqual(results, [['hit.blob', 'hit.parquet']] * 2) + + def test_delete_union_no_resurrection_and_no_query_no_index_io(self): + add = self.entry('data.parquet', 45) + blob = self.entry('data.blob', 45) + metas = [self.write_meta('add', [add, blob]), + self.write_meta('delete', [replace(add, kind=1), replace(blob, kind=1)]), + self.write_meta('gap', [self.entry('lo', 0), self.entry('hi', 100)])] + manager = self.manifest_file_manager + for enabled in (False, True): + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) + with patch.object(manager, 'read', wraps=manager.read) as read_body: + entries = manager.read_entries_parallel(metas[:2], row_ranges=[Range(50, 50)]) + self.assertEqual(entries, []) + self.assertEqual(len(read_body.call_args_list), 2) + with patch.object(self.table.file_io, 'new_input_stream', + wraps=self.table.file_io.new_input_stream) as opened: + manager.read_entries_parallel(metas) + self.assertTrue(all(not call[0][0].endswith(SUFFIX) for call in opened.call_args_list)) + # Missing and corrupt objects retain their manifests and read the full body. + path = manager.manifest_path + '/gap' + SUFFIX + for bad in (None, b'partial'): + if bad is None: + os.unlink(path) + else: + Path(path).write_bytes(bad) + with patch.object(manager, 'read', wraps=manager.read) as read_body: + entries = manager.read_entries_parallel(metas[2:], row_ranges=[Range(50, 50)]) + self.assertEqual(len(entries), 2) + self.assertEqual(read_body.call_count, 1) + self.assertIsNone(read_body.call_args[1]['selected_blocks']) + with patch.object(self.table.file_io, 'new_input_stream', side_effect=InterruptedError('stop')): + with self.assertRaises(InterruptedError): + read_sidecar(self.table.file_io, path, metas[0], [Range(0, 0)], Settings()) + + def test_sidecar_cancellation_during_open(self): + self._check_sidecar_cancellation('open') + + def test_sidecar_cancellation_during_read(self): + self._check_sidecar_cancellation('read') + + def test_sidecar_cancellation_during_close(self): + self._check_sidecar_cancellation('close') + + def _check_sidecar_cancellation(self, phase): + for failure_type in (ArrowCancelled, CancelledError, InterruptedError): + with self.subTest(phase=phase, failure_type=failure_type): + self._check_sidecar_io_failure(phase, failure_type('cancelled'), cancelled=True) + + def test_sidecar_io_failures_fall_back_to_manifest(self): + for phase in ('open', 'read', 'close'): + for failure_type in (FileNotFoundError, TimeoutError, OSError): + with self.subTest(phase=phase, failure_type=failure_type): + self._check_sidecar_io_failure(phase, failure_type('unavailable'), cancelled=False) + + def test_sidecar_cancellation_survives_close_failure(self): + for failure_type in (ArrowCancelled, CancelledError, InterruptedError): + with self.subTest(failure_type=failure_type): + self._check_sidecar_io_failure('read', failure_type('cancelled'), cancelled=True, + close_failure=OSError('close failed')) + + def test_sidecar_wrapped_cancellation_propagates(self): + for failure_type in (ArrowCancelled, CancelledError, InterruptedError): + with self.subTest(failure_type=failure_type): + cancellation = failure_type('cancelled') + wrapped = OSError('wrapped failure') + wrapped.__cause__ = cancellation + self._check_sidecar_io_failure('open', wrapped, cancelled=True, + expected_failure=cancellation) + + def test_sidecar_exception_cycle_falls_back(self): + first = OSError('first') + second = OSError('second') + first.__cause__ = second + second.__cause__ = first + self._check_sidecar_io_failure('open', first, cancelled=False) + + def _check_sidecar_io_failure(self, phase, failure, cancelled, close_failure=None, + expected_failure=None): + manager = self.manifest_file_manager + meta = self.write_meta('failure-' + phase + '-' + type(failure).__name__, + [self.entry('data.parquet', 100)]) + index_path = str(Path(manager.manifest_path, sidecar_file_name(meta))) + body_path = str(Path(manager.manifest_path, meta.file_name)) + stream = (FailingIndexInput(Path(index_path).read_bytes(), failure, phase, close_failure) + if phase != 'open' else None) + original_open = self.table.file_io.new_input_stream + + def open_stream(path): + if path == index_path: + if phase == 'open': + raise failure + return stream + return original_open(path) + + with patch.object(self.table.file_io, 'new_input_stream', side_effect=open_stream) as opened, \ + patch.object(manager, 'read', wraps=manager.read) as read_body: + if cancelled: + expected = failure if expected_failure is None else expected_failure + with self.assertRaises(type(expected)) as raised: + manager.read_entries_parallel([meta], row_ranges=[Range(100, 100)]) + self.assertIs(raised.exception, expected) + read_body.assert_not_called() + self.assertEqual([call.args[0] for call in opened.call_args_list], [index_path]) + else: + entries = manager.read_entries_parallel([meta], row_ranges=[Range(100, 100)]) + self.assertEqual([entry.file.file_name for entry in entries], ['data.parquet']) + read_body.assert_called_once() + self.assertIsNone(read_body.call_args.kwargs['selected_blocks']) + self.assertEqual([call.args[0] for call in opened.call_args_list], [index_path, body_path]) + if stream is not None: + self.assertTrue(stream.closed) + + def test_rolling_merge_limits_and_abort_cleanup(self): + entries = [self.entry('file-%d' % i, i * 1000) for i in range(300)] + manager = self.manifest_file_manager + metas = manager.rolling_write(entries, 300, 'rolling') + self.assertGreater(len(metas), 1) + for meta in metas: + actual = manager.read(meta.file_name) + data = Path(manager.manifest_path, meta.file_name + SUFFIX).read_bytes() + for e in actual: + self.assertTrue(intersects(data, meta, [Range(e.file.first_row_id, e.file.first_row_id)], Settings())) + gap = actual[0].file.first_row_id + 10 + self.assertFalse(intersects(data, meta, [Range(gap, gap)], Settings())) + from pypaimon.manifest.manifest_file_merger import ManifestFileMerger + merger = ManifestFileMerger(manager, 1000000, 2) + merged = merger.merge(metas) + # Merger returns both the final manifest list and newly written outputs. + outputs = merged[0] if isinstance(merged, tuple) else merged + for meta in outputs: + self.assertTrue(Path(manager.manifest_path, meta.file_name + SUFFIX).exists()) + for meta in metas: + self.assertIsNotNone(sidecar_file_name(meta)) + manager.delete(meta) + self.assertFalse(Path(manager.manifest_path, meta.file_name + SUFFIX).exists()) + original = self.table.file_io.new_output_stream + + def fail(path): + if path.endswith(SUFFIX): + raise OSError('sidecar write failed') + return original(path) + with patch.object(self.table.file_io, 'new_output_stream', side_effect=fail): + with self.assertRaises(RuntimeError): + manager.write('failed', entries[:1]) + self.assertFalse(Path(manager.manifest_path, 'failed').exists()) + self.assertFalse(Path(manager.manifest_path, 'failed' + SUFFIX).exists()) + unknown = manager.write('unknown', [self.entry('legacy', None)]) + self.assertIsNotNone(sidecar_file_name(unknown)) + data = Path(manager.manifest_path, sidecar_file_name(unknown)).read_bytes() + self.assertEqual(len(select(data, unknown, [Range(100, 100)], Settings()).blocks), 1) + huge = manager.write('huge', [self.entry('one', 0, MAX_ROW_ID), + self.entry('two', MAX_ROW_ID, 1)]) + self.assertIsNotNone(sidecar_file_name(huge)) + data = Path(manager.manifest_path, sidecar_file_name(huge)).read_bytes() + self.assertEqual(len(select(data, huge, [Range(50, 50)], Settings()).blocks), 1) diff --git a/paimon-python/pypaimon/tests/reader_append_only_test.py b/paimon-python/pypaimon/tests/reader_append_only_test.py index 480cd4ebbe4b..1c783bfe0611 100644 --- a/paimon-python/pypaimon/tests/reader_append_only_test.py +++ b/paimon-python/pypaimon/tests/reader_append_only_test.py @@ -1090,7 +1090,8 @@ def test_is_in_with_partitions(self): def counting_read(self_mgr, manifest_file_name, manifest_entry_filter=None, drop_stats=True, early_entry_filter=None, - early_record_filter=None, partition_filter=None): + early_record_filter=None, partition_filter=None, + selected_blocks=None): # avro_total = every entry in the manifest (no manifest-file pruning # here: single file, is_in spans its partition stats). path = f"{self_mgr.manifest_path}/{manifest_file_name}" @@ -1100,7 +1101,8 @@ def counting_read(self_mgr, manifest_file_name, return original_read( self_mgr, manifest_file_name, manifest_entry_filter, drop_stats, - early_entry_filter, early_record_filter, partition_filter) + early_entry_filter, early_record_filter, partition_filter, + selected_blocks=selected_blocks) def counting_dfm_init(self_dfm, *args, **kwargs): entry_counts['constructed'] += 1 diff --git a/paimon-python/pypaimon/write/file_store_commit.py b/paimon-python/pypaimon/write/file_store_commit.py index 66e88f2f5a68..2fd81fda2cc7 100644 --- a/paimon-python/pypaimon/write/file_store_commit.py +++ b/paimon-python/pypaimon/write/file_store_commit.py @@ -1140,8 +1140,7 @@ def _clean_up_reuse_tmp_manifests( if ml_name: try: for meta in self.manifest_list_manager.read(ml_name): - self.table.file_io.delete_quietly( - f"{self.manifest_file_manager.manifest_path}/{meta.file_name}") + self.manifest_file_manager.delete(meta) except Exception: pass self.table.file_io.delete_quietly(f"{manifest_path}/{ml_name}") @@ -1160,8 +1159,7 @@ def _clean_up_no_reuse_tmp_manifests( if base_manifest_list: self.table.file_io.delete_quietly(f"{manifest_path}/{base_manifest_list}") for meta in merge_new_files: - self.table.file_io.delete_quietly( - f"{self.manifest_file_manager.manifest_path}/{meta.file_name}") + self.manifest_file_manager.delete(meta) def abort(self, commit_messages: List[CommitMessage]): """Abort commit and delete files. Uses external_path if available to ensure proper scheme handling."""