From 7a029ee48a8d9ba396af68bbea0c542ac20782b0 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 08:47:24 +0800 Subject: [PATCH 1/4] [fs] Fail fast on zero-byte skip in HadoopCompliantFileIO wrappers skipFully looped while (bytes > 0) subtracting whatever in.skip returned. A blocking stream at end of file returns 0 from skip, so a small forward seek (up to 1 MiB, used by the vectored-read helpers) past a truncated or shorter-than-expected file spun the loop forever instead of failing the read with an end-of-file error. Throw EOFException when a skip call makes no progress. The same loop was replicated in HadoopFileIO and in every HadoopCompliantFileIO wrapper (azure, cosn, gs, jindo, obs, oss, s3); all copies are updated. Assisted-by: GLM-5.3 --- .../apache/paimon/fs/hadoop/HadoopFileIO.java | 11 +++- .../fs/hadoop/HadoopFileIOSkipFullyTest.java | 64 +++++++++++++++++++ .../paimon/azure/HadoopCompliantFileIO.java | 11 +++- .../paimon/cosn/HadoopCompliantFileIO.java | 11 +++- .../paimon/gs/HadoopCompliantFileIO.java | 11 +++- .../paimon/jindo/HadoopCompliantFileIO.java | 11 +++- .../paimon/obs/HadoopCompliantFileIO.java | 11 +++- .../paimon/oss/HadoopCompliantFileIO.java | 11 +++- .../paimon/s3/HadoopCompliantFileIO.java | 11 +++- 9 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java index 3114baad2e83..9a814dfbc211 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options; +import java.io.EOFException; import java.io.IOException; import java.io.OutputStreamWriter; import java.lang.reflect.InvocationTargetException; @@ -309,7 +310,15 @@ public void forceSeek(long seekPos) throws IOException { */ public void skipFully(long bytes) throws IOException { while (bytes > 0) { - bytes -= in.skip(bytes); + long skipped = in.skip(bytes); + if (skipped <= 0) { + // a blocking stream returning 0 from skip means EOF; looping here + // would spin forever instead of failing the read + throw new EOFException( + String.format( + "Unexpected end of stream while skipping %s bytes.", bytes)); + } + bytes -= skipped; } } } diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java new file mode 100644 index 000000000000..fa46303a0c96 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fs.hadoop; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.io.EOFException; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Tests that {@code HadoopSeekableInputStream#skipFully} fails fast at end of stream instead of + * spinning on a zero-byte skip. + */ +class HadoopFileIOSkipFullyTest { + + @Test + @Timeout(10) + void skipFullyFailsFastAtEndOfStream() throws Exception { + FSDataInputStream in = mock(FSDataInputStream.class); + // a blocking stream at EOF keeps returning 0 from skip; the mock rethrows after a + // few zero-skips so the unfixed loop surfaces as the wrong exception instead of + // spinning forever + when(in.skip(anyLong())) + .thenReturn(0L, 0L, 0L) + .thenThrow(new IOException("mock exhausted")); + + Class clazz = + Class.forName("org.apache.paimon.fs.hadoop.HadoopFileIO$HadoopSeekableInputStream"); + Constructor constructor = clazz.getDeclaredConstructor(FSDataInputStream.class); + constructor.setAccessible(true); + Object stream = constructor.newInstance(in); + Method skipFully = clazz.getDeclaredMethod("skipFully", long.class); + skipFully.setAccessible(true); + + assertThatThrownBy(() -> skipFully.invoke(stream, 4096L)) + .hasRootCauseInstanceOf(EOFException.class) + .hasRootCauseMessage("Unexpected end of stream while skipping 4096 bytes."); + } +} diff --git a/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java index f928bde84b9b..91d6775ffd87 100644 --- a/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java @@ -30,6 +30,7 @@ import javax.annotation.Nullable; +import java.io.EOFException; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -215,7 +216,15 @@ public void forceSeek(long seekPos) throws IOException { */ public void skipFully(long bytes) throws IOException { while (bytes > 0) { - bytes -= in.skip(bytes); + long skipped = in.skip(bytes); + if (skipped <= 0) { + // a blocking stream returning 0 from skip means EOF; looping here + // would spin forever instead of failing the read + throw new EOFException( + String.format( + "Unexpected end of stream while skipping %s bytes.", bytes)); + } + bytes -= skipped; } } } diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java index 36e9a1e829e4..9b4490dbefbb 100644 --- a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java @@ -30,6 +30,7 @@ import javax.annotation.Nullable; +import java.io.EOFException; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -215,7 +216,15 @@ public void forceSeek(long seekPos) throws IOException { */ public void skipFully(long bytes) throws IOException { while (bytes > 0) { - bytes -= in.skip(bytes); + long skipped = in.skip(bytes); + if (skipped <= 0) { + // a blocking stream returning 0 from skip means EOF; looping here + // would spin forever instead of failing the read + throw new EOFException( + String.format( + "Unexpected end of stream while skipping %s bytes.", bytes)); + } + bytes -= skipped; } } } diff --git a/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java index 227cddcedda7..295020513dd1 100644 --- a/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import java.io.EOFException; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -215,7 +216,15 @@ public void forceSeek(long seekPos) throws IOException { */ public void skipFully(long bytes) throws IOException { while (bytes > 0) { - bytes -= in.skip(bytes); + long skipped = in.skip(bytes); + if (skipped <= 0) { + // a blocking stream returning 0 from skip means EOF; looping here + // would spin forever instead of failing the read + throw new EOFException( + String.format( + "Unexpected end of stream while skipping %s bytes.", bytes)); + } + bytes -= skipped; } } } diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index 0ba8ef98d1b7..77c7af857ee6 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -36,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.EOFException; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; @@ -349,7 +350,15 @@ public void forceSeek(long seekPos) throws IOException { */ public void skipFully(long bytes) throws IOException { while (bytes > 0) { - bytes -= in.skip(bytes); + long skipped = in.skip(bytes); + if (skipped <= 0) { + // a blocking stream returning 0 from skip means EOF; looping here + // would spin forever instead of failing the read + throw new EOFException( + String.format( + "Unexpected end of stream while skipping %s bytes.", bytes)); + } + bytes -= skipped; } } } diff --git a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java index 199b2ee2dd60..613167e548e3 100644 --- a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java @@ -31,6 +31,7 @@ import javax.annotation.Nullable; +import java.io.EOFException; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -236,7 +237,15 @@ public void forceSeek(long seekPos) throws IOException { */ public void skipFully(long bytes) throws IOException { while (bytes > 0) { - bytes -= in.skip(bytes); + long skipped = in.skip(bytes); + if (skipped <= 0) { + // a blocking stream returning 0 from skip means EOF; looping here + // would spin forever instead of failing the read + throw new EOFException( + String.format( + "Unexpected end of stream while skipping %s bytes.", bytes)); + } + bytes -= skipped; } } } diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java index ab48da87cdfa..ea21d8487b07 100644 --- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import java.io.EOFException; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Map; @@ -244,7 +245,15 @@ public void forceSeek(long seekPos) throws IOException { */ public void skipFully(long bytes) throws IOException { while (bytes > 0) { - bytes -= in.skip(bytes); + long skipped = in.skip(bytes); + if (skipped <= 0) { + // a blocking stream returning 0 from skip means EOF; looping here + // would spin forever instead of failing the read + throw new EOFException( + String.format( + "Unexpected end of stream while skipping %s bytes.", bytes)); + } + bytes -= skipped; } } } diff --git a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java index a662e8a07592..01ffece5ba4f 100644 --- a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import java.io.EOFException; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -236,7 +237,15 @@ public void forceSeek(long seekPos) throws IOException { */ public void skipFully(long bytes) throws IOException { while (bytes > 0) { - bytes -= in.skip(bytes); + long skipped = in.skip(bytes); + if (skipped <= 0) { + // a blocking stream returning 0 from skip means EOF; looping here + // would spin forever instead of failing the read + throw new EOFException( + String.format( + "Unexpected end of stream while skipping %s bytes.", bytes)); + } + bytes -= skipped; } } } From b51161e301cba01c6fe0a17a0545a612699fa897 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 15:33:04 +0800 Subject: [PATCH 2/4] fix: delegate skipFully to hadoop's IOUtils The hand-rolled loop declared EOF on the first zero-byte skip, but InputStream.skip may return 0 without being at the end, and the issue's own reference, org.apache.hadoop.io.IOUtils.skipFully, resolves that by probing with read(). It ships in hadoop-common, which every one of these modules already depends on for FSDataInputStream, so delegating both fixes the detection and removes eight byte-identical copies of the loop. Co-Authored-By: Claude Code --- .../apache/paimon/fs/hadoop/HadoopFileIO.java | 17 ++---- .../fs/hadoop/HadoopFileIOSkipFullyTest.java | 52 +++++++++++++------ .../paimon/azure/HadoopCompliantFileIO.java | 17 ++---- .../paimon/cosn/HadoopCompliantFileIO.java | 17 ++---- .../paimon/gs/HadoopCompliantFileIO.java | 17 ++---- .../paimon/jindo/HadoopCompliantFileIO.java | 17 ++---- .../paimon/obs/HadoopCompliantFileIO.java | 17 ++---- .../paimon/oss/HadoopCompliantFileIO.java | 17 ++---- .../paimon/s3/HadoopCompliantFileIO.java | 17 ++---- 9 files changed, 76 insertions(+), 112 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java index 9a814dfbc211..80b7b1ffb2bc 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java @@ -38,8 +38,8 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.io.IOUtils; -import java.io.EOFException; import java.io.IOException; import java.io.OutputStreamWriter; import java.lang.reflect.InvocationTargetException; @@ -309,17 +309,10 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - while (bytes > 0) { - long skipped = in.skip(bytes); - if (skipped <= 0) { - // a blocking stream returning 0 from skip means EOF; looping here - // would spin forever instead of failing the read - throw new EOFException( - String.format( - "Unexpected end of stream while skipping %s bytes.", bytes)); - } - bytes -= skipped; - } + // hadoop's helper probes with read() before calling it EOF, because skip may return + // 0 without being at the end. The loop this replaces treated that 0 as progress and + // spun forever. + IOUtils.skipFully(in, bytes); } } diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java index fa46303a0c96..0ad84fdda5ff 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java @@ -20,35 +20,58 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; import java.io.EOFException; -import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.Method; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** - * Tests that {@code HadoopSeekableInputStream#skipFully} fails fast at end of stream instead of - * spinning on a zero-byte skip. + * {@code HadoopSeekableInputStream#skipFully} turns a short forward seek into skips. A stream that + * returns 0 from {@code skip} used to spin forever; a 0 has to be resolved by reading, because + * {@link java.io.InputStream#skip} may return it without being at the end. */ class HadoopFileIOSkipFullyTest { @Test - @Timeout(10) - void skipFullyFailsFastAtEndOfStream() throws Exception { + void skipFullyThrowsWhenTheStreamReallyEnds() throws Exception { FSDataInputStream in = mock(FSDataInputStream.class); - // a blocking stream at EOF keeps returning 0 from skip; the mock rethrows after a - // few zero-skips so the unfixed loop surfaces as the wrong exception instead of - // spinning forever - when(in.skip(anyLong())) - .thenReturn(0L, 0L, 0L) - .thenThrow(new IOException("mock exhausted")); + when(in.skip(anyLong())).thenReturn(0L); + // the read probe is what distinguishes EOF from a transient zero + when(in.read()).thenReturn(-1); + assertThatThrownBy(() -> skipFully(in, 4096L)).hasRootCauseInstanceOf(EOFException.class); + verify(in).read(); + } + + @Test + void skipFullyContinuesAfterATransientZero() throws Exception { + FSDataInputStream in = mock(FSDataInputStream.class); + // 0 first, then progress: the old loop threw here, and before that it spun + when(in.skip(anyLong())).thenReturn(0L, 4095L); + when(in.read()).thenReturn(7); + + assertThatCode(() -> skipFully(in, 4096L)).doesNotThrowAnyException(); + // the probe consumed one byte, so only the remaining 4095 are skipped + verify(in).read(); + } + + @Test + void skipFullyIsANoOpForNothingToSkip() throws Exception { + FSDataInputStream in = mock(FSDataInputStream.class); + + assertThatCode(() -> skipFully(in, 0L)).doesNotThrowAnyException(); + verify(in, never()).skip(anyLong()); + } + + private static void skipFully(FSDataInputStream in, long bytes) throws Exception { Class clazz = Class.forName("org.apache.paimon.fs.hadoop.HadoopFileIO$HadoopSeekableInputStream"); Constructor constructor = clazz.getDeclaredConstructor(FSDataInputStream.class); @@ -56,9 +79,6 @@ void skipFullyFailsFastAtEndOfStream() throws Exception { Object stream = constructor.newInstance(in); Method skipFully = clazz.getDeclaredMethod("skipFully", long.class); skipFully.setAccessible(true); - - assertThatThrownBy(() -> skipFully.invoke(stream, 4096L)) - .hasRootCauseInstanceOf(EOFException.class) - .hasRootCauseMessage("Unexpected end of stream while skipping 4096 bytes."); + skipFully.invoke(stream, bytes); } } diff --git a/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java index 91d6775ffd87..fec1c643853b 100644 --- a/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java @@ -24,13 +24,13 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import javax.annotation.Nullable; -import java.io.EOFException; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -215,17 +215,10 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - while (bytes > 0) { - long skipped = in.skip(bytes); - if (skipped <= 0) { - // a blocking stream returning 0 from skip means EOF; looping here - // would spin forever instead of failing the read - throw new EOFException( - String.format( - "Unexpected end of stream while skipping %s bytes.", bytes)); - } - bytes -= skipped; - } + // hadoop's helper probes with read() before calling it EOF, because skip may return + // 0 without being at the end. The loop this replaces treated that 0 as progress and + // spun forever. + IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java index 9b4490dbefbb..87e6354258a5 100644 --- a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java @@ -24,13 +24,13 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import javax.annotation.Nullable; -import java.io.EOFException; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -215,17 +215,10 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - while (bytes > 0) { - long skipped = in.skip(bytes); - if (skipped <= 0) { - // a blocking stream returning 0 from skip means EOF; looping here - // would spin forever instead of failing the read - throw new EOFException( - String.format( - "Unexpected end of stream while skipping %s bytes.", bytes)); - } - bytes -= skipped; - } + // hadoop's helper probes with read() before calling it EOF, because skip may return + // 0 without being at the end. The loop this replaces treated that 0 as progress and + // spun forever. + IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java index 295020513dd1..1dccb7148515 100644 --- a/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java @@ -24,11 +24,11 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; -import java.io.EOFException; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -215,17 +215,10 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - while (bytes > 0) { - long skipped = in.skip(bytes); - if (skipped <= 0) { - // a blocking stream returning 0 from skip means EOF; looping here - // would spin forever instead of failing the read - throw new EOFException( - String.format( - "Unexpected end of stream while skipping %s bytes.", bytes)); - } - bytes -= skipped; - } + // hadoop's helper probes with read() before calling it EOF, because skip may return + // 0 without being at the end. The loop this replaces treated that 0 as progress and + // spun forever. + IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index 77c7af857ee6..34c03ab7ffdb 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -31,12 +31,12 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import com.aliyun.jindodata.common.JindoHadoopSystem; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.EOFException; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; @@ -349,17 +349,10 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - while (bytes > 0) { - long skipped = in.skip(bytes); - if (skipped <= 0) { - // a blocking stream returning 0 from skip means EOF; looping here - // would spin forever instead of failing the read - throw new EOFException( - String.format( - "Unexpected end of stream while skipping %s bytes.", bytes)); - } - bytes -= skipped; - } + // hadoop's helper probes with read() before calling it EOF, because skip may return + // 0 without being at the end. The loop this replaces treated that 0 as progress and + // spun forever. + IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java index 613167e548e3..a4b8551f2b2c 100644 --- a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java @@ -25,13 +25,13 @@ import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import javax.annotation.Nullable; -import java.io.EOFException; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -236,17 +236,10 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - while (bytes > 0) { - long skipped = in.skip(bytes); - if (skipped <= 0) { - // a blocking stream returning 0 from skip means EOF; looping here - // would spin forever instead of failing the read - throw new EOFException( - String.format( - "Unexpected end of stream while skipping %s bytes.", bytes)); - } - bytes -= skipped; - } + // hadoop's helper probes with read() before calling it EOF, because skip may return + // 0 without being at the end. The loop this replaces treated that 0 as progress and + // spun forever. + IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java index ea21d8487b07..d501f87c6a50 100644 --- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java @@ -25,11 +25,11 @@ import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; -import java.io.EOFException; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Map; @@ -244,17 +244,10 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - while (bytes > 0) { - long skipped = in.skip(bytes); - if (skipped <= 0) { - // a blocking stream returning 0 from skip means EOF; looping here - // would spin forever instead of failing the read - throw new EOFException( - String.format( - "Unexpected end of stream while skipping %s bytes.", bytes)); - } - bytes -= skipped; - } + // hadoop's helper probes with read() before calling it EOF, because skip may return + // 0 without being at the end. The loop this replaces treated that 0 as progress and + // spun forever. + IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java index 01ffece5ba4f..bac231aa12be 100644 --- a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java @@ -25,11 +25,11 @@ import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; -import java.io.EOFException; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -236,17 +236,10 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - while (bytes > 0) { - long skipped = in.skip(bytes); - if (skipped <= 0) { - // a blocking stream returning 0 from skip means EOF; looping here - // would spin forever instead of failing the read - throw new EOFException( - String.format( - "Unexpected end of stream while skipping %s bytes.", bytes)); - } - bytes -= skipped; - } + // hadoop's helper probes with read() before calling it EOF, because skip may return + // 0 without being at the end. The loop this replaces treated that 0 as progress and + // spun forever. + IOUtils.skipFully(in, bytes); } } From 260407f1d6b5bb71930b322fb18132e933064640 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 15:36:23 +0800 Subject: [PATCH 3/4] style: keep the hadoop import block sorted Co-Authored-By: Claude Code --- .../java/org/apache/paimon/azure/HadoopCompliantFileIO.java | 2 +- .../main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java | 2 +- .../main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java | 2 +- .../java/org/apache/paimon/jindo/HadoopCompliantFileIO.java | 2 +- .../main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java | 2 +- .../main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java | 2 +- .../main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java index fec1c643853b..0a16262891c8 100644 --- a/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java @@ -24,10 +24,10 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.IOUtils; import javax.annotation.Nullable; diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java index 87e6354258a5..b5d9c3aa8878 100644 --- a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java @@ -24,10 +24,10 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.IOUtils; import javax.annotation.Nullable; diff --git a/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java index 1dccb7148515..eec1e0d53862 100644 --- a/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java @@ -24,10 +24,10 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.IOUtils; import java.io.IOException; import java.util.Map; diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index 34c03ab7ffdb..93593e5ba19d 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -31,9 +31,9 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import com.aliyun.jindodata.common.JindoHadoopSystem; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java index a4b8551f2b2c..119a74e33709 100644 --- a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java @@ -25,10 +25,10 @@ import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.IOUtils; import javax.annotation.Nullable; diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java index d501f87c6a50..1bdba586e8bb 100644 --- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java @@ -25,10 +25,10 @@ import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.IOUtils; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java index bac231aa12be..a968a3d82b92 100644 --- a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java @@ -25,10 +25,10 @@ import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.IOUtils; import java.io.IOException; import java.util.Map; From 194fc463ec54854cb1271adca99e90e52cb3a3a1 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 13 Sep 2026 18:38:10 +0800 Subject: [PATCH 4/4] test: pin the whole zero-skip conversation, and correct what the old loop did Bounds the stub so a looping caller fails instead of hanging the fork, pins skip-probe-skip in order with no other interaction, and fixes two comments that claimed more than any assertion checked. --- .../apache/paimon/fs/hadoop/HadoopFileIO.java | 5 ++--- .../fs/hadoop/HadoopFileIOSkipFullyTest.java | 21 +++++++++++++++---- .../paimon/azure/HadoopCompliantFileIO.java | 5 ++--- .../paimon/cosn/HadoopCompliantFileIO.java | 5 ++--- .../paimon/gs/HadoopCompliantFileIO.java | 5 ++--- .../paimon/jindo/HadoopCompliantFileIO.java | 5 ++--- .../paimon/obs/HadoopCompliantFileIO.java | 5 ++--- .../paimon/oss/HadoopCompliantFileIO.java | 5 ++--- .../paimon/s3/HadoopCompliantFileIO.java | 5 ++--- 9 files changed, 33 insertions(+), 28 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java index 80b7b1ffb2bc..4f2a23ef1a2a 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java @@ -309,9 +309,8 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - // hadoop's helper probes with read() before calling it EOF, because skip may return - // 0 without being at the end. The loop this replaces treated that 0 as progress and - // spun forever. + // hadoop's helper probes with read() before calling it EOF, because skip may return 0 + // without being at the end. The loop this replaces subtracted that 0 and asked again. IOUtils.skipFully(in, bytes); } } diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java index 0ad84fdda5ff..3acb94f935fe 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java @@ -20,6 +20,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.junit.jupiter.api.Test; +import org.mockito.InOrder; import java.io.EOFException; import java.lang.reflect.Constructor; @@ -28,9 +29,11 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; /** @@ -43,7 +46,11 @@ class HadoopFileIOSkipFullyTest { @Test void skipFullyThrowsWhenTheStreamReallyEnds() throws Exception { FSDataInputStream in = mock(FSDataInputStream.class); - when(in.skip(anyLong())).thenReturn(0L); + // a caller that reads a 0 as no progress asks again, forever. Fail on the second call so + // this test reports that rather than hanging the fork, which has no timeout to save it. + when(in.skip(anyLong())) + .thenReturn(0L) + .thenThrow(new AssertionError("skip was called again after returning 0")); // the read probe is what distinguishes EOF from a transient zero when(in.read()).thenReturn(-1); @@ -54,13 +61,19 @@ void skipFullyThrowsWhenTheStreamReallyEnds() throws Exception { @Test void skipFullyContinuesAfterATransientZero() throws Exception { FSDataInputStream in = mock(FSDataInputStream.class); - // 0 first, then progress: the old loop threw here, and before that it spun + // 0 first, then progress. The fail-fast revision threw here; the loop before it did not + // probe at all, so the read is what pins this case. when(in.skip(anyLong())).thenReturn(0L, 4095L); when(in.read()).thenReturn(7); assertThatCode(() -> skipFully(in, 4096L)).doesNotThrowAnyException(); - // the probe consumed one byte, so only the remaining 4095 are skipped - verify(in).read(); + // the probe consumed one byte, so the second skip asks for the remaining 4095, and that + // is the whole conversation: an in-order verify alone would allow extra probes + InOrder inOrder = inOrder(in); + inOrder.verify(in).skip(4096L); + inOrder.verify(in).read(); + inOrder.verify(in).skip(4095L); + verifyNoMoreInteractions(in); } @Test diff --git a/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java index 0a16262891c8..2642a9d3fbae 100644 --- a/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-azure-impl/src/main/java/org/apache/paimon/azure/HadoopCompliantFileIO.java @@ -215,9 +215,8 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - // hadoop's helper probes with read() before calling it EOF, because skip may return - // 0 without being at the end. The loop this replaces treated that 0 as progress and - // spun forever. + // hadoop's helper probes with read() before calling it EOF, because skip may return 0 + // without being at the end. The loop this replaces subtracted that 0 and asked again. IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java index b5d9c3aa8878..06450d9f1baa 100644 --- a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java @@ -215,9 +215,8 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - // hadoop's helper probes with read() before calling it EOF, because skip may return - // 0 without being at the end. The loop this replaces treated that 0 as progress and - // spun forever. + // hadoop's helper probes with read() before calling it EOF, because skip may return 0 + // without being at the end. The loop this replaces subtracted that 0 and asked again. IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java index eec1e0d53862..0919408f6194 100644 --- a/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/HadoopCompliantFileIO.java @@ -215,9 +215,8 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - // hadoop's helper probes with read() before calling it EOF, because skip may return - // 0 without being at the end. The loop this replaces treated that 0 as progress and - // spun forever. + // hadoop's helper probes with read() before calling it EOF, because skip may return 0 + // without being at the end. The loop this replaces subtracted that 0 and asked again. IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index 93593e5ba19d..64fc3e07c708 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -349,9 +349,8 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - // hadoop's helper probes with read() before calling it EOF, because skip may return - // 0 without being at the end. The loop this replaces treated that 0 as progress and - // spun forever. + // hadoop's helper probes with read() before calling it EOF, because skip may return 0 + // without being at the end. The loop this replaces subtracted that 0 and asked again. IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java index 119a74e33709..bbdbc339af26 100644 --- a/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-obs-impl/src/main/java/org/apache/paimon/obs/HadoopCompliantFileIO.java @@ -236,9 +236,8 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - // hadoop's helper probes with read() before calling it EOF, because skip may return - // 0 without being at the end. The loop this replaces treated that 0 as progress and - // spun forever. + // hadoop's helper probes with read() before calling it EOF, because skip may return 0 + // without being at the end. The loop this replaces subtracted that 0 and asked again. IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java index 1bdba586e8bb..c3d794fae848 100644 --- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java @@ -244,9 +244,8 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - // hadoop's helper probes with read() before calling it EOF, because skip may return - // 0 without being at the end. The loop this replaces treated that 0 as progress and - // spun forever. + // hadoop's helper probes with read() before calling it EOF, because skip may return 0 + // without being at the end. The loop this replaces subtracted that 0 and asked again. IOUtils.skipFully(in, bytes); } } diff --git a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java index a968a3d82b92..5965d7e6704f 100644 --- a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java @@ -236,9 +236,8 @@ public void forceSeek(long seekPos) throws IOException { * @param bytes the number of bytes to skip. */ public void skipFully(long bytes) throws IOException { - // hadoop's helper probes with read() before calling it EOF, because skip may return - // 0 without being at the end. The loop this replaces treated that 0 as progress and - // spun forever. + // hadoop's helper probes with read() before calling it EOF, because skip may return 0 + // without being at the end. The loop this replaces subtracted that 0 and asked again. IOUtils.skipFully(in, bytes); } }