[python] Support newer PyArrow versions

.github/workflows/ci-python.yml

@@ -136,7 +136,7 @@ jobs:
             pip install torch --index-url https://download.pytorch.org/whl/cpu
             python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 'isal>=1.8,<2' zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 pylance==0.39.0 cramjam pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6' 'datafusion>=54,<55' datasketches
             if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)"; then
-              python -m pip install pyarrow==24.0.0 numpy==2.4.6 pandas==2.3.3 flake8==7.1.2
+              python -m pip install pyarrow==23.0.0 numpy==2.4.6 pandas==2.3.3 flake8==7.1.2
             else
               python -m pip install pyarrow==16.0.0 numpy==1.24.3 pandas==2.0.3 flake8==4.0.1
             fi
@@ -158,11 +158,15 @@ jobs:
               python -m pip install "./paimon-python[sql]"
             fi
             python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
-            if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)"; then
-              # Vortex requires pyarrow >= 17, while Pypaimon requires pyarrow < 20 and != 19.
+            if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
               python -m pip install vortex-data==0.70.0 pyarrow==18.1.0
+            elif python -c "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)"; then
+              python -m pip install vortex-data==0.70.0 pyarrow==23.0.0
             fi
             python -m pip install 'paimon-mosaic>=0.1.0'
+            if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)"; then
+              python -c "import pyarrow; assert pyarrow.__version__ == '23.0.0', pyarrow.__version__"
+            fi
             if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
               python -m pip check
             fi
@@ -176,6 +180,28 @@ jobs:
           chmod +x paimon-python/dev/lint-python.sh
           ./paimon-python/dev/lint-python.sh -e pytest_torch
 
+  pyarrow-compatibility:
+    name: PyArrow 23 compatibility
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    container: "python:3.12-slim"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r paimon-python/dev/requirements.txt \
+            pyarrow==23.0.0 pytest
+
+      - name: Run filesystem compatibility tests
+        working-directory: paimon-python
+        run: |
+          python -m unittest -q \
+            pypaimon.tests.oss_legacy_mode_test \
+            pypaimon.tests.file_io_test
+
   rust-plan:
     name: Rust Plan
     timeout-minutes: 90

paimon-python/README.md

@@ -14,6 +14,9 @@ Pypaimon requires Python 3.6+.
 
 The core dependencies are listed in `dev/requirements.txt`.
 The development dependencies are listed in `dev/requirements-dev.txt`.
+PyArrow 20 through 23 are supported, while the recommended range remains
+`pyarrow>=16,!=19.0.0,<20`. pip may select 23 unless this range is pinned
+explicitly.
 
 # OSS metadata commits
 

paimon-python/dev/requirements.txt

@@ -31,7 +31,7 @@ polars>=1,<2; python_version=="3.8"
 polars>=1.32,<2; python_version>="3.9"
 pyarrow>=6,<7; python_version < "3.7"
 pyarrow>=7,<13; python_version >= "3.7" and python_version < "3.8"
-pyarrow>=16,!=19.0.0,<20; python_version >= "3.8"
+pyarrow>=16,!=19.0.0,<24; python_version >= "3.8"
 pyroaring<=0.3.3; python_version < "3.7"
 pyroaring<=0.4.5; python_version == "3.7"
 pyroaring>=1.0.0; python_version >= "3.8"
@@ -40,5 +40,6 @@ requests>=2.21.0,<3
 urllib3>=1.26,<3
 zstandard>=0.19,<1
 backports.zstd>=1.0.0,<1.4.0; python_version >= "3.9" and python_version < "3.14"
+boto3>=1.36,<1.44; python_version >= "3.10"
 cramjam>=1.3.0,<3; python_version>="3.7"
 pyyaml>=5.4,<7

paimon-python/pypaimon/catalog/filesystem_catalog.py

@@ -103,13 +103,14 @@ def drop_database(self, name: str, ignore_if_not_exists: bool = False, cascade:
 
         # Check if database still has tables
         remaining_tables = self.list_tables(name)
-        if remaining_tables and not cascade:
-            raise ValueError(
-                f"Database {name} is not empty. "
-                f"Use cascade=True to drop all tables first."
-            )
-
-        self.file_io.delete(db_path, True)
+        if remaining_tables:
+            if cascade:
+                raise OSError(f"Database {name} changed during drop; remaining tables: "
+                              f"{remaining_tables}")
+            raise ValueError(f"Database {name} is not empty. "
+                             "Use cascade=True to drop all tables first.")
+
+        self.file_io.delete(db_path, False)
 
     def list_tables(self, database_name: str) -> list:
         try:

paimon-python/pypaimon/common/options/config.py

@@ -54,6 +54,9 @@ class S3Options:
         "S3 security token")
     S3_ENDPOINT = ConfigOptions.key("fs.s3.endpoint").string_type().no_default_value().with_description("S3 endpoint")
     S3_REGION = ConfigOptions.key("fs.s3.region").string_type().no_default_value().with_description("S3 region")
+    S3_DELETE_BATCH_ENABLED = ConfigOptions.key(
+        "fs.s3.delete.batch-enabled").boolean_type().default_value(False).with_description(
+            "Use native S3 batch deletion for custom endpoints")
 
 
 class GcsOptions: