feat(fs): introduce Jindo file system support

CMakeLists.txt

@@ -52,6 +52,7 @@ option(PAIMON_USE_UBSAN "Use Undefined Behavior Sanitizer" OFF)
 option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON)
 option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
+option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
 option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
 if(PAIMON_ENABLE_LUCENE)
     message(FATAL_ERROR "PAIMON_ENABLE_LUCENE is temporarily disabled. Please configure with -DPAIMON_ENABLE_LUCENE=OFF."
@@ -64,6 +65,9 @@ endif()
 if(PAIMON_ENABLE_AVRO)
     add_definitions(-DPAIMON_ENABLE_AVRO)
 endif()
+if(PAIMON_ENABLE_JINDO)
+    add_definitions(-DPAIMON_ENABLE_JINDO)
+endif()
 if(PAIMON_USE_CXX11_ABI)
     add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1)
 else()
@@ -405,6 +409,11 @@ if(PAIMON_BUILD_TESTS)
         list(APPEND TEST_STATIC_LINK_LIBS paimon_avro_file_format_shared)
         list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
     endif()
+    if(PAIMON_ENABLE_JINDO)
+        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
+        list(APPEND TEST_STATIC_LINK_LIBS paimon_jindo_file_system_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
+    endif()
     if(PAIMON_ENABLE_LUMINA)
         list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
         list(APPEND TEST_STATIC_LINK_LIBS paimon_lumina_index_shared)
@@ -437,6 +446,9 @@ config_summary_message()
 
 add_subdirectory(src/paimon)
 add_subdirectory(src/paimon/fs/local)
+if(PAIMON_ENABLE_JINDO)
+    add_subdirectory(src/paimon/fs/jindo)
+endif()
 add_subdirectory(src/paimon/format/blob)
 add_subdirectory(src/paimon/format/orc)
 add_subdirectory(src/paimon/format/parquet)

ci/scripts/build_paimon.sh

@@ -39,6 +39,7 @@ CMAKE_ARGS=(
     "-G Ninja"
     "-DCMAKE_BUILD_TYPE=${build_type}"
     "-DPAIMON_BUILD_TESTS=ON"
+    "-DPAIMON_ENABLE_JINDO=ON"
     "-DPAIMON_ENABLE_LUCENE=OFF"
 )
 

cmake_modules/ThirdpartyToolchain.cmake

@@ -281,6 +281,56 @@ else()
     endif()
 endif()
 
+if(APPLE)
+    set(JINDOSDK_C_DYNAMIC_LIB_NAME "jindosdk_c.${PAIMON_JINDOSDK_C_BUILD_VERSION}")
+    set(JINDOSDK_C_DYNAMIC_LIB_FILE "lib${JINDOSDK_C_DYNAMIC_LIB_NAME}.dylib")
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")
+        set(JINDOSDK_C_BUILD_SHA256_CHECKSUM
+            "${PAIMON_JINDOSDK_C_MACOS_X86_64_BUILD_SHA256_CHECKSUM}")
+        if(DEFINED ENV{PAIMON_JINDOSDK_C_MACOS_X86_64_URL})
+            set(JINDOSDK_C_SOURCE_URL "$ENV{PAIMON_JINDOSDK_C_MACOS_X86_64_URL}")
+        else()
+            set_urls(JINDOSDK_C_SOURCE_URL
+                     "https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-macos-11_0-x86_64.tar.gz"
+            )
+        endif()
+    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64")
+        set(JINDOSDK_C_BUILD_SHA256_CHECKSUM
+            "${PAIMON_JINDOSDK_C_MACOS_AARCH64_BUILD_SHA256_CHECKSUM}")
+        if(DEFINED ENV{PAIMON_JINDOSDK_C_MACOS_AARCH64_URL})
+            set(JINDOSDK_C_SOURCE_URL "$ENV{PAIMON_JINDOSDK_C_MACOS_AARCH64_URL}")
+        else()
+            set_urls(JINDOSDK_C_SOURCE_URL
+                     "https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-macos-11_0-aarch64.tar.gz"
+            )
+        endif()
+    endif()
+else()
+    set(JINDOSDK_C_DYNAMIC_LIB_NAME "jindosdk_c")
+    set(JINDOSDK_C_DYNAMIC_LIB_FILE "lib${JINDOSDK_C_DYNAMIC_LIB_NAME}.so")
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")
+        set(JINDOSDK_C_BUILD_SHA256_CHECKSUM
+            "${PAIMON_JINDOSDK_C_LINUX_X86_64_BUILD_SHA256_CHECKSUM}")
+        if(DEFINED ENV{PAIMON_JINDOSDK_C_LINUX_X86_64_URL})
+            set(JINDOSDK_C_SOURCE_URL "$ENV{PAIMON_JINDOSDK_C_LINUX_X86_64_URL}")
+        else()
+            set_urls(JINDOSDK_C_SOURCE_URL
+                     "https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-linux.tar.gz"
+            )
+        endif()
+    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64")
+        set(JINDOSDK_C_BUILD_SHA256_CHECKSUM
+            "${PAIMON_JINDOSDK_C_LINUX_AARCH64_BUILD_SHA256_CHECKSUM}")
+        if(DEFINED ENV{PAIMON_JINDOSDK_C_LINUX_AARCH64_URL})
+            set(JINDOSDK_C_SOURCE_URL "$ENV{PAIMON_JINDOSDK_C_LINUX_AARCH64_URL}")
+        else()
+            set_urls(JINDOSDK_C_SOURCE_URL
+                     "https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-linux-el7-aarch64.tar.gz"
+            )
+        endif()
+    endif()
+endif()
+
 set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 set(EP_C_FLAGS "${CMAKE_C_FLAGS}")
 string(REPLACE "-Wglobal-constructors" "" EP_CXX_FLAGS ${EP_CXX_FLAGS})
@@ -1105,6 +1155,97 @@ macro(build_lz4)
     add_dependencies(lz4 lz4_ep)
 endmacro()
 
+macro(build_jindosdk_c)
+    message(STATUS "Building jindosdk-c from precompiled package")
+
+    set(JINDOSDK_C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jindosdk_ep-install")
+    set(JINDOSDK_C_HOME "${JINDOSDK_C_PREFIX}")
+    set(JINDOSDK_C_INCLUDE_DIR "${JINDOSDK_C_PREFIX}/include")
+    set(JINDOSDK_C_LIB_DIR "${JINDOSDK_C_PREFIX}/lib/native")
+    set(JINDOSDK_C_DYNAMIC_LIB "${JINDOSDK_C_LIB_DIR}/${JINDOSDK_C_DYNAMIC_LIB_FILE}")
+
+    # Extract and install jindosdk from precompiled package
+    externalproject_add(jindosdk_ep
+                        URL ${JINDOSDK_C_SOURCE_URL}
+                        URL_HASH "SHA256=${JINDOSDK_C_BUILD_SHA256_CHECKSUM}"
+                        ${THIRDPARTY_LOG_OPTIONS}
+                        CONFIGURE_COMMAND ""
+                        BUILD_COMMAND ""
+                        INSTALL_COMMAND bash -c
+                                        "cp -r <SOURCE_DIR>/include/* ${JINDOSDK_C_INCLUDE_DIR}"
+                        COMMAND bash -c
+                                "cp -r <SOURCE_DIR>/lib/native/${JINDOSDK_C_DYNAMIC_LIB_FILE}* ${JINDOSDK_C_LIB_DIR}"
+                        BUILD_BYPRODUCTS "${JINDOSDK_C_DYNAMIC_LIB}")
+
+    # The include directory must exist before it is referenced by a target.
+    file(MAKE_DIRECTORY "${JINDOSDK_C_INCLUDE_DIR}")
+    file(MAKE_DIRECTORY "${JINDOSDK_C_LIB_DIR}")
+
+    add_library(jindosdk::c_sdk SHARED IMPORTED)
+    set_target_properties(jindosdk::c_sdk
+                          PROPERTIES IMPORTED_LOCATION "${JINDOSDK_C_DYNAMIC_LIB}"
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${JINDOSDK_C_INCLUDE_DIR}")
+    list(APPEND JINDOSDK_INCLUDE_DIR ${JINDOSDK_C_INCLUDE_DIR})
+
+    add_dependencies(jindosdk::c_sdk jindosdk_ep)
+    install(DIRECTORY "${JINDOSDK_C_LIB_DIR}/"
+            DESTINATION ${CMAKE_INSTALL_LIBDIR}
+            FILES_MATCHING
+            PATTERN "${JINDOSDK_C_DYNAMIC_LIB_FILE}*")
+
+endmacro()
+
+macro(build_jindosdk_nextarch)
+    message(STATUS "Building jindosdk-nextarch from local source")
+
+    set(JINDOSDK_NEXTARCH_PREFIX
+        "${CMAKE_CURRENT_BINARY_DIR}/jindosdk-nextarch_ep-install")
+    set(JINDOSDK_NEXTARCH_HOME "${JINDOSDK_NEXTARCH_PREFIX}")
+    set(JINDOSDK_NEXTARCH_INCLUDE_DIR "${JINDOSDK_NEXTARCH_PREFIX}/include")
+    set(JINDOSDK_NEXTARCH_LIB_DIR "${JINDOSDK_NEXTARCH_PREFIX}/lib")
+    set(JINDOSDK_NEXTARCH_SOURCE_DIR "${CMAKE_SOURCE_DIR}/third_party/jindosdk-nextarch")
+    set(JINDOSDK_NEXTARCH_STATIC_LIB
+        "${JINDOSDK_NEXTARCH_LIB_DIR}/libjindosdk-nextarch.a")
+
+    # Get jindosdk dependencies (headers and dynamic library)
+    get_target_property(JINDOSDK_C_INCLUDE_DIR jindosdk::c_sdk
+                        INTERFACE_INCLUDE_DIRECTORIES)
+    get_target_property(JINDOSDK_C_LIBRARY_LOCATION jindosdk::c_sdk IMPORTED_LOCATION)
+    get_filename_component(JINDOSDK_C_DIR_ROOT "${JINDOSDK_C_INCLUDE_DIR}" DIRECTORY)
+
+    # Compile flags for jindosdk-nextarch
+    set(JINDOSDK_NEXTARCH_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}")
+    set(JINDOSDK_NEXTARCH_CMAKE_C_FLAGS "${EP_C_FLAGS}")
+    set(JINDOSDK_NEXTARCH_CMAKE_ARGS
+        ${EP_COMMON_CMAKE_ARGS}
+        "-DCMAKE_INSTALL_PREFIX=${JINDOSDK_NEXTARCH_PREFIX}"
+        "-DCMAKE_CXX_FLAGS=${JINDOSDK_NEXTARCH_CMAKE_CXX_FLAGS}"
+        "-DCMAKE_C_FLAGS=${JINDOSDK_NEXTARCH_CMAKE_C_FLAGS}"
+        -DJINDOSDK_ROOT=${JINDOSDK_C_DIR_ROOT}
+        -DJINDOSDK_LIBRARY_NAME=${JINDOSDK_C_DYNAMIC_LIB_NAME})
+
+    externalproject_add(jindosdk-nextarch_ep
+                        SOURCE_DIR ${JINDOSDK_NEXTARCH_SOURCE_DIR}
+                        CMAKE_ARGS ${JINDOSDK_NEXTARCH_CMAKE_ARGS}
+                        BUILD_BYPRODUCTS "${JINDOSDK_NEXTARCH_STATIC_LIB}"
+                        DEPENDS jindosdk::c_sdk ${THIRDPARTY_LOG_OPTIONS})
+
+    # The include directory must exist before it is referenced by a target.
+    file(MAKE_DIRECTORY "${JINDOSDK_NEXTARCH_INCLUDE_DIR}")
+    file(MAKE_DIRECTORY "${JINDOSDK_NEXTARCH_LIB_DIR}")
+
+    add_library(jindosdk::nextarch STATIC IMPORTED)
+    set_target_properties(jindosdk::nextarch
+                          PROPERTIES IMPORTED_LOCATION "${JINDOSDK_NEXTARCH_STATIC_LIB}"
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${JINDOSDK_NEXTARCH_INCLUDE_DIR}")
+    target_link_libraries(jindosdk::nextarch INTERFACE jindosdk::c_sdk pthread dl)
+    list(APPEND JINDOSDK_INCLUDE_DIR ${JINDOSDK_NEXTARCH_INCLUDE_DIR})
+
+    add_dependencies(jindosdk::nextarch jindosdk-nextarch_ep)
+endmacro()
+
 macro(build_protobuf)
     message(STATUS "Building protobuf from source")
     set(PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf_ep-install")
@@ -1672,6 +1813,10 @@ if(PAIMON_ENABLE_ORC)
     resolve_dependency(Protobuf)
     resolve_dependency(ORC)
 endif()
+if(PAIMON_ENABLE_JINDO)
+    build_jindosdk_c()
+    build_jindosdk_nextarch()
+endif()
 if(PAIMON_ENABLE_LUCENE)
     build_boost()
     build_lucene()

docs/source/build_system.rst

@@ -86,6 +86,7 @@ Paimon provides a set of built-in optional plugins that you can link to as neede
 - File system plugins:
 
   - ``paimon_local_file_system_shared`` / ``paimon_local_file_system_static``
+  - ``paimon_jindo_file_system_shared`` / ``paimon_jindo_file_system_static``
 
 - Index plugins:
 

docs/source/building.rst

@@ -123,6 +123,7 @@ boolean flags to ``cmake``.
 
 * ``-DPAIMON_ENABLE_ORC=ON``: Paimon integration with Apache ORC
 * ``-DPAIMON_ENABLE_AVRO=ON``: Apache Avro libraries and Paimon integration
+* ``-DPAIMON_ENABLE_JINDO=ON``: Support for Alibaba Jindo filesystems
 
 Third-party dependency source
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/paimon/fs/jindo/CMakeLists.txt

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if(PAIMON_ENABLE_JINDO)
+    set(PAIMON_JINDO_FILE_SYSTEM jindo_file_system.cpp jindo_file_system_factory.cpp)
+
+    add_paimon_lib(paimon_jindo_file_system
+                   SOURCES
+                   ${PAIMON_JINDO_FILE_SYSTEM}
+                   EXTRA_INCLUDES
+                   ${JINDOSDK_INCLUDE_DIR}
+                   DEPENDENCIES
+                   paimon_shared
+                   jindosdk::nextarch
+                   STATIC_LINK_LIBS
+                   jindosdk::nextarch
+                   fmt
+                   SHARED_LINK_LIBS
+                   paimon_shared
+                   SHARED_LINK_FLAGS
+                   ${PAIMON_VERSION_SCRIPT_FLAGS})
+endif()

src/paimon/fs/jindo/jindo_file_status.h

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <utility>
+
+#include "JdoFileInfo.hpp"  // NOLINT(build/include_subdir)
+#include "paimon/fs/file_system.h"
+
+namespace paimon::jindo {
+class JindoBasicFileStatus : public BasicFileStatus {
+ public:
+    explicit JindoBasicFileStatus(JdoFileInfo&& file_info) : file_info_(std::move(file_info)) {}
+
+    std::string GetPath() const override {
+        return file_info_.getPath();
+    }
+
+    bool IsDir() const override {
+        return file_info_.isDir();
+    }
+
+ private:
+    JdoFileInfo file_info_;
+};
+
+class JindoFileStatus : public FileStatus {
+ public:
+    explicit JindoFileStatus(JdoFileInfo&& file_info) : file_info_(std::move(file_info)) {}
+
+    std::string GetPath() const override {
+        return file_info_.getPath();
+    }
+
+    uint64_t GetLen() const override {
+        return file_info_.getLength();
+    }
+
+    int64_t GetModificationTime() const override {
+        return file_info_.getMtime();
+    }
+
+    bool IsDir() const override {
+        return file_info_.isDir();
+    }
+
+ private:
+    JdoFileInfo file_info_;
+};
+}  // namespace paimon::jindo