From 0bb070779db3102f45c63716e62331edcd77b610 Mon Sep 17 00:00:00 2001 From: Peter Cseh Date: Wed, 2 Sep 2026 10:15:23 +0000 Subject: [PATCH 1/4] [lake] Fail fast when a lake plugin is shadowed by a jar in lib A LakeStoragePlugin can be provided either from the plugins directory, where it is loaded in an isolated class loader together with the dependencies bundled next to it, or from the main classpath. Providing it from both locations cannot work: classes under "org.apache.fluss." are loaded parent-first, so the copy on the main classpath shadows the one in the plugins directory while the dependencies bundled next to the plugin stay invisible. This is what happens when a fluss-lake- jar is copied into /lib in addition to plugins//. Paimon then fails deep inside CatalogContext with a bare "NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration", which gives no hint about the actual cause. Detect the duplicate up front and fail with a message naming both locations and the jar to remove. Resolution is unchanged when the plugin comes from only one location, so the classpath-only setups used by the Flink tiering job keep working. This addresses item (1) of the analysis in #1314. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019dqbppMMENgRMdJxwv5GuM --- .../lakestorage/LakeStoragePluginSetUp.java | 71 +++++++++++++------ .../lake/lakestorage/LakeStorageTest.java | 31 ++++++++ .../TestingClasspathLakeStoragePlugin.java | 36 ++++++++++ ...e.fluss.lake.lakestorage.LakeStoragePlugin | 17 +++++ 4 files changed, 135 insertions(+), 20 deletions(-) create mode 100644 fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/TestingClasspathLakeStoragePlugin.java create mode 100644 fluss-common/src/test/resources/META-INF/services/org.apache.fluss.lake.lakestorage.LakeStoragePlugin diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStoragePluginSetUp.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStoragePluginSetUp.java index cc5270e1cdb..7b629f9d1dd 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStoragePluginSetUp.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStoragePluginSetUp.java @@ -17,11 +17,12 @@ package org.apache.fluss.lake.lakestorage; +import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.plugin.PluginManager; -import org.apache.fluss.shaded.guava32.com.google.common.collect.Iterators; import javax.annotation.Nullable; +import java.util.Collections; import java.util.Iterator; import java.util.Objects; import java.util.ServiceLoader; @@ -32,35 +33,65 @@ */ public class LakeStoragePluginSetUp { + /** + * Finds the {@link LakeStoragePlugin} for the given datalake format, from the plugins directory + * or from the main classpath. + * + *

Providing it from both cannot work: {@code org.apache.fluss.} is parent-first, so the copy + * on the main classpath shadows the plugin and is loaded without the dependencies bundled next + * to it. That is rejected here rather than failing later with a {@link NoClassDefFoundError}. + */ public static LakeStoragePlugin fromDataLakeFormat( final String dataLakeFormat, @Nullable final PluginManager pluginManager) { - // now, load lake storage plugin - Iterator lakeStoragePluginIterator = - getAllLakeStoragePlugins(pluginManager); + LakeStoragePlugin fromPluginsDir = + findByIdentifier(loadFromPluginManager(pluginManager), dataLakeFormat); + LakeStoragePlugin fromClasspath = findByIdentifier(loadFromClasspath(), dataLakeFormat); - while (lakeStoragePluginIterator.hasNext()) { - LakeStoragePlugin lakeStoragePlugin = lakeStoragePluginIterator.next(); + if (fromPluginsDir != null && fromClasspath != null) { + throw new FlussRuntimeException( + String.format( + "Found two LakeStoragePlugin for datalake format '%s': one in the plugins " + + "directory and one on the main classpath (usually /lib). " + + "The copy on the main classpath shadows the one in the plugins " + + "directory, but is loaded without the dependencies bundled next to " + + "the plugin, such as Hadoop. This later fails with errors like " + + "'NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration'. " + + "Remove the fluss-lake-%s jar from /lib and keep it only " + + "in /plugins/%s/.", + dataLakeFormat, dataLakeFormat, dataLakeFormat)); + } + + LakeStoragePlugin lakeStoragePlugin = + fromPluginsDir != null ? fromPluginsDir : fromClasspath; + if (lakeStoragePlugin == null) { + throw new UnsupportedOperationException( + "No LakeStoragePlugin can be found for datalake format: " + dataLakeFormat); + } + return PluginLakeStorageWrapper.of(lakeStoragePlugin); + } + + @Nullable + private static LakeStoragePlugin findByIdentifier( + Iterator lakeStoragePlugins, String dataLakeFormat) { + while (lakeStoragePlugins.hasNext()) { + LakeStoragePlugin lakeStoragePlugin = lakeStoragePlugins.next(); if (Objects.equals(lakeStoragePlugin.identifier(), dataLakeFormat)) { - return PluginLakeStorageWrapper.of(lakeStoragePlugin); + return lakeStoragePlugin; } } - - // if come here, means we haven't found LakeStoragePlugin match the configured - // datalake, throw exception - throw new UnsupportedOperationException( - "No LakeStoragePlugin can be found for datalake format: " + dataLakeFormat); + return null; } - private static Iterator getAllLakeStoragePlugins( + private static Iterator loadFromPluginManager( @Nullable PluginManager pluginManager) { - final Iterator pluginIteratorSPI = - ServiceLoader.load( - LakeStoragePlugin.class, LakeStoragePlugin.class.getClassLoader()) - .iterator(); if (pluginManager == null) { - return pluginIteratorSPI; - } else { - return Iterators.concat(pluginManager.load(LakeStoragePlugin.class), pluginIteratorSPI); + return Collections.emptyIterator(); } + return pluginManager.load(LakeStoragePlugin.class); + } + + private static Iterator loadFromClasspath() { + return ServiceLoader.load(LakeStoragePlugin.class, LakeStoragePlugin.class.getClassLoader()) + .iterator(); } } diff --git a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java index b527bb6e483..25f97ccb80f 100644 --- a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java @@ -18,6 +18,7 @@ package org.apache.fluss.lake.lakestorage; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.lake.source.LakeSource; @@ -98,6 +99,36 @@ void testWithPluginManager() throws Exception { .isInstanceOf(TestPaimonLakeCatalog.class); } + @Test + void testPluginFoundInBothPluginsDirAndClasspath() { + // same identifier from the plugin manager and from SPI, as when the jar is also in lib + final Map, Iterator> lakeStoragePlugins = new HashMap<>(); + lakeStoragePlugins.put( + LakeStoragePlugin.class, + Collections.singletonList(new TestingClasspathLakeStoragePlugin()).iterator()); + + assertThatThrownBy( + () -> + LakeStoragePluginSetUp.fromDataLakeFormat( + TestingClasspathLakeStoragePlugin.IDENTIFIER, + new TestingPluginManager(lakeStoragePlugins))) + .isInstanceOf(FlussRuntimeException.class) + .hasMessageContaining("Found two LakeStoragePlugin for datalake format") + .hasMessageContaining(TestingClasspathLakeStoragePlugin.IDENTIFIER) + .hasMessageContaining("/lib"); + } + + @Test + void testPluginFoundOnClasspathOnly() { + LakeStoragePlugin lakeStoragePlugin = + LakeStoragePluginSetUp.fromDataLakeFormat( + TestingClasspathLakeStoragePlugin.IDENTIFIER, null); + + assertThat(lakeStoragePlugin).isInstanceOf(PluginLakeStorageWrapper.class); + assertThat(lakeStoragePlugin.identifier()) + .isEqualTo(TestingClasspathLakeStoragePlugin.IDENTIFIER); + } + private static class TestingPluginManager implements PluginManager { private final Map, Iterator> plugins; diff --git a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/TestingClasspathLakeStoragePlugin.java b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/TestingClasspathLakeStoragePlugin.java new file mode 100644 index 00000000000..2e4c3f82032 --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/TestingClasspathLakeStoragePlugin.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.lakestorage; + +import org.apache.fluss.config.Configuration; + +/** A {@link LakeStoragePlugin} registered via SPI on the main classpath. */ +public class TestingClasspathLakeStoragePlugin implements LakeStoragePlugin { + + public static final String IDENTIFIER = "test-classpath-plugin"; + + @Override + public String identifier() { + return IDENTIFIER; + } + + @Override + public LakeStorage createLakeStorage(Configuration configuration) { + throw new UnsupportedOperationException("Not implemented"); + } +} diff --git a/fluss-common/src/test/resources/META-INF/services/org.apache.fluss.lake.lakestorage.LakeStoragePlugin b/fluss-common/src/test/resources/META-INF/services/org.apache.fluss.lake.lakestorage.LakeStoragePlugin new file mode 100644 index 00000000000..d29c0f9ea0b --- /dev/null +++ b/fluss-common/src/test/resources/META-INF/services/org.apache.fluss.lake.lakestorage.LakeStoragePlugin @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.fluss.lake.lakestorage.TestingClasspathLakeStoragePlugin From b7793b7e774b524b17b5b391badea76aab25ced4 Mon Sep 17 00:00:00 2001 From: Peter Cseh Date: Wed, 2 Sep 2026 10:15:23 +0000 Subject: [PATCH 2/4] [test] Cover the plugins directory layout of a deployed cluster Add tests for the plugin layout users actually deploy, where each plugin lives in its own sub directory of /plugins and the main classpath is /lib. They cover several plugins installed side by side, isolation between them, and the shadowing that breaks a deployment: a class also present on the main classpath is loaded from there instead of from the plugin jar whenever its package is parent-first, leaving the dependencies bundled next to the plugin invisible. A companion test pins the healthy case, where the same layout resolves from the plugin jar. This addresses item (5) of the analysis in #1314. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019dqbppMMENgRMdJxwv5GuM --- .../plugin/PluginDirectoryLayoutTest.java | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 fluss-common/src/test/java/org/apache/fluss/plugin/PluginDirectoryLayoutTest.java diff --git a/fluss-common/src/test/java/org/apache/fluss/plugin/PluginDirectoryLayoutTest.java b/fluss-common/src/test/java/org/apache/fluss/plugin/PluginDirectoryLayoutTest.java new file mode 100644 index 00000000000..6e367d5a6c7 --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/plugin/PluginDirectoryLayoutTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.plugin; + +import org.apache.fluss.plugin.jar.plugina.DynamicClassA; +import org.apache.fluss.plugin.jar.plugina.TestServiceA; +import org.apache.fluss.shaded.guava32.com.google.common.collect.Lists; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.fluss.utils.Preconditions.checkState; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Tests for the plugin directory layout of a deployed cluster, where each plugin lives in its own + * sub directory of {@code /plugins} and the main classpath is {@code /lib}. + */ +class PluginDirectoryLayoutTest extends PluginTestBase { + + @TempDir private Path tmp; + + @Test + void testMultiplePluginsLoadedSimultaneously() throws Exception { + PluginManager pluginManager = + createPluginManager( + new String[] {TestSpi.class.getName(), OtherTestSpi.class.getName()}, + "multi"); + + List plugins = Lists.newArrayList(pluginManager.load(TestSpi.class)); + + assertThat(plugins).hasSize(2); + Set classLoaders = new HashSet<>(); + for (TestSpi plugin : plugins) { + assertThat(plugin.testMethod()).isNotNull(); + assertThat(plugin.getClassLoader()).isNotSameAs(PARENT_CLASS_LOADER); + classLoaders.add(plugin.getClassLoader()); + } + assertThat(classLoaders).hasSize(2); + } + + @Test + void testPluginsDoNotSeeEachOthersClasses() throws Exception { + PluginManager pluginManager = + createPluginManager( + new String[] {TestSpi.class.getName(), OtherTestSpi.class.getName()}, + "isolation"); + + List plugins = Lists.newArrayList(pluginManager.load(TestSpi.class)); + assertThat(plugins).hasSize(2); + + // DynamicClassA is bundled in plugin A's jar only + ClassLoader loaderOfA = classLoaderOf(plugins, TestServiceA.class.getName()); + ClassLoader loaderOfB = otherClassLoader(plugins, loaderOfA); + + assertThat(loaderOfA.loadClass(DynamicClassA.class.getName())).isNotNull(); + assertThatThrownBy(() -> loaderOfB.loadClass(DynamicClassA.class.getName())) + .isInstanceOf(ClassNotFoundException.class); + } + + /** The main classpath copy wins, leaving the dependencies bundled next to the plugin unused. */ + @Test + void testClassOnMainClasspathShadowsPluginCopy() throws Exception { + // TestServiceA is on the main classpath, standing in for a jar also copied into lib + PluginManager pluginManager = + createPluginManager( + new String[] { + TestSpi.class.getName(), + OtherTestSpi.class.getName(), + TestServiceA.class.getName() + }, + "shadowed"); + + List plugins = Lists.newArrayList(pluginManager.load(TestSpi.class)); + TestSpi fromPluginA = pluginNamed(plugins, TestServiceA.class.getName()); + + assertThat(fromPluginA).isInstanceOf(TestServiceA.class); + assertThat(fromPluginA.getClassLoader()).isSameAs(PARENT_CLASS_LOADER); + } + + @Test + void testPluginCopyIsUsedWhenNotOnMainClasspath() throws Exception { + PluginManager pluginManager = + createPluginManager( + new String[] {TestSpi.class.getName(), OtherTestSpi.class.getName()}, + "not-shadowed"); + + List plugins = Lists.newArrayList(pluginManager.load(TestSpi.class)); + TestSpi fromPluginA = pluginNamed(plugins, TestServiceA.class.getName()); + + // same class name, but a distinct class loaded from the plugin jar + assertThat(fromPluginA).isNotInstanceOf(TestServiceA.class); + assertThat(fromPluginA.getClassLoader()).isNotSameAs(PARENT_CLASS_LOADER); + } + + /** Builds a {@code plugins/} root holding plugin A and plugin B in separate sub directories. */ + private PluginManager createPluginManager(String[] parentPatterns, String name) + throws Exception { + File pluginRootFolder = new File(tmp.toFile(), name); + File pluginAFolder = new File(pluginRootFolder, "A"); + File pluginBFolder = new File(pluginRootFolder, "B"); + checkState(pluginAFolder.mkdirs()); + checkState(pluginBFolder.mkdirs()); + Files.copy(locateJarFile(PLUGIN_A).toPath(), Paths.get(pluginAFolder.toString(), PLUGIN_A)); + Files.copy(locateJarFile(PLUGIN_B).toPath(), Paths.get(pluginBFolder.toString(), PLUGIN_B)); + + Collection descriptors = + new DirectoryBasedPluginFinder(pluginRootFolder.toPath()).findPlugins(); + checkState(descriptors.size() == 2); + return new DefaultPluginManager(descriptors, PARENT_CLASS_LOADER, parentPatterns); + } + + private static TestSpi pluginNamed(List plugins, String className) { + for (TestSpi plugin : plugins) { + if (plugin.getClass().getName().equals(className)) { + return plugin; + } + } + throw new AssertionError("No plugin found with class name " + className); + } + + private static ClassLoader classLoaderOf(List plugins, String className) { + return pluginNamed(plugins, className).getClassLoader(); + } + + private static ClassLoader otherClassLoader(List plugins, ClassLoader classLoader) { + for (TestSpi plugin : plugins) { + if (plugin.getClassLoader() != classLoader) { + return plugin.getClassLoader(); + } + } + throw new AssertionError("Expected a second, distinct plugin class loader"); + } +} From 5f4e0dd600f2008e0fc4e84919357bfbd949f72b Mon Sep 17 00:00:00 2001 From: Peter Cseh Date: Wed, 2 Sep 2026 10:15:23 +0000 Subject: [PATCH 3/4] [docs] Document plugin placement and add plugin troubleshooting Two docs pages linked to /downloads#filesystem-jars, but the downloads page had no such section. Add it, listing the filesystem plugin jars of the latest release and which of them the binary release already ships, so both links resolve. Also spell out the choice between the two artifacts serving hdfs://: fluss-fs-hdfs bundles Hadoop and is what the distribution ships, while the thin fluss-fs-hadoop expects Hadoop via HADOOP_CLASSPATH. Exactly one of them should be installed. The filesystem overview still credited fluss-fs-hadoop for the bundled plugin, which has been fluss-fs-hdfs since the artifacts were split. Add a troubleshooting page explaining how plugins are loaded, why a jar copied into /lib shadows the copy under plugins/ and loses the dependencies bundled next to it, and what each resulting error means. Warn about it where users hit it: the filesystem overview, the lakehouse deployment guide, and the downloads page. The lakehouse guide also now says explicitly that its ${FLINK_HOME}/lib instructions are about the Flink installation, not the Fluss one. This addresses items (2) and (6) of the analysis in #1314. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019dqbppMMENgRMdJxwv5GuM --- .../deploying-streaming-lakehouse.md | 11 +- .../tiered-storage/filesystems/overview.md | 17 ++- .../maintenance/troubleshooting-plugins.md | 126 ++++++++++++++++++ website/src/pages/downloads.md | 43 +++++- 4 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 website/docs/maintenance/troubleshooting-plugins.md diff --git a/website/docs/install-deploy/deploying-streaming-lakehouse.md b/website/docs/install-deploy/deploying-streaming-lakehouse.md index e3752fc6aa8..d5316ab69ed 100644 --- a/website/docs/install-deploy/deploying-streaming-lakehouse.md +++ b/website/docs/install-deploy/deploying-streaming-lakehouse.md @@ -124,6 +124,14 @@ See [set_cluster_configs](../engine-flink/procedures.md#set_cluster_configs) for Add JARs to `${FLUSS_HOME}/plugins//` based on your configuration: +:::warning +These JARs go in `${FLUSS_HOME}/plugins//`, together with the `fluss-lake-` JAR and +the dependencies it needs. Do not also place them in `${FLUSS_HOME}/lib`: a copy on the main +classpath shadows the plugin and is loaded without its bundled dependencies, so the CoordinatorServer +fails to start with errors such as `NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration`. +See [Troubleshooting Plugins](../maintenance/troubleshooting-plugins.md). +::: + @@ -169,7 +177,8 @@ The Tiering Service is a Flink job that continuously tiers data from Fluss to th ### Flink JARs -Add the following to `${FLINK_HOME}/lib`: +Add the following to `${FLINK_HOME}/lib` — this is the **Flink** installation, not +`${FLUSS_HOME}/lib`: diff --git a/website/docs/maintenance/tiered-storage/filesystems/overview.md b/website/docs/maintenance/tiered-storage/filesystems/overview.md index c02abbda67d..fa52dfe0ca9 100644 --- a/website/docs/maintenance/tiered-storage/filesystems/overview.md +++ b/website/docs/maintenance/tiered-storage/filesystems/overview.md @@ -30,7 +30,7 @@ Never use local file system as remote storage in production as it is not fault-t ## Pluggable File Systems The Fluss project supports the following file systems: -- **[HDFS](hdfs.md)** is supported by `fluss-fs-hadoop` and registered under the `hdfs://` URI scheme. HDFS filesystem is included in default Fluss binary distribution, so you can use it directly without manual installation. +- **[HDFS](hdfs.md)** is supported by `fluss-fs-hdfs` and registered under the `hdfs://` URI scheme. HDFS filesystem is included in default Fluss binary distribution, so you can use it directly without manual installation. `fluss-fs-hdfs` bundles its own Hadoop; the thin `fluss-fs-hadoop` artifact provides the same scheme for setups that supply Hadoop through `HADOOP_CLASSPATH`. Install exactly one of them, see [Choosing an HDFS JAR](/downloads#choosing-an-hdfs-jar). - **[Aliyun OSS](oss.md)** is supported by `fluss-fs-oss` and registered under the `oss://` URI scheme. OSS filesystem is included in default Fluss binary distribution, so you can use it directly without manual installation. @@ -42,4 +42,17 @@ The Fluss project supports the following file systems: - **[Tencent Cloud COS](cos.md)** is supported by `fluss-fs-cos` and registered under the `cosn://` URI scheme. Please make sure to [manually install the COS plugin](cos.md#install-cos-plugin-manually). -The implementation is based on [Hadoop Project](https://hadoop.apache.org/) but is self-contained with no dependency footprint. \ No newline at end of file +The implementation is based on [Hadoop Project](https://hadoop.apache.org/) but is self-contained with no dependency footprint. + +## Installing a Filesystem Plugin + +A filesystem plugin JAR goes into its own sub directory of `${FLUSS_HOME}/plugins`, named after the +URI scheme, for example `${FLUSS_HOME}/plugins/s3/`. Each sub directory is loaded in an isolated +class loader together with the dependencies next to it. + +:::warning +Never copy a plugin JAR into `${FLUSS_HOME}/lib` as well. The copy on the main classpath shadows the +one in the plugins directory and is loaded without its bundled dependencies, which fails at runtime +with errors such as `NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration`. See +[Troubleshooting Plugins](../../troubleshooting-plugins.md). +::: \ No newline at end of file diff --git a/website/docs/maintenance/troubleshooting-plugins.md b/website/docs/maintenance/troubleshooting-plugins.md new file mode 100644 index 00000000000..9f12a3eb8ea --- /dev/null +++ b/website/docs/maintenance/troubleshooting-plugins.md @@ -0,0 +1,126 @@ +--- +title: Troubleshooting Plugins +sidebar_position: 6 +--- + +# Troubleshooting Plugins + +Fluss loads filesystem, lake format and metric reporter implementations as *plugins*. Most plugin +problems come down to a single question: **is the JAR in the right directory?** This page explains +how plugins are loaded and how to recognise the errors that follow when they are not. + +## How plugins are loaded + +A Fluss installation has two very different places to put a JAR: + +``` +${FLUSS_HOME}/ +├── lib/ # the main classpath, shared by everything +└── plugins/ # one isolated sub directory per plugin + ├── hdfs/ + │ └── fluss-fs-hdfs-.jar + ├── s3/ + │ └── fluss-fs-s3-.jar + └── paimon/ + ├── fluss-lake-paimon-.jar + ├── paimon-bundle-.jar + └── flink-shaded-hadoop-2-uber-.jar +``` + +Every sub directory of `plugins/` is loaded in its own class loader, together with the dependencies +that sit next to it. That is what lets the S3 plugin and the Paimon plugin each carry their own, +possibly conflicting, versions of a library without interfering. + +The plugins directory defaults to `plugins` and can be overridden with the `FLUSS_PLUGINS_DIR` +environment variable. + +:::warning +Do not copy a plugin JAR into `${FLUSS_HOME}/lib`. Classes in `org.apache.fluss.` and +`org.apache.hadoop.` are loaded *parent-first*, so a copy on the main classpath wins over the copy +in the plugins directory — but it is loaded **without** the dependencies bundled next to the plugin. +The plugin then fails at runtime even though the JAR is clearly present. +::: + +The parent-first packages are controlled by `plugin.classloader.parent-first-patterns.default`, and +can be extended with `plugin.classloader.parent-first-patterns.additional`. + +## `${FLUSS_HOME}/lib` is not `${FLINK_HOME}/lib` + +The [Tiering Service](../streaming-lakehouse/tiering-service.md) is a **Flink** job, and its JARs — +including `fluss-lake-` — do go into `${FLINK_HOME}/lib`. That instruction is about the +Flink installation, not the Fluss one. Putting the same JAR into `${FLUSS_HOME}/lib` is what breaks +the Fluss servers. + +| JAR | Goes in | +|-----|---------| +| Filesystem plugin (`fluss-fs-*`) | `${FLUSS_HOME}/plugins//` | +| Lake format plugin (`fluss-lake-*`) on the servers | `${FLUSS_HOME}/plugins//` | +| Lake format plugin for the Tiering Service | `${FLINK_HOME}/lib` | +| Flink connector (`fluss-flink-*`) | `${FLINK_HOME}/lib` | + +## Common errors + +### Found two LakeStoragePlugin for datalake format + +``` +Found two LakeStoragePlugin for datalake format 'paimon': one in the plugins directory +and one on the main classpath (usually /lib). ... +``` + +The same lake format is installed twice. Delete the `fluss-lake-` JAR from +`${FLUSS_HOME}/lib` and keep only the one in `${FLUSS_HOME}/plugins//`, then restart the +server. This check exists so the deployment fails immediately with an explanation, rather than later +with the `NoClassDefFoundError` below. + +### NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration + +``` +java.lang.NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration + at org.apache.paimon.catalog.CatalogContext.(...) +Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hdfs.HdfsConfiguration +``` + +The Paimon plugin was loaded from the main classpath instead of from +`${FLUSS_HOME}/plugins/paimon/`, so the `flink-shaded-hadoop-2-uber` JAR bundled next to it was +never visible. Remove the copy from `${FLUSS_HOME}/lib`. + +Adding a Hadoop JAR to `${FLUSS_HOME}/lib` also makes the error disappear, but it is a workaround: +it puts Hadoop on the shared classpath for every component, which is exactly the version conflict +the plugin isolation is meant to prevent. Prefer removing the misplaced JAR. + +### Hadoop is not in the classpath, or some classes are missing + +``` +Cannot support file system for 'hdfs' via Hadoop, because Hadoop is not in the +classpath, or some classes are missing from the classpath. +``` + +The `hdfs://` scheme resolved to an implementation that has no Hadoop with it. Check that +`${FLUSS_HOME}/plugins/hdfs/` contains `fluss-fs-hdfs`, which bundles Hadoop. If you deliberately +use the thin `fluss-fs-hadoop` artifact instead, Hadoop must be supplied through `HADOOP_CLASSPATH`: + +```bash +export HADOOP_CLASSPATH=`hadoop classpath` +``` + +Do not install both artifacts — see [Choosing an HDFS JAR](/downloads#choosing-an-hdfs-jar). + +### No LakeStoragePlugin can be found for datalake format + +``` +No LakeStoragePlugin can be found for datalake format: paimon +``` + +`datalake.format` is set, but the matching plugin is not installed at all. Add the +`fluss-lake-` JAR and its dependencies to `${FLUSS_HOME}/plugins//` and restart. See +[Deploying Streaming Lakehouse](../install-deploy/deploying-streaming-lakehouse.md). + +## Checklist + +When a plugin misbehaves, check in this order: + +1. The JAR is in `${FLUSS_HOME}/plugins//`, in its own sub directory. +2. No copy of it is in `${FLUSS_HOME}/lib`. +3. The dependencies it needs sit next to it in the same sub directory. +4. Only one artifact provides each URI scheme or lake format. +5. The servers were restarted after the change — plugins are discovered at startup. diff --git a/website/src/pages/downloads.md b/website/src/pages/downloads.md index 15e84f76488..6e1305a8010 100644 --- a/website/src/pages/downloads.md +++ b/website/src/pages/downloads.md @@ -43,4 +43,45 @@ Read the [release blog](/blog/releases/0.8/) about the new features and signific ### Verifying Downloads -Downloaded Apache Fluss (Incubating) artifacts can be verified by following [this tutorial](https://www.apache.org/info/verification.html) of the Apache Software Foundation using the Apache Fluss (Incubating) release-signing [KEYS](https://downloads.apache.org/incubator/fluss/KEYS). \ No newline at end of file +Downloaded Apache Fluss (Incubating) artifacts can be verified by following [this tutorial](https://www.apache.org/info/verification.html) of the Apache Software Foundation using the Apache Fluss (Incubating) release-signing [KEYS](https://downloads.apache.org/incubator/fluss/KEYS). + +------------------ + +## Filesystem JARs + +Fluss reaches [remote storage](/docs/maintenance/tiered-storage/remote-storage) through pluggable filesystem plugins. The binary +release already ships the HDFS, S3 and OSS plugins under `plugins//`, so those work out of +the box. The JARs below are for adding a filesystem that is not bundled, for upgrading a single +plugin in place, or for the Tiering Service running on Flink. + +:::warning +A filesystem plugin JAR belongs in `${FLUSS_HOME}/plugins//` and **must not** also be copied +into `${FLUSS_HOME}/lib`. A copy on the main classpath shadows the one in the plugins directory and +is loaded without the dependencies bundled next to it, which fails at runtime with errors such as +`NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration`. See +[Troubleshooting Plugins](/docs/maintenance/troubleshooting-plugins). +::: + +The following are the plugins of the latest stable release, Apache Fluss (Incubating) 0.9.1: + +| Filesystem | URI scheme | In the binary release | JAR | +|------------|------------|-----------------------|-----| +| HDFS | `hdfs://` | yes | [fluss-fs-hdfs-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-hdfs/0.9.1-incubating/fluss-fs-hdfs-0.9.1-incubating.jar) | +| AWS S3 | `s3://` | yes | [fluss-fs-s3-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-s3/0.9.1-incubating/fluss-fs-s3-0.9.1-incubating.jar) | +| Aliyun OSS | `oss://` | yes | [fluss-fs-oss-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-oss/0.9.1-incubating/fluss-fs-oss-0.9.1-incubating.jar) | +| Google Cloud Storage | `gs://` | no | [fluss-fs-gs-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-gs/0.9.1-incubating/fluss-fs-gs-0.9.1-incubating.jar) | +| Azure Blob Storage | `abfs://` | no | [fluss-fs-azure-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-azure/0.9.1-incubating/fluss-fs-azure-0.9.1-incubating.jar) | +| HuaweiCloud OBS | `obs://` | no | [fluss-fs-obs-0.9.1-incubating.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-fs-obs/0.9.1-incubating/fluss-fs-obs-0.9.1-incubating.jar) | + +### Choosing an HDFS JAR + +Two artifacts provide the `hdfs://` scheme and they differ only in whether Hadoop travels with them. +Pick one, never both: + +| Artifact | Hadoop dependencies | Use it when | +|----------|---------------------|-------------| +| `fluss-fs-hdfs` | bundled, self-contained (~34 MB) | Default, and what the binary release ships. Works on machines with no Hadoop installation. | +| `fluss-fs-hadoop` | not bundled (~10 KB) | You provide Hadoop yourself through `HADOOP_CLASSPATH`, for example to match your cluster's Hadoop version or to use Kerberos. | + +Deploying both at once gives two implementations of the same scheme, so use exactly one. See +[HDFS](/docs/maintenance/tiered-storage/filesystems/hdfs) for the Hadoop configuration options. From 3e41d791392ee420d7a547f18163dd24d9df5c81 Mon Sep 17 00:00:00 2001 From: gezapeti <2914398+gezapeti@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:19:09 +0200 Subject: [PATCH 4/4] Update downloads.md --- website/src/pages/downloads.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/src/pages/downloads.md b/website/src/pages/downloads.md index 6e1305a8010..ca13e1221ed 100644 --- a/website/src/pages/downloads.md +++ b/website/src/pages/downloads.md @@ -59,7 +59,7 @@ A filesystem plugin JAR belongs in `${FLUSS_HOME}/plugins//` and **must into `${FLUSS_HOME}/lib`. A copy on the main classpath shadows the one in the plugins directory and is loaded without the dependencies bundled next to it, which fails at runtime with errors such as `NoClassDefFoundError: org/apache/hadoop/hdfs/HdfsConfiguration`. See -[Troubleshooting Plugins](/docs/maintenance/troubleshooting-plugins). +[Troubleshooting Plugins](/docs/next/maintenance/troubleshooting-plugins). ::: The following are the plugins of the latest stable release, Apache Fluss (Incubating) 0.9.1: @@ -84,4 +84,4 @@ Pick one, never both: | `fluss-fs-hadoop` | not bundled (~10 KB) | You provide Hadoop yourself through `HADOOP_CLASSPATH`, for example to match your cluster's Hadoop version or to use Kerberos. | Deploying both at once gives two implementations of the same scheme, so use exactly one. See -[HDFS](/docs/maintenance/tiered-storage/filesystems/hdfs) for the Hadoop configuration options. +[HDFS](/docs/next/maintenance/tiered-storage/filesystems/hdfs) for the Hadoop configuration options.