Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 36 additions & 22 deletions .github/workflows/core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -332,8 +332,35 @@ jobs:
strategy:
fail-fast: false
matrix:
python: [ 3.9 ]
java: [ 11, 17 ]
include:
- java: 11
python: '3.9'
spark: '3.3'
scala: '2.12'
- java: 17
python: '3.9'
spark: '3.4'
scala: '2.12'
- java: 11
python: '3.9'
spark: '3.5'
scala: '2.12'
- java: 17
python: '3.9'
spark: '3.5'
scala: '2.13'
- java: 17
python: '3.9'
spark: '4.0'
scala: '2.13'
- java: 17
python: '3.14'
spark: '4.1'
scala: '2.13'
- java: 17
python: '3.14'
spark: '4.2'
scala: '2.13'
steps:
- name: Checkout
uses: actions/checkout@v5
Expand All @@ -356,7 +383,7 @@ jobs:
restore-keys: |
${{ runner.os }}-zeppelin-
- name: install environment
run: ./mvnw install -DskipTests -pl spark-submit,spark/scala-2.12,spark/scala-2.13 -am ${MAVEN_ARGS}
run: ./mvnw install -DskipTests -pl spark-submit,spark/scala-${{ matrix.scala }} -am ${MAVEN_ARGS}
- name: Setup conda environment with python ${{ matrix.python }}
uses: conda-incubator/setup-miniconda@v4
with:
Expand All @@ -367,27 +394,14 @@ jobs:
channel-priority: strict
auto-activate: false
use-mamba: true
- name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.12 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.3 tests with scala-2.13 and python-${{ matrix.python }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.4 tests with scala-2.13 and python-${{ matrix.python }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.5 tests with scala-2.13 and python-${{ matrix.python }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-4.0 tests with scala-2.13 and python-${{ matrix.python }}
if: ${{ matrix.java == 17 }}
- name: run spark-${{ matrix.spark }} tests with scala-${{ matrix.scala }}
run: |
# Python 3.14 uses protobuf 4+, whose C runtime rejects the legacy generated stub.
if python -c 'import sys; sys.exit(sys.version_info < (3, 14))'; then
export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
fi
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-4.0 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-${{ matrix.spark }} -Pspark-scala-${{ matrix.scala }} -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}

# Livy 0.8.0 tested with Spark 3.2
# https://github.com/apache/incubator-livy/blob/v0.8.0-incubating/dev/docker/livy-dev-spark/Dockerfile#L20
Expand Down
6 changes: 5 additions & 1 deletion docs/setup/basics/how_to_build.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ You can directly start Zeppelin by running the following command after successfu

To be noticed, the spark profiles here only affect the unit test (no need to specify `SPARK_HOME`) of spark interpreter.
Zeppelin doesn't require you to build with different spark to make different versions of spark work in Zeppelin.
You can run different versions of Spark in Zeppelin as long as you specify `SPARK_HOME`. Actually Zeppelin supports all the versions of Spark from 3.3 to 3.5.
You can run different versions of Spark in Zeppelin as long as you specify `SPARK_HOME`.
Actually Zeppelin supports all the versions of Spark from 3.3 to 4.2, Spark 4.x requires JDK 17.

To build with a specific Spark version or scala versions, define one or more of the following profiles and options:

Expand All @@ -94,6 +95,9 @@ Set spark major version
Available profiles are

```
-Pspark-4.2
-Pspark-4.1
-Pspark-4.0
-Pspark-3.5
-Pspark-3.4
-Pspark-3.3
Expand Down
22 changes: 21 additions & 1 deletion spark/interpreter/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -456,7 +456,7 @@
<profile>
<id>spark-scala-2.13</id>
<properties>
<spark.scala.version>2.13.16</spark.scala.version>
<spark.scala.version>2.13.18</spark.scala.version>
<spark.scala.binary.version>2.13</spark.scala.binary.version>
</properties>
</profile>
Expand All @@ -472,6 +472,26 @@
</properties>
</profile>

<profile>
<id>spark-4.2</id>
<properties>
<spark.version>4.2.0</spark.version>
<protobuf.version>4.33.5</protobuf.version>
<py4j.version>0.10.9.9</py4j.version>
<libthrift.version>0.16.0</libthrift.version>
</properties>
</profile>

<profile>
<id>spark-4.1</id>
<properties>
<spark.version>4.1.3</spark.version>
<protobuf.version>4.33.0</protobuf.version>
<py4j.version>0.10.9.9</py4j.version>
<libthrift.version>0.16.0</libthrift.version>
</properties>
</profile>

<profile>
<id>spark-4.0</id>
<properties>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@
public class SparkVersion {
private static final Logger LOGGER = LoggerFactory.getLogger(SparkVersion.class);

public static final SparkVersion SPARK_3_3_0 = SparkVersion.fromVersionString("3.3.0");
static final SparkVersion SPARK_3_3_0 = SparkVersion.fromVersionString("3.3.0");

public static final SparkVersion SPARK_3_5_0 = SparkVersion.fromVersionString("3.5.0");
static final SparkVersion SPARK_3_5_0 = SparkVersion.fromVersionString("3.5.0");

public static final SparkVersion SPARK_4_1_0 = SparkVersion.fromVersionString("4.1.0");
static final SparkVersion SPARK_4_3_0 = SparkVersion.fromVersionString("4.3.0");

public static final SparkVersion MIN_SUPPORTED_VERSION = SPARK_3_3_0;
public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_4_1_0;
public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_4_3_0;

private int version;
private int majorVersion;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ void testUnsupportedVersion() {
assertTrue(SparkVersion.fromVersionString("2.3.0").isUnsupportedVersion());
assertTrue(SparkVersion.fromVersionString("0.9.0").isUnsupportedVersion());
assertTrue(SparkVersion.UNSUPPORTED_FUTURE_VERSION.isUnsupportedVersion());
assertTrue(SparkVersion.fromVersionString("4.3.0").isUnsupportedVersion());
assertFalse(SparkVersion.fromVersionString("4.1.0").isUnsupportedVersion());
assertFalse(SparkVersion.fromVersionString("4.2.0").isUnsupportedVersion());
// should not support spark2 version of HDP 2.5
assertTrue(SparkVersion.fromVersionString("2.0.0.2.5.0.0-1245").isUnsupportedVersion());
}
Expand Down
29 changes: 29 additions & 0 deletions testing/env_python_3.14.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
name: python_3
channels:
- conda-forge
- defaults
dependencies:
- pandas>=2.2,<3.0
- numpy>=1.21
- pyarrow>=18
- pycodestyle
- scipy
- grpcio>=1.76,<2
- protobuf
- pandasql
- sqlalchemy
- ipython
- ipykernel
- jupyter_client
- hvplot
- holoviews
- plotnine
- seaborn
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- jinja2
- pip
Loading