diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index efeb04ce1e1..1985c81d6aa 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -332,8 +332,35 @@ jobs: strategy: fail-fast: false matrix: - python: [ 3.9 ] - java: [ 11, 17 ] + include: + - java: 11 + python: '3.9' + spark: '3.3' + scala: '2.12' + - java: 17 + python: '3.9' + spark: '3.4' + scala: '2.12' + - java: 11 + python: '3.9' + spark: '3.5' + scala: '2.12' + - java: 17 + python: '3.9' + spark: '3.5' + scala: '2.13' + - java: 17 + python: '3.9' + spark: '4.0' + scala: '2.13' + - java: 17 + python: '3.14' + spark: '4.1' + scala: '2.13' + - java: 17 + python: '3.14' + spark: '4.2' + scala: '2.13' steps: - name: Checkout uses: actions/checkout@v5 @@ -356,7 +383,7 @@ jobs: restore-keys: | ${{ runner.os }}-zeppelin- - name: install environment - run: ./mvnw install -DskipTests -pl spark-submit,spark/scala-2.12,spark/scala-2.13 -am ${MAVEN_ARGS} + run: ./mvnw install -DskipTests -pl spark-submit,spark/scala-${{ matrix.scala }} -am ${MAVEN_ARGS} - name: Setup conda environment with python ${{ matrix.python }} uses: conda-incubator/setup-miniconda@v4 with: @@ -367,27 +394,14 @@ jobs: channel-priority: strict auto-activate: false use-mamba: true - - name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }} - run: | - rm -rf spark/interpreter/metastore_db - ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.12 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} - - name: run spark-3.3 tests with scala-2.13 and python-${{ matrix.python }} - run: | - rm -rf spark/interpreter/metastore_db - ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} - - name: run spark-3.4 tests with scala-2.13 and python-${{ matrix.python }} - run: | - rm -rf spark/interpreter/metastore_db - ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} - - name: run spark-3.5 tests with scala-2.13 and python-${{ matrix.python }} - run: | - rm -rf spark/interpreter/metastore_db - ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} - - name: run spark-4.0 tests with scala-2.13 and python-${{ matrix.python }} - if: ${{ matrix.java == 17 }} + - name: run spark-${{ matrix.spark }} tests with scala-${{ matrix.scala }} run: | + # Python 3.14 uses protobuf 4+, whose C runtime rejects the legacy generated stub. + if python -c 'import sys; sys.exit(sys.version_info < (3, 14))'; then + export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + fi rm -rf spark/interpreter/metastore_db - ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-4.0 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} + ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-${{ matrix.spark }} -Pspark-scala-${{ matrix.scala }} -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} # Livy 0.8.0 tested with Spark 3.2 # https://github.com/apache/incubator-livy/blob/v0.8.0-incubating/dev/docker/livy-dev-spark/Dockerfile#L20 diff --git a/docs/setup/basics/how_to_build.md b/docs/setup/basics/how_to_build.md index 176d7a9146c..97746488a46 100644 --- a/docs/setup/basics/how_to_build.md +++ b/docs/setup/basics/how_to_build.md @@ -83,7 +83,8 @@ You can directly start Zeppelin by running the following command after successfu To be noticed, the spark profiles here only affect the unit test (no need to specify `SPARK_HOME`) of spark interpreter. Zeppelin doesn't require you to build with different spark to make different versions of spark work in Zeppelin. -You can run different versions of Spark in Zeppelin as long as you specify `SPARK_HOME`. Actually Zeppelin supports all the versions of Spark from 3.3 to 3.5. +You can run different versions of Spark in Zeppelin as long as you specify `SPARK_HOME`. +Actually Zeppelin supports all the versions of Spark from 3.3 to 4.2, Spark 4.x requires JDK 17. To build with a specific Spark version or scala versions, define one or more of the following profiles and options: @@ -94,6 +95,9 @@ Set spark major version Available profiles are ``` +-Pspark-4.2 +-Pspark-4.1 +-Pspark-4.0 -Pspark-3.5 -Pspark-3.4 -Pspark-3.3 diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml index a51f045ae8e..4dced787777 100644 --- a/spark/interpreter/pom.xml +++ b/spark/interpreter/pom.xml @@ -456,7 +456,7 @@ spark-scala-2.13 - 2.13.16 + 2.13.18 2.13 @@ -472,6 +472,26 @@ + + spark-4.2 + + 4.2.0 + 4.33.5 + 0.10.9.9 + 0.16.0 + + + + + spark-4.1 + + 4.1.3 + 4.33.0 + 0.10.9.9 + 0.16.0 + + + spark-4.0 diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java index 2db447985ae..3051524a1c8 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java @@ -25,14 +25,14 @@ public class SparkVersion { private static final Logger LOGGER = LoggerFactory.getLogger(SparkVersion.class); - public static final SparkVersion SPARK_3_3_0 = SparkVersion.fromVersionString("3.3.0"); + static final SparkVersion SPARK_3_3_0 = SparkVersion.fromVersionString("3.3.0"); - public static final SparkVersion SPARK_3_5_0 = SparkVersion.fromVersionString("3.5.0"); + static final SparkVersion SPARK_3_5_0 = SparkVersion.fromVersionString("3.5.0"); - public static final SparkVersion SPARK_4_1_0 = SparkVersion.fromVersionString("4.1.0"); + static final SparkVersion SPARK_4_3_0 = SparkVersion.fromVersionString("4.3.0"); public static final SparkVersion MIN_SUPPORTED_VERSION = SPARK_3_3_0; - public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_4_1_0; + public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_4_3_0; private int version; private int majorVersion; diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java index 06aa392e4ca..acd04f58b6d 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java @@ -35,6 +35,9 @@ void testUnsupportedVersion() { assertTrue(SparkVersion.fromVersionString("2.3.0").isUnsupportedVersion()); assertTrue(SparkVersion.fromVersionString("0.9.0").isUnsupportedVersion()); assertTrue(SparkVersion.UNSUPPORTED_FUTURE_VERSION.isUnsupportedVersion()); + assertTrue(SparkVersion.fromVersionString("4.3.0").isUnsupportedVersion()); + assertFalse(SparkVersion.fromVersionString("4.1.0").isUnsupportedVersion()); + assertFalse(SparkVersion.fromVersionString("4.2.0").isUnsupportedVersion()); // should not support spark2 version of HDP 2.5 assertTrue(SparkVersion.fromVersionString("2.0.0.2.5.0.0-1245").isUnsupportedVersion()); } diff --git a/testing/env_python_3.14.yml b/testing/env_python_3.14.yml new file mode 100644 index 00000000000..18a1d60031b --- /dev/null +++ b/testing/env_python_3.14.yml @@ -0,0 +1,29 @@ +name: python_3 +channels: + - conda-forge + - defaults +dependencies: + - pandas>=2.2,<3.0 + - numpy>=1.21 + - pyarrow>=18 + - pycodestyle + - scipy + - grpcio>=1.76,<2 + - protobuf + - pandasql + - sqlalchemy + - ipython + - ipykernel + - jupyter_client + - hvplot + - holoviews + - plotnine + - seaborn + - intake + - intake-parquet + - intake-xarray + - altair + - vega_datasets + - plotly + - jinja2 + - pip