diff --git a/.github/workflows/test-verification-loop.yml b/.github/workflows/test-verification-loop.yml new file mode 100644 index 000000000000..504ccfa71c88 --- /dev/null +++ b/.github/workflows/test-verification-loop.yml @@ -0,0 +1,299 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Test Verification Loop + +on: + # run workflow manually to run custom test configurations + workflow_dispatch: + inputs: + + command: + description: "Test ant command" + required: true + default: "-Dtest-unit-sys-prop.ignore.random.failures=true -f java/api.java test" + type: string + + iterations: + description: "Number of Iterations (note: 60min timeout)" + required: true + default: "10" + type: choice + options: + - "1" + - "10" + - "25" + - "100" + + jdk_vendor: + description: "JDK vendor to use" + required: true + default: "temurin" + type: choice + options: + - "corretto" + - "dragonwell" + - "graalvm" + - "graalvm-community" + - "jetbrains" + - "kona" + - "liberica" + - "liberica-nik" + - "microsoft" + - "oracle" + - "oracle-openjdk" + - "redhat" + - "sapmachine" + - "semeru" + - "temurin" + - "zulu" + + jdk_version: + description: "JDK version to use" + required: true + default: "25" + type: string + + os_runner: + description: "OS Runner" + required: true + default: "ubuntu-latest" + type: choice + options: + - "ubuntu-latest" + - "macos-latest" + - "windows-latest" + - "ubuntu-26.04-arm" + - "windows-11-arm" + + stop_on_failure: + description: "Stop on Failure? (note: job will appear green if disabled -> check the steps)" + required: true + default: true + type: boolean + +permissions: + contents: read + +# cancel other workflow run in the same head-base group if it exists +concurrency: + group: dep-checker-${{ github.head_ref || github.run_id }}-${{ github.base_ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + + verification-loop: + name: Loop ${{ inputs.iterations }}x on ${{ inputs.os_runner }}, ${{ inputs.jdk_vendor }} ${{ inputs.jdk_version }} + runs-on: ${{ inputs.os_runner }} + timeout-minutes: 60 + steps: + + - name: Checkout ${{ github.ref }} ( ${{ github.sha }} ) + uses: actions/checkout@v7 + with: + persist-credentials: false + submodules: false + show-progress: false + + - name: Set up ${{ inputs.jdk_vendor }} ${{ inputs.jdk_version }} + uses: actions/setup-java@v6 + with: + java-version: ${{ inputs.jdk_version }} + distribution: ${{ inputs.jdk_vendor }} + problem-matcher: false + + - name: Restoring dependencies from cache + uses: actions/cache/restore@v6 + with: + path: ~/.hgexternalcache + key: ${{ runner.os }}-${{ hashFiles('*/external/binaries-list', '*/*/external/binaries-list') }} + restore-keys: ${{ runner.os }}- + + - name: Build NetBeans + run: ant -quiet -Dcluster.config=full build-nozip + + - name: Setup Xvfb + if: contains(matrix.os, 'ubuntu') && success() + run: | + echo "DISPLAY=:99.0" >> $GITHUB_ENV + Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & + + # branched loop unrolling using yaml, yey! + + - &run-test + name: Run Test + if: ${{ steps.check1.outputs.stop_tests != 'true' && steps.check2.outputs.stop_tests != 'true' && steps.check3.outputs.stop_tests != 'true' }} + continue-on-error: ${{ !inputs.stop_on_failure }} + env: + COMMAND: ${{ inputs.command }} + run: bash -c "ant $COMMAND" + + - name: Stop after 1? + if: ${{ inputs.iterations == '1' }} + id: check1 + run: echo "stop_tests=true" >> "$GITHUB_OUTPUT" + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test +#10 + + - name: Stop after 10? + if: ${{ inputs.iterations == '10' }} + id: check2 + run: echo "stop_tests=true" >> "$GITHUB_OUTPUT" + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test +#20 + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + + - name: Stop after 25? + if: ${{ inputs.iterations == '25' }} + id: check3 + run: echo "stop_tests=true" >> "$GITHUB_OUTPUT" + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test +#30 + + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test +#40 + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test +#50 + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test +#60 + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test +#70 + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test +#80 + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test +#90 + + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test + - *run-test +#100 + + - name: Upload Test XMLs + if: always() + uses: actions/upload-artifact@v7 + with: + name: test-results + path: "./*/*/build/test/*/results/TEST-*.xml" + retention-days: 7 + + - name: Create Test Summary + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # v2.6 + if: always() + with: + paths: "./*/*/build/test/*/results/TEST-*.xml" + show: "all" +