diff --git a/.github/workflows/tests@v1.yml b/.github/workflows/tests@v1.yml index 273bcd63115..936720539b8 100644 --- a/.github/workflows/tests@v1.yml +++ b/.github/workflows/tests@v1.yml @@ -129,8 +129,30 @@ jobs: key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }} - name: Run unit tests + env: + COVERAGE: "true" run: make test-unit + # Flattened into one file per module, so the artifact layout does not depend on how many + # modules happened to produce data. + - name: Collect coverage execution data + if: ${{ !cancelled() }} + run: | + shopt -s nullglob + mkdir -p coverage-exec + for exec_file in */target/jacoco.exec; do + cp "$exec_file" "coverage-exec/${exec_file%%/*}.exec" + done + ls -l coverage-exec + + - name: Upload coverage execution data + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ !cancelled() }} + with: + name: coverage-exec-unit + path: coverage-exec/ + if-no-files-found: warn + - name: Copy test results if: always() run: | @@ -237,8 +259,29 @@ jobs: id: run-integration-tests env: CASSANDRA_VERSION_RESOLVED: ${{ steps.cassandra-version.outputs.value }} + COVERAGE: "true" run: make test-integration-cassandra + # Flattened into one file per module, so the artifact layout does not depend on how many + # modules happened to produce data. + - name: Collect coverage execution data + if: ${{ !cancelled() }} + run: | + shopt -s nullglob + mkdir -p coverage-exec + for exec_file in */target/jacoco.exec; do + cp "$exec_file" "coverage-exec/${exec_file%%/*}.exec" + done + ls -l coverage-exec + + - name: Upload coverage execution data + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ !cancelled() }} + with: + name: coverage-exec-cassandra-${{ matrix.cassandra-version }} + path: coverage-exec/ + if-no-files-found: warn + - name: Copy test results if: steps.run-integration-tests.outcome == 'failure' run: | @@ -336,8 +379,29 @@ jobs: id: run-integration-tests env: SCYLLA_VERSION_RESOLVED: ${{ steps.scylla-version.outputs.value }} + COVERAGE: "true" run: make test-integration-scylla + # Flattened into one file per module, so the artifact layout does not depend on how many + # modules happened to produce data. + - name: Collect coverage execution data + if: ${{ !cancelled() }} + run: | + shopt -s nullglob + mkdir -p coverage-exec + for exec_file in */target/jacoco.exec; do + cp "$exec_file" "coverage-exec/${exec_file%%/*}.exec" + done + ls -l coverage-exec + + - name: Upload coverage execution data + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ !cancelled() }} + with: + name: coverage-exec-scylla-${{ matrix.scylla-version }} + path: coverage-exec/ + if-no-files-found: warn + - name: Copy test results if: steps.run-integration-tests.outcome == 'failure' run: | @@ -370,3 +434,67 @@ jobs: detailed_summary: true updateComment: false skip_annotations: true + + coverage-report: + name: Coverage report + runs-on: ubuntu-latest + needs: [unit-tests, cassandra-integration-tests, scylla-integration-tests] + # Runs even when a test lane failed: partial coverage data is still worth reporting, and + # continue-on-error keeps a flaky integration test from turning this metric into a second + # failure on the pull request. + if: ${{ !cancelled() }} + continue-on-error: true + timeout-minutes: 20 + + env: + # Overrides the Makefile default of `mvn -B -X -ntp`; this job has nothing to debug and -X + # buys a log measured in hundreds of megabytes. + MVNCMD: mvn -B -ntp + + steps: + - name: Checkout source + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + + - name: Set up JDK 8 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: 8 + distribution: 'temurin' + + - name: Restore maven repository cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-8-maven-${{ hashFiles('**/pom.xml') }} + + - name: Download coverage execution data + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: coverage-exec-* + path: coverage-exec + + # jacoco:report-aggregate picks up every *.exec in a module's target directory, so each lane's + # data only has to land there under a name of its own. + - name: Place execution data next to the classes it was recorded against + run: | + shopt -s nullglob + for lane in coverage-exec/*/; do + lane_name="$(basename "$lane")" + for exec_file in "$lane"*.exec; do + module="$(basename "$exec_file" .exec)" + mkdir -p "$module/target" + cp "$exec_file" "$module/target/jacoco-${lane_name#coverage-exec-}.exec" + done + done + find . -name 'jacoco-*.exec' -printf '%p\t%s bytes\n' + + - name: Aggregate coverage + run: make coverage-report + + - name: Upload coverage report + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ !cancelled() }} + with: + name: coverage-report + path: driver-coverage-report/target/site/jacoco-aggregate + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 9458be86d97..d9e2683b75d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ target/ -cobertura-history/ -testing/ .settings .classpath .project diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4b7611a6d25..949c53ef371 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -125,6 +125,47 @@ sudo ifconfig lo0 alias 127.0.1.2 up ... ``` +### Code coverage + +Coverage is measured with [JaCoCo](https://www.jacoco.org/jacoco/) and is off by default: the agent +slows every forked test JVM down, so it is opt-in through the `coverage` Maven profile. Pass +`COVERAGE=true` to any of the `test-*` Make targets to enable it, then aggregate: + +``` +make test-unit COVERAGE=true +make coverage-report +``` + +`make coverage-report` reads whatever execution data is already on disk, so several lanes can be +combined into one number: + +``` +make test-unit COVERAGE=true +mv driver-core/target/jacoco.exec driver-core/target/jacoco-unit.exec +make test-integration-scylla COVERAGE=true +make coverage-report +``` + +The report lands in `driver-coverage-report/target/site/jacoco-aggregate` (HTML, XML and CSV), and +`make clean-coverage` removes it along with the execution data. In CI, the unit and integration jobs +upload their execution data and the "Coverage report" job aggregates it; the percentage shows up in +that job's summary and the HTML report is attached as an artifact. + +Two things to know about the scope of the report: + +- It covers the modules that `driver-coverage-report` depends on: `driver-core`, + `driver-mapping` and `driver-extras`. `driver-examples`, `driver-tests/**` and `driver-dist` + are out of scope. +- Only Surefire is instrumented. That covers the unit tests and, because 3.x runs its integration + tests as TestNG `short`-group tests, the integration tests as well. The Failsafe-run tests in + `driver-tests/**` (OSGi, shading) are left out: the OSGi ones load the driver inside their own + Pax Exam container, and those modules are outside the report's scope in any case. + +JaCoCo matches execution data to classes by checksum, so the data has to come from the same build of +the classes the report is rendered against. If a report shows code you know was exercised as +uncovered, look for `Execution data for class ... does not match` in the Maven log; the usual cause +is stale execution data from before a recompile, which `make clean-coverage` clears. + ## Updating GitHub Actions workflows GitHub Actions workflows in this repository pin all third-party actions to specific commit SHAs diff --git a/Makefile b/Makefile index 76430900850..fb204296b5b 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,18 @@ SONATYPE_TOKEN_PASSWORD ?= RELEASE_SKIP_TESTS ?= false RELEASE_TARGET_TAG ?= +# Set COVERAGE=true on any of the test-* targets to attach the JaCoCo agent to the forked test +# JVMs; `make coverage-report` then aggregates whatever execution data is on disk. +COVERAGE ?= false +ifeq ($(filter true 1,$(COVERAGE)),) + MVN_COVERAGE := + COVERAGE_PREREQ := +else + MVN_COVERAGE := -Pcoverage + COVERAGE_PREREQ := .clean-coverage-data +endif +COVERAGE_REPORT_DIR := driver-coverage-report/target/site/jacoco-aggregate + ifeq (${CCM_CONFIG_DIR},) CCM_CONFIG_DIR = ~/.ccm endif @@ -67,6 +79,16 @@ export PATH := $(MAKEFILE_PATH)/bin:$(PATH) $(MAKE) install-cassandra-ccm fi +# JaCoCo appends to its execution data files by default, which is what lets a single lane +# accumulate coverage across several forks. The flip side is that data from an earlier run +# survives a recompile, and classes that changed in between are then reported as uncovered +# because their checksum no longer matches. Truncating before a run is the fix. +# +# Only jacoco.exec, the file the agent is about to write, is removed. Data that was renamed +# out of the way to keep one lane's results while another runs is left alone. +.clean-coverage-data: + @find . -name 'jacoco.exec' -delete + .prepare-environment-update-aio-max-nr: @if (( $$(< /proc/sys/fs/aio-max-nr) < 2097152 )); then echo 2097152 | sudo tee /proc/sys/fs/aio-max-nr >/dev/null @@ -240,10 +262,10 @@ check: fix: $(MVNCMD) fmt:format -test-unit: - $(MVNCMD) test -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true +test-unit: $(COVERAGE_PREREQ) + $(MVNCMD) test $(MVN_COVERAGE) -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true -test-integration-scylla: .prepare-scylla-ccm resolve-scylla-version .prepare-environment-update-aio-max-nr +test-integration-scylla: .prepare-scylla-ccm resolve-scylla-version .prepare-environment-update-aio-max-nr $(COVERAGE_PREREQ) @if [[ -z "$${SCYLLA_VERSION_RESOLVED}" ]]; then SCYLLA_VERSION_RESOLVED=`cat '${SCYLLA_VERSION_FILE}'` fi @@ -251,9 +273,9 @@ test-integration-scylla: .prepare-scylla-ccm resolve-scylla-version .prepare-env echo "ScyllaDB version ${SCYLLA_VERSION} was not resolved" exit 1 fi - mvn -B verify -Pshort -Dscylla.version=$${SCYLLA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true + mvn -B verify -Pshort $(MVN_COVERAGE) -Dscylla.version=$${SCYLLA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true -test-integration-cassandra: .prepare-scylla-ccm resolve-cassandra-version +test-integration-cassandra: .prepare-scylla-ccm resolve-cassandra-version $(COVERAGE_PREREQ) @if [[ -z "$${CASSANDRA_VERSION_RESOLVED}" ]]; then CASSANDRA_VERSION_RESOLVED=`cat '${CASSANDRA_VERSION_FILE}'` fi @@ -261,7 +283,30 @@ test-integration-cassandra: .prepare-scylla-ccm resolve-cassandra-version echo "Cassandra version ${CASSANDRA_VERSION} was not resolved" exit 1 fi - mvn -B verify -Pshort -Dcassandra.version=$${CASSANDRA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true + mvn -B verify -Pshort $(MVN_COVERAGE) -Dcassandra.version=$${CASSANDRA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true + +# Aggregates the execution data left behind by any COVERAGE=true test run into a single report. +# Tests are skipped here on purpose: this only reads what is already on disk, so the same target +# works for one local lane and for execution data collected from several CI jobs. +coverage-report: + @set -o pipefail + if [[ -z "$$(find . -name 'jacoco*.exec' -print -quit)" ]]; then + echo 'No JaCoCo execution data found.' + echo "Run the tests with COVERAGE=true first, e.g. 'make test-unit COVERAGE=true'." + exit 1 + fi + rm -rf '${COVERAGE_REPORT_DIR}' + $(MVNCMD) -Pcoverage -DskipTests verify -pl driver-coverage-report -am -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true + if [[ ! -f '${COVERAGE_REPORT_DIR}/jacoco.xml' ]]; then + echo 'Maven produced no report at ${COVERAGE_REPORT_DIR}/jacoco.xml.' + exit 1 + fi + echo 'HTML report: ${COVERAGE_REPORT_DIR}/index.html' + python3 -c 'import sys, xml.etree.ElementTree as ET; r = ET.parse(sys.argv[1]).getroot(); c = next(x for x in r.findall("counter") if x.get("type") == "LINE"); missed, covered = int(c.get("missed")), int(c.get("covered")); total = missed + covered; print("Line coverage: {}/{} ({:.2f}%)".format(covered, total, 100.0 * covered / total if total else 0.0)); sys.exit("No lines are recorded as covered: the execution data is either missing or does not match these classes. Look for a checksum mismatch warning in the Maven log.") if covered == 0 else None' '${COVERAGE_REPORT_DIR}/jacoco.xml' | tee -a "$${GITHUB_STEP_SUMMARY:-/dev/null}" + +clean-coverage: + @find . -name 'jacoco*.exec' -delete + rm -rf '${COVERAGE_REPORT_DIR}' check-no-compile-warnings: @$(MAKE) compile-all | grep WARNING >/tmp/all-compile-warnings.log || true diff --git a/driver-coverage-report/pom.xml b/driver-coverage-report/pom.xml new file mode 100644 index 00000000000..7e301889e02 --- /dev/null +++ b/driver-coverage-report/pom.xml @@ -0,0 +1,147 @@ + + + + 4.0.0 + + + com.scylladb + scylla-driver-parent + 3.11.5.19-SNAPSHOT + + + scylla-driver-coverage-report + pom + Java Driver for Scylla and Apache Cassandra - Coverage report + Aggregates the JaCoCo execution data produced by the other modules into a single + coverage report. Produces no released artifact. + + + + + + com.scylladb + scylla-driver-core + + + + com.scylladb + scylla-driver-mapping + + + + com.scylladb + scylla-driver-extras + + + + + + + + + + + + org.codehaus.mojo + clirr-maven-plugin + + true + + + + + maven-source-plugin + + true + + + + + maven-javadoc-plugin + + true + + + + + maven-gpg-plugin + + true + + + + + maven-install-plugin + + true + + + + + maven-deploy-plugin + + true + + + + + + + + + + + + coverage + + + + org.jacoco + jacoco-maven-plugin + + + report-aggregate + verify + + report-aggregate + + + Java Driver for Scylla and Apache Cassandra 3.x + + HTML + XML + CSV + + ${project.build.directory}/site/jacoco-aggregate + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index ea0d22cae71..16e6ea20f5f 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,7 @@ driver-examples driver-tests driver-dist + driver-coverage-report @@ -93,6 +94,12 @@ updated. --> 1.5.9 3.5.4 + 0.8.14 + + 127.0.1. unit @@ -734,13 +741,19 @@ + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + maven-surefire-plugin ${surefire.version} ${test.groups} false - -Djdk.attach.allowAttachSelf=true + -Djdk.attach.allowAttachSelf=true @{jacoco.argline} alphabetical ${cassandra.version} @@ -869,7 +882,7 @@ true central - scylla-driver-tests-parent,scylla-driver-tests-osgi,scylla-driver-tests-osgi-common,scylla-driver-tests-osgi-shaded,scylla-driver-tests-shading,scylla-driver-tests-shading-shaded,scylla-driver-tests-shading-unshaded,scylla-driver-tests-osgi-unshaded,scylla-driver-tests-stress,scylla-driver-dist,scylla-driver-examples + scylla-driver-coverage-report,scylla-driver-tests-parent,scylla-driver-tests-osgi,scylla-driver-tests-osgi-common,scylla-driver-tests-osgi-shaded,scylla-driver-tests-shading,scylla-driver-tests-shading-shaded,scylla-driver-tests-shading-unshaded,scylla-driver-tests-osgi-unshaded,scylla-driver-tests-stress,scylla-driver-dist,scylla-driver-examples ${release.autopublish} validated @@ -941,6 +954,36 @@ + + + coverage + + + + org.jacoco + jacoco-maven-plugin + + + prepare-agent + + prepare-agent + + + + jacoco.argline + + + + + + + +