Add code coverage measurement for unit and integration tests (3.x)

.github/workflows/tests@v1.yml

@@ -129,8 +129,30 @@ jobs:
           key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
 
       - name: Run unit tests
+        env:
+          COVERAGE: "true"
         run: make test-unit
 
+      # Flattened into one file per module, so the artifact layout does not depend on how many
+      # modules happened to produce data.
+      - name: Collect coverage execution data
+        if: ${{ !cancelled() }}
+        run: |
+          shopt -s nullglob
+          mkdir -p coverage-exec
+          for exec_file in */target/jacoco.exec; do
+            cp "$exec_file" "coverage-exec/${exec_file%%/*}.exec"
+          done
+          ls -l coverage-exec
+
+      - name: Upload coverage execution data
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ !cancelled() }}
+        with:
+          name: coverage-exec-unit
+          path: coverage-exec/
+          if-no-files-found: warn
+
       - name: Copy test results
         if: always()
         run: |
@@ -237,8 +259,29 @@ jobs:
         id: run-integration-tests
         env:
           CASSANDRA_VERSION_RESOLVED: ${{ steps.cassandra-version.outputs.value }}
+          COVERAGE: "true"
         run: make test-integration-cassandra
 
+      # Flattened into one file per module, so the artifact layout does not depend on how many
+      # modules happened to produce data.
+      - name: Collect coverage execution data
+        if: ${{ !cancelled() }}
+        run: |
+          shopt -s nullglob
+          mkdir -p coverage-exec
+          for exec_file in */target/jacoco.exec; do
+            cp "$exec_file" "coverage-exec/${exec_file%%/*}.exec"
+          done
+          ls -l coverage-exec
+
+      - name: Upload coverage execution data
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ !cancelled() }}
+        with:
+          name: coverage-exec-cassandra-${{ matrix.cassandra-version }}
+          path: coverage-exec/
+          if-no-files-found: warn
+
       - name: Copy test results
         if: steps.run-integration-tests.outcome == 'failure'
         run: |
@@ -336,8 +379,29 @@ jobs:
         id: run-integration-tests
         env:
           SCYLLA_VERSION_RESOLVED: ${{ steps.scylla-version.outputs.value }}
+          COVERAGE: "true"
         run: make test-integration-scylla
 
+      # Flattened into one file per module, so the artifact layout does not depend on how many
+      # modules happened to produce data.
+      - name: Collect coverage execution data
+        if: ${{ !cancelled() }}
+        run: |
+          shopt -s nullglob
+          mkdir -p coverage-exec
+          for exec_file in */target/jacoco.exec; do
+            cp "$exec_file" "coverage-exec/${exec_file%%/*}.exec"
+          done
+          ls -l coverage-exec
+
+      - name: Upload coverage execution data
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ !cancelled() }}
+        with:
+          name: coverage-exec-scylla-${{ matrix.scylla-version }}
+          path: coverage-exec/
+          if-no-files-found: warn
+
       - name: Copy test results
         if: steps.run-integration-tests.outcome == 'failure'
         run: |
@@ -370,3 +434,67 @@ jobs:
           detailed_summary: true
           updateComment: false
           skip_annotations: true
+
+  coverage-report:
+    name: Coverage report
+    runs-on: ubuntu-latest
+    needs: [unit-tests, cassandra-integration-tests, scylla-integration-tests]
+    # Runs even when a test lane failed: partial coverage data is still worth reporting, and
+    # continue-on-error keeps a flaky integration test from turning this metric into a second
+    # failure on the pull request.
+    if: ${{ !cancelled() }}
+    continue-on-error: true
+    timeout-minutes: 20
+
+    env:
+      # Overrides the Makefile default of `mvn -B -X -ntp`; this job has nothing to debug and -X
+      # buys a log measured in hundreds of megabytes.
+      MVNCMD: mvn -B -ntp
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+        with:
+          java-version: 8
+          distribution: 'temurin'
+
+      - name: Restore maven repository cache
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-8-maven-${{ hashFiles('**/pom.xml') }}
+
+      - name: Download coverage execution data
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          pattern: coverage-exec-*
+          path: coverage-exec
+
+      # jacoco:report-aggregate picks up every *.exec in a module's target directory, so each lane's
+      # data only has to land there under a name of its own.
+      - name: Place execution data next to the classes it was recorded against
+        run: |
+          shopt -s nullglob
+          for lane in coverage-exec/*/; do
+            lane_name="$(basename "$lane")"
+            for exec_file in "$lane"*.exec; do
+              module="$(basename "$exec_file" .exec)"
+              mkdir -p "$module/target"
+              cp "$exec_file" "$module/target/jacoco-${lane_name#coverage-exec-}.exec"
+            done
+          done
+          find . -name 'jacoco-*.exec' -printf '%p\t%s bytes\n'
+
+      - name: Aggregate coverage
+        run: make coverage-report
+
+      - name: Upload coverage report
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ !cancelled() }}
+        with:
+          name: coverage-report
+          path: driver-coverage-report/target/site/jacoco-aggregate
+          if-no-files-found: error

.gitignore

@@ -1,6 +1,4 @@
 target/
-cobertura-history/
-testing/
 .settings
 .classpath
 .project

CONTRIBUTING.md

@@ -125,6 +125,47 @@ sudo ifconfig lo0 alias 127.0.1.2 up
 ...
 ```
 
+### Code coverage
+
+Coverage is measured with [JaCoCo](https://www.jacoco.org/jacoco/) and is off by default: the agent
+slows every forked test JVM down, so it is opt-in through the `coverage` Maven profile. Pass
+`COVERAGE=true` to any of the `test-*` Make targets to enable it, then aggregate:
+
+```
+make test-unit COVERAGE=true
+make coverage-report
+```
+
+`make coverage-report` reads whatever execution data is already on disk, so several lanes can be
+combined into one number:
+
+```
+make test-unit COVERAGE=true
+mv driver-core/target/jacoco.exec driver-core/target/jacoco-unit.exec
+make test-integration-scylla COVERAGE=true
+make coverage-report
+```
+
+The report lands in `driver-coverage-report/target/site/jacoco-aggregate` (HTML, XML and CSV), and
+`make clean-coverage` removes it along with the execution data. In CI, the unit and integration jobs
+upload their execution data and the "Coverage report" job aggregates it; the percentage shows up in
+that job's summary and the HTML report is attached as an artifact.
+
+Two things to know about the scope of the report:
+
+- It covers the modules that `driver-coverage-report` depends on: `driver-core`,
+  `driver-mapping` and `driver-extras`. `driver-examples`, `driver-tests/**` and `driver-dist`
+  are out of scope.
+- Only Surefire is instrumented. That covers the unit tests and, because 3.x runs its integration
+  tests as TestNG `short`-group tests, the integration tests as well. The Failsafe-run tests in
+  `driver-tests/**` (OSGi, shading) are left out: the OSGi ones load the driver inside their own
+  Pax Exam container, and those modules are outside the report's scope in any case.
+
+JaCoCo matches execution data to classes by checksum, so the data has to come from the same build of
+the classes the report is rendered against. If a report shows code you know was exercised as
+uncovered, look for `Execution data for class ... does not match` in the Maven log; the usual cause
+is stale execution data from before a recompile, which `make clean-coverage` clears.
+
 ## Updating GitHub Actions workflows
 
 GitHub Actions workflows in this repository pin all third-party actions to specific commit SHAs

Makefile

@@ -22,6 +22,18 @@ SONATYPE_TOKEN_PASSWORD ?=
 RELEASE_SKIP_TESTS ?= false
 RELEASE_TARGET_TAG ?=
 
+# Set COVERAGE=true on any of the test-* targets to attach the JaCoCo agent to the forked test
+# JVMs; `make coverage-report` then aggregates whatever execution data is on disk.
+COVERAGE ?= false
+ifeq ($(filter true 1,$(COVERAGE)),)
+	MVN_COVERAGE :=
+	COVERAGE_PREREQ :=
+else
+	MVN_COVERAGE := -Pcoverage
+	COVERAGE_PREREQ := .clean-coverage-data
+endif
+COVERAGE_REPORT_DIR := driver-coverage-report/target/site/jacoco-aggregate
+
 ifeq (${CCM_CONFIG_DIR},)
 	CCM_CONFIG_DIR = ~/.ccm
 endif
@@ -67,6 +79,16 @@ export PATH := $(MAKEFILE_PATH)/bin:$(PATH)
   		$(MAKE) install-cassandra-ccm
 	fi
 
+# JaCoCo appends to its execution data files by default, which is what lets a single lane
+# accumulate coverage across several forks. The flip side is that data from an earlier run
+# survives a recompile, and classes that changed in between are then reported as uncovered
+# because their checksum no longer matches. Truncating before a run is the fix.
+#
+# Only jacoco.exec, the file the agent is about to write, is removed. Data that was renamed
+# out of the way to keep one lane's results while another runs is left alone.
+.clean-coverage-data:
+	@find . -name 'jacoco.exec' -delete
+
 .prepare-environment-update-aio-max-nr:
 	@if (( $$(< /proc/sys/fs/aio-max-nr) < 2097152 )); then
 		echo 2097152 | sudo tee /proc/sys/fs/aio-max-nr >/dev/null
@@ -240,28 +262,51 @@ check:
 fix:
 	$(MVNCMD) fmt:format
 
-test-unit:
-	$(MVNCMD) test -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
+test-unit: $(COVERAGE_PREREQ)
+	$(MVNCMD) test $(MVN_COVERAGE) -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
 
-test-integration-scylla: .prepare-scylla-ccm resolve-scylla-version .prepare-environment-update-aio-max-nr
+test-integration-scylla: .prepare-scylla-ccm resolve-scylla-version .prepare-environment-update-aio-max-nr $(COVERAGE_PREREQ)
 	@if [[ -z "$${SCYLLA_VERSION_RESOLVED}" ]]; then
 		SCYLLA_VERSION_RESOLVED=`cat '${SCYLLA_VERSION_FILE}'`
 	fi
 	if [[ -z "$${SCYLLA_VERSION_RESOLVED}" ]]; then
 		echo "ScyllaDB version ${SCYLLA_VERSION} was not resolved"
 		exit 1
 	fi
-	mvn -B verify -Pshort -Dscylla.version=$${SCYLLA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
+	mvn -B verify -Pshort $(MVN_COVERAGE) -Dscylla.version=$${SCYLLA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
 
-test-integration-cassandra: .prepare-scylla-ccm resolve-cassandra-version
+test-integration-cassandra: .prepare-scylla-ccm resolve-cassandra-version $(COVERAGE_PREREQ)
 	@if [[ -z "$${CASSANDRA_VERSION_RESOLVED}" ]]; then
 		CASSANDRA_VERSION_RESOLVED=`cat '${CASSANDRA_VERSION_FILE}'`
 	fi
 	if [[ -z "$${CASSANDRA_VERSION_RESOLVED}" ]]; then
 		echo "Cassandra version ${CASSANDRA_VERSION} was not resolved"
 		exit 1
 	fi
-	mvn -B verify -Pshort -Dcassandra.version=$${CASSANDRA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
+	mvn -B verify -Pshort $(MVN_COVERAGE) -Dcassandra.version=$${CASSANDRA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
+
+# Aggregates the execution data left behind by any COVERAGE=true test run into a single report.
+# Tests are skipped here on purpose: this only reads what is already on disk, so the same target
+# works for one local lane and for execution data collected from several CI jobs.
+coverage-report:
+	@set -o pipefail
+	if [[ -z "$$(find . -name 'jacoco*.exec' -print -quit)" ]]; then
+		echo 'No JaCoCo execution data found.'
+		echo "Run the tests with COVERAGE=true first, e.g. 'make test-unit COVERAGE=true'."
+		exit 1
+	fi
+	rm -rf '${COVERAGE_REPORT_DIR}'
+	$(MVNCMD) -Pcoverage -DskipTests verify -pl driver-coverage-report -am -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
+	if [[ ! -f '${COVERAGE_REPORT_DIR}/jacoco.xml' ]]; then
+		echo 'Maven produced no report at ${COVERAGE_REPORT_DIR}/jacoco.xml.'
+		exit 1
+	fi
+	echo 'HTML report: ${COVERAGE_REPORT_DIR}/index.html'
+	python3 -c 'import sys, xml.etree.ElementTree as ET; r = ET.parse(sys.argv[1]).getroot(); c = next(x for x in r.findall("counter") if x.get("type") == "LINE"); missed, covered = int(c.get("missed")), int(c.get("covered")); total = missed + covered; print("Line coverage: {}/{} ({:.2f}%)".format(covered, total, 100.0 * covered / total if total else 0.0)); sys.exit("No lines are recorded as covered: the execution data is either missing or does not match these classes. Look for a checksum mismatch warning in the Maven log.") if covered == 0 else None' '${COVERAGE_REPORT_DIR}/jacoco.xml' | tee -a "$${GITHUB_STEP_SUMMARY:-/dev/null}"
+
+clean-coverage:
+	@find . -name 'jacoco*.exec' -delete
+	rm -rf '${COVERAGE_REPORT_DIR}'
 
 check-no-compile-warnings:
 	@$(MAKE) compile-all | grep WARNING >/tmp/all-compile-warnings.log || true