diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index a9b631b..8fef970 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -67,3 +67,9 @@ jobs:
     - name: Test with pytest
       run: |
         spin test -- --doctest-modules
+    - name: Smoke test benchmarks
+      # Runs a single benchmark in asv dry-run mode so that CI catches
+      # future incompatibilities between spin bench, spin and asv.
+      run: |
+        asv machine --yes
+        spin bench --quick
diff --git a/.spin/cmds.py b/.spin/cmds.py
index 91973eb..4b742cf 100644
--- a/.spin/cmds.py
+++ b/.spin/cmds.py
@@ -9,12 +9,16 @@
 @click.command()
 @click.option('--tests', '-t', multiple=True,
               help='Select benchmarks by name or regular expression.')
+@click.option('--quick', '-q', is_flag=True, default=False,
+              help="Run each benchmark only once (timings won't be accurate).")
 @meson.build_dir_option
 @click.pass_context
-def bench(ctx, tests, build_dir):
+def bench(ctx, tests, quick, build_dir):
     """Run benchmarks against the local build without saving results."""
     args = [sys.executable, '-m', 'asv', 'run', '--python=same',
             '--dry-run', '--show-stderr']
+    if quick:
+        args.append('--quick')
     for test in tests:
         args.extend(['--bench', test])
     ctx.invoke(meson.build, build_dir=build_dir)