diff --git a/.github/workflows/pr_title.yaml b/.github/workflows/pr_title.yaml
index 29ae0b8a7..3ed55c0e7 100644
--- a/.github/workflows/pr_title.yaml
+++ b/.github/workflows/pr_title.yaml
@@ -3,6 +3,10 @@ name: PR Title Check
 on:
   pull_request_target:
     types: [opened, edited, synchronize, reopened]
+
+permissions:
+  pull-requests: write
+
 jobs:
   commitlint:
     name: PR title / description conforms to semantic-release
@@ -22,42 +26,60 @@ jobs:
               "body-leading-blank": [0, "always"]
             }
           }' > .commitlintrc.js
-      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
+      - id: commitlint
+        continue-on-error: true
+        run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
         env:
           COMMIT_MSG: >
             ${{ github.event.pull_request.title }}
 
             ${{ github.event.pull_request.body }}
-      - if: failure()
-        uses: actions/github-script@v9
+      - uses: actions/github-script@v9
         with:
           script: |
-            const message = `**ACTION NEEDED**
+            // Hidden marker used to find this job's comment regardless of wording.
+            const marker = "<!-- pr-title-check -->";
+            const message = `${marker}
+            **ACTION NEEDED**
+
+            Substrait follows the [Conventional Commits
+            specification](https://www.conventionalcommits.org/en/v1.0.0/) for
+            release automation.
 
-              Substrait follows the [Conventional Commits
-              specification](https://www.conventionalcommits.org/en/v1.0.0/) for
-              release automation.
+            The PR title and description are used as the merge commit message.
+            Please update your PR title and description to match the specification.
+            `;
+            const passed = "${{ steps.commitlint.outcome }}" === "success";
 
-              The PR title and description are used as the merge commit message.\
-              Please update your PR title and description to match the specification.
-              `
-            // Get list of current comments
+            // Find an existing comment from this job, if any.
             const comments = await github.paginate(github.rest.issues.listComments, {
               owner: context.repo.owner,
               repo: context.repo.repo,
-              issue_number: context.issue.number
+              issue_number: context.issue.number,
             });
-            // Check if this job already commented
-            for (const comment of comments) {
-              if (comment.body === message) {
-                return // Already commented
+            const existing = comments.find((c) => c.body.includes(marker));
+
+            if (passed) {
+              // Clean up the comment now that the check passes.
+              if (existing) {
+                await github.rest.issues.deleteComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  comment_id: existing.id,
+                });
               }
+              return;
             }
-            // Post the comment about Conventional Commits
-            github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-              body: message
-            })
-            core.setFailed(message)
+
+            // Check failed: post the comment if it isn't already there, then fail the job.
+            if (!existing) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: message,
+              });
+            }
+            core.setFailed(
+              "PR title and description do not follow the Conventional Commits specification."
+            );