From 75f16f0ed10d740b94f825d6cb749d3febfeed5e Mon Sep 17 00:00:00 2001 From: Arjan Kapteijn Date: Thu, 22 Oct 2026 15:30:23 +0200 Subject: [PATCH 1/3] fix: moderate image descriptions and poll options before auto-approving The AI check only saw the body and the image pixels, so text placed in poll options or image descriptions went public on auto-approval unchecked. Co-Authored-By: Claude Opus 5.5 --- app/Actions/UpdateCommitAiDecision.php | 20 +++++++++- tests/Unit/Jobs/ModerateCommitTest.php | 51 ++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/app/Actions/UpdateCommitAiDecision.php b/app/Actions/UpdateCommitAiDecision.php index d3be2a46a..6ec3d242c 100644 --- a/app/Actions/UpdateCommitAiDecision.php +++ b/app/Actions/UpdateCommitAiDecision.php @@ -7,6 +7,7 @@ use App\Enums\CommitStatus; use App\Models\Commit; use App\Models\CommitImage; +use App\Models\PollOption; use App\Services\OpenAI; use Illuminate\Support\Facades\Storage; @@ -35,11 +36,13 @@ public function handle(Commit $commit): void fn (CommitImage $image): string => 'data:image/webp;base64,'.base64_encode((string) Storage::get($image->path)), )->all()); - $scores = $this->openAI->moderate($commit->body, $images); + $text = $this->text($commit); + + $scores = $this->openAI->moderate($text, $images); $categories = array_keys($scores); - if ($commit->body !== '' && $this->openAI->isHarmful($commit->body)) { + if ($text !== '' && $this->openAI->isHarmful($text)) { $categories[] = 'harmful-text'; } @@ -60,4 +63,17 @@ public function handle(Commit $commit): void $this->updateCommitStatuses->handle([$commit->id], CommitStatus::Approved, onlyPending: true); } } + + /** + * All the text that goes public with the commit: the body, the image descriptions, and the poll options. A clean + * commit is approved, so nothing the author wrote may be left unchecked. + */ + private function text(Commit $commit): string + { + return collect([$commit->body]) + ->merge($commit->images->map(fn (CommitImage $image): ?string => $image->description)) + ->merge($commit->poll?->options->map(fn (PollOption $option): string => $option->label) ?? []) + ->filter(fn (?string $part): bool => $part !== null && $part !== '') + ->implode("\n\n"); + } } diff --git a/tests/Unit/Jobs/ModerateCommitTest.php b/tests/Unit/Jobs/ModerateCommitTest.php index 5f6c08bcb..66660296a 100644 --- a/tests/Unit/Jobs/ModerateCommitTest.php +++ b/tests/Unit/Jobs/ModerateCommitTest.php @@ -10,6 +10,7 @@ use App\Models\CommitImage; use App\Models\CommitReport; use App\Models\Notification; +use App\Models\Poll; use App\Models\User; use GuzzleHttp\Promise\PromiseInterface; use Illuminate\Http\Client\Request; @@ -240,6 +241,56 @@ function reviewResponses(array $responses = []): Closure ]); }); +it('checks the image descriptions and poll options along with the body', function (): void { + Storage::fake(); + Storage::put('commit-images/first.webp', 'first'); + + Http::fake([ + 'api.openai.com/v1/moderations' => Http::response(moderationResponse()), + 'api.openai.com/v1/responses' => reviewResponses(), + ]); + + $commit = Commit::factory()->pending()->create(['body' => 'Which editor?']); + CommitImage::factory()->for($commit)->create(['path' => 'commit-images/first.webp', 'position' => 0, 'description' => 'A screenshot']); + Poll::factory()->for($commit)->unopened()->withOptions('Vim', 'Emacs')->create(); + + dispatch_sync(new ModerateCommit($commit)); + + Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/moderations') + && $request['input'][0] === ['type' => 'text', 'text' => "Which editor?\n\nA screenshot\n\nVim\n\nEmacs"]); + + Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/responses') + && $request['text']['format']['name'] === 'harmful' + && $request['input'][1]['content'] === [['type' => 'input_text', 'text' => "Which editor?\n\nA screenshot\n\nVim\n\nEmacs"]]); +}); + +it('keeps a commit pending when only its image descriptions or poll options are flagged', function (string $where): void { + Storage::fake(); + Storage::put('commit-images/first.webp', 'first'); + + Http::fake([ + 'api.openai.com/v1/moderations' => fn (Request $request): PromiseInterface => Http::response( + str_contains((string) json_encode($request['input']), 'buy my course') ? moderationResponse(['harassment']) : moderationResponse(), + ), + 'api.openai.com/v1/responses' => reviewResponses(), + ]); + + $commit = Commit::factory()->pending()->create(['body' => 'Which editor?']); + CommitImage::factory()->for($commit)->create([ + 'path' => 'commit-images/first.webp', + 'position' => 0, + 'description' => $where === 'image description' ? 'buy my course' : null, + ]); + Poll::factory()->for($commit)->unopened()->withOptions(...$where === 'poll option' ? ['buy my course', 'Emacs'] : ['Vim', 'Emacs'])->create(); + + dispatch_sync(new ModerateCommit($commit)); + + expect($commit->refresh()) + ->ai_decision->toBe(CommitStatus::Rejected) + ->ai_categories->toBe(['harassment']) + ->status->toBe(CommitStatus::Pending); +})->with(['image description', 'poll option']); + it('checks only the images of a commit without a body', function (): void { Storage::fake(); Storage::put('commit-images/first.webp', 'first'); From e0e1bfc2d46e0f027da60a0cb1e5fa7e16a17264 Mon Sep 17 00:00:00 2001 From: Arjan Kapteijn Date: Thu, 29 Oct 2026 13:36:38 +0100 Subject: [PATCH 2/3] test: match the harmful check's request shape after the SDK switch Co-Authored-By: Claude Opus 5.5 --- tests/Unit/Jobs/ModerateCommitTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Unit/Jobs/ModerateCommitTest.php b/tests/Unit/Jobs/ModerateCommitTest.php index c6a8069be..d03451bb7 100644 --- a/tests/Unit/Jobs/ModerateCommitTest.php +++ b/tests/Unit/Jobs/ModerateCommitTest.php @@ -261,7 +261,7 @@ function reviewResponses(array $responses = []): Closure && $request['input'][0] === ['type' => 'text', 'text' => "Which editor?\n\nA screenshot\n\nVim\n\nEmacs"]); Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/responses') - && $request['text']['format']['name'] === 'harmful' + && $request['text']['format']['schema']['required'] === ['harmful'] && $request['input'][1]['content'] === [['type' => 'input_text', 'text' => "Which editor?\n\nA screenshot\n\nVim\n\nEmacs"]]); }); From a688f36970ae4374528e3f27d28ef91a9de65b0b Mon Sep 17 00:00:00 2001 From: Arjan Kapteijn Date: Thu, 29 Oct 2026 14:17:15 +0100 Subject: [PATCH 3/3] test: cover moderating an image description on a commit without a body Co-Authored-By: Claude Opus 5.5 --- tests/Unit/Jobs/ModerateCommitTest.php | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/Unit/Jobs/ModerateCommitTest.php b/tests/Unit/Jobs/ModerateCommitTest.php index d03451bb7..1c83cf16c 100644 --- a/tests/Unit/Jobs/ModerateCommitTest.php +++ b/tests/Unit/Jobs/ModerateCommitTest.php @@ -292,6 +292,28 @@ function reviewResponses(array $responses = []): Closure ->status->toBe(CommitStatus::Pending); })->with(['image description', 'poll option']); +it('checks the image description of a commit without a body as its text', function (): void { + Storage::fake(); + Storage::put('commit-images/first.webp', 'first'); + + Http::fake([ + 'api.openai.com/v1/moderations' => Http::response(moderationResponse()), + 'api.openai.com/v1/responses' => reviewResponses(), + ]); + + $commit = Commit::factory()->pending()->create(['body' => '']); + CommitImage::factory()->for($commit)->create(['path' => 'commit-images/first.webp', 'position' => 0, 'description' => 'A screenshot']); + + dispatch_sync(new ModerateCommit($commit)); + + Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/moderations') + && $request['input'][0] === ['type' => 'text', 'text' => 'A screenshot']); + + Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/responses') + && $request['text']['format']['schema']['required'] === ['harmful'] + && $request['input'][1]['content'] === [['type' => 'input_text', 'text' => 'A screenshot']]); +}); + it('checks only the images of a commit without a body', function (): void { Storage::fake(); Storage::put('commit-images/first.webp', 'first');