fix: moderate image descriptions and poll options before auto-approving

app/Actions/UpdateCommitAiDecision.php

@@ -7,6 +7,7 @@
 use App\Enums\CommitStatus;
 use App\Models\Commit;
 use App\Models\CommitImage;
+use App\Models\PollOption;
 use App\Services\OpenAI;
 use Illuminate\Support\Facades\Storage;
 
@@ -35,11 +36,13 @@ public function handle(Commit $commit): void
             fn (CommitImage $image): string => 'data:image/webp;base64,'.base64_encode((string) Storage::get($image->path)),
         )->all());
 
-        $scores = $this->openAI->moderate($commit->body, $images);
+        $text = $this->text($commit);
+
+        $scores = $this->openAI->moderate($text, $images);
 
         $categories = array_keys($scores);
 
-        if ($commit->body !== '' && $this->openAI->isHarmful($commit->body)) {
+        if ($text !== '' && $this->openAI->isHarmful($text)) {
             $categories[] = 'harmful-text';
         }
 
@@ -56,4 +59,17 @@ public function handle(Commit $commit): void
             $this->updateCommitStatuses->handle([$commit->id], CommitStatus::Approved, onlyPending: true);
         }
     }
+
+    /**
+     * All the text that goes public with the commit: the body, the image descriptions, and the poll options. A clean
+     * commit is approved, so nothing the author wrote may be left unchecked.
+     */
+    private function text(Commit $commit): string
+    {
+        return collect([$commit->body])
+            ->merge($commit->images->map(fn (CommitImage $image): ?string => $image->description))
+            ->merge($commit->poll?->options->map(fn (PollOption $option): string => $option->label) ?? [])
+            ->filter(fn (?string $part): bool => $part !== null && $part !== '')
+            ->implode("\n\n");
+    }
 }

tests/Unit/Jobs/ModerateCommitTest.php

@@ -10,6 +10,7 @@
 use App\Models\CommitImage;
 use App\Models\CommitReport;
 use App\Models\Notification;
+use App\Models\Poll;
 use App\Models\User;
 use GuzzleHttp\Promise\PromiseInterface;
 use Illuminate\Http\Client\Request;
@@ -241,6 +242,78 @@ function reviewResponses(array $responses = []): Closure
     ]);
 })->with(['sexual', 'sexual/minors']);
 
+it('checks the image descriptions and poll options along with the body', function (): void {
+    Storage::fake();
+    Storage::put('commit-images/first.webp', 'first');
+
+    Http::fake([
+        'api.openai.com/v1/moderations' => Http::response(moderationResponse()),
+        'api.openai.com/v1/responses' => reviewResponses(),
+    ]);
+
+    $commit = Commit::factory()->pending()->create(['body' => 'Which editor?']);
+    CommitImage::factory()->for($commit)->create(['path' => 'commit-images/first.webp', 'position' => 0, 'description' => 'A screenshot']);
+    Poll::factory()->for($commit)->unopened()->withOptions('Vim', 'Emacs')->create();
+
+    dispatch_sync(new ModerateCommit($commit));
+
+    Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/moderations')
+        && $request['input'][0] === ['type' => 'text', 'text' => "Which editor?\n\nA screenshot\n\nVim\n\nEmacs"]);
+
+    Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/responses')
+        && $request['text']['format']['schema']['required'] === ['harmful']
+        && $request['input'][1]['content'] === [['type' => 'input_text', 'text' => "Which editor?\n\nA screenshot\n\nVim\n\nEmacs"]]);
+});
+
+it('keeps a commit pending when only its image descriptions or poll options are flagged', function (string $where): void {
+    Storage::fake();
+    Storage::put('commit-images/first.webp', 'first');
+
+    Http::fake([
+        'api.openai.com/v1/moderations' => fn (Request $request): PromiseInterface => Http::response(
+            str_contains((string) json_encode($request['input']), 'buy my course') ? moderationResponse(['harassment']) : moderationResponse(),
+        ),
+        'api.openai.com/v1/responses' => reviewResponses(),
+    ]);
+
+    $commit = Commit::factory()->pending()->create(['body' => 'Which editor?']);
+    CommitImage::factory()->for($commit)->create([
+        'path' => 'commit-images/first.webp',
+        'position' => 0,
+        'description' => $where === 'image description' ? 'buy my course' : null,
+    ]);
+    Poll::factory()->for($commit)->unopened()->withOptions(...$where === 'poll option' ? ['buy my course', 'Emacs'] : ['Vim', 'Emacs'])->create();
+
+    dispatch_sync(new ModerateCommit($commit));
+
+    expect($commit->refresh())
+        ->ai_decision->toBe(CommitStatus::Rejected)
+        ->ai_categories->toBe(['harassment'])
+        ->status->toBe(CommitStatus::Pending);
+})->with(['image description', 'poll option']);
+
+it('checks the image description of a commit without a body as its text', function (): void {
+    Storage::fake();
+    Storage::put('commit-images/first.webp', 'first');
+
+    Http::fake([
+        'api.openai.com/v1/moderations' => Http::response(moderationResponse()),
+        'api.openai.com/v1/responses' => reviewResponses(),
+    ]);
+
+    $commit = Commit::factory()->pending()->create(['body' => '']);
+    CommitImage::factory()->for($commit)->create(['path' => 'commit-images/first.webp', 'position' => 0, 'description' => 'A screenshot']);
+
+    dispatch_sync(new ModerateCommit($commit));
+
+    Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/moderations')
+        && $request['input'][0] === ['type' => 'text', 'text' => 'A screenshot']);
+
+    Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/responses')
+        && $request['text']['format']['schema']['required'] === ['harmful']
+        && $request['input'][1]['content'] === [['type' => 'input_text', 'text' => 'A screenshot']]);
+});
+
 it('checks only the images of a commit without a body', function (): void {
     Storage::fake();
     Storage::put('commit-images/first.webp', 'first');