diff --git a/app/Actions/UpdateCommitAiDecision.php b/app/Actions/UpdateCommitAiDecision.php index 57c98e791..115ab0832 100644 --- a/app/Actions/UpdateCommitAiDecision.php +++ b/app/Actions/UpdateCommitAiDecision.php @@ -7,6 +7,7 @@ use App\Enums\CommitStatus; use App\Models\Commit; use App\Models\CommitImage; +use App\Models\PollOption; use App\Services\OpenAI; use Illuminate\Support\Facades\Storage; @@ -35,11 +36,13 @@ public function handle(Commit $commit): void fn (CommitImage $image): string => 'data:image/webp;base64,'.base64_encode((string) Storage::get($image->path)), )->all()); - $scores = $this->openAI->moderate($commit->body, $images); + $text = $this->text($commit); + + $scores = $this->openAI->moderate($text, $images); $categories = array_keys($scores); - if ($commit->body !== '' && $this->openAI->isHarmful($commit->body)) { + if ($text !== '' && $this->openAI->isHarmful($text)) { $categories[] = 'harmful-text'; } @@ -56,4 +59,17 @@ public function handle(Commit $commit): void $this->updateCommitStatuses->handle([$commit->id], CommitStatus::Approved, onlyPending: true); } } + + /** + * All the text that goes public with the commit: the body, the image descriptions, and the poll options. A clean + * commit is approved, so nothing the author wrote may be left unchecked. + */ + private function text(Commit $commit): string + { + return collect([$commit->body]) + ->merge($commit->images->map(fn (CommitImage $image): ?string => $image->description)) + ->merge($commit->poll?->options->map(fn (PollOption $option): string => $option->label) ?? []) + ->filter(fn (?string $part): bool => $part !== null && $part !== '') + ->implode("\n\n"); + } } diff --git a/tests/Unit/Jobs/ModerateCommitTest.php b/tests/Unit/Jobs/ModerateCommitTest.php index b8049202f..1c83cf16c 100644 --- a/tests/Unit/Jobs/ModerateCommitTest.php +++ b/tests/Unit/Jobs/ModerateCommitTest.php @@ -10,6 +10,7 @@ use App\Models\CommitImage; use App\Models\CommitReport; use App\Models\Notification; +use App\Models\Poll; use App\Models\User; use GuzzleHttp\Promise\PromiseInterface; use Illuminate\Http\Client\Request; @@ -241,6 +242,78 @@ function reviewResponses(array $responses = []): Closure ]); })->with(['sexual', 'sexual/minors']); +it('checks the image descriptions and poll options along with the body', function (): void { + Storage::fake(); + Storage::put('commit-images/first.webp', 'first'); + + Http::fake([ + 'api.openai.com/v1/moderations' => Http::response(moderationResponse()), + 'api.openai.com/v1/responses' => reviewResponses(), + ]); + + $commit = Commit::factory()->pending()->create(['body' => 'Which editor?']); + CommitImage::factory()->for($commit)->create(['path' => 'commit-images/first.webp', 'position' => 0, 'description' => 'A screenshot']); + Poll::factory()->for($commit)->unopened()->withOptions('Vim', 'Emacs')->create(); + + dispatch_sync(new ModerateCommit($commit)); + + Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/moderations') + && $request['input'][0] === ['type' => 'text', 'text' => "Which editor?\n\nA screenshot\n\nVim\n\nEmacs"]); + + Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/responses') + && $request['text']['format']['schema']['required'] === ['harmful'] + && $request['input'][1]['content'] === [['type' => 'input_text', 'text' => "Which editor?\n\nA screenshot\n\nVim\n\nEmacs"]]); +}); + +it('keeps a commit pending when only its image descriptions or poll options are flagged', function (string $where): void { + Storage::fake(); + Storage::put('commit-images/first.webp', 'first'); + + Http::fake([ + 'api.openai.com/v1/moderations' => fn (Request $request): PromiseInterface => Http::response( + str_contains((string) json_encode($request['input']), 'buy my course') ? moderationResponse(['harassment']) : moderationResponse(), + ), + 'api.openai.com/v1/responses' => reviewResponses(), + ]); + + $commit = Commit::factory()->pending()->create(['body' => 'Which editor?']); + CommitImage::factory()->for($commit)->create([ + 'path' => 'commit-images/first.webp', + 'position' => 0, + 'description' => $where === 'image description' ? 'buy my course' : null, + ]); + Poll::factory()->for($commit)->unopened()->withOptions(...$where === 'poll option' ? ['buy my course', 'Emacs'] : ['Vim', 'Emacs'])->create(); + + dispatch_sync(new ModerateCommit($commit)); + + expect($commit->refresh()) + ->ai_decision->toBe(CommitStatus::Rejected) + ->ai_categories->toBe(['harassment']) + ->status->toBe(CommitStatus::Pending); +})->with(['image description', 'poll option']); + +it('checks the image description of a commit without a body as its text', function (): void { + Storage::fake(); + Storage::put('commit-images/first.webp', 'first'); + + Http::fake([ + 'api.openai.com/v1/moderations' => Http::response(moderationResponse()), + 'api.openai.com/v1/responses' => reviewResponses(), + ]); + + $commit = Commit::factory()->pending()->create(['body' => '']); + CommitImage::factory()->for($commit)->create(['path' => 'commit-images/first.webp', 'position' => 0, 'description' => 'A screenshot']); + + dispatch_sync(new ModerateCommit($commit)); + + Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/moderations') + && $request['input'][0] === ['type' => 'text', 'text' => 'A screenshot']); + + Http::assertSent(fn (Request $request): bool => str_ends_with($request->url(), '/responses') + && $request['text']['format']['schema']['required'] === ['harmful'] + && $request['input'][1]['content'] === [['type' => 'input_text', 'text' => 'A screenshot']]); +}); + it('checks only the images of a commit without a body', function (): void { Storage::fake(); Storage::put('commit-images/first.webp', 'first');