feat: list related commits under every thread

KNOWLEDGE_BASE.md

@@ -256,6 +256,12 @@ Deleting a parent does not automatically delete other people's replies: an autho
 
 Sources: [reply validation](app/Http/Requests/CreateCommitReplyRequest.php), [commit page](app/Http/Controllers/CommitController.php), [reply notifications](app/Actions/CreateReplyNotifications.php), [commit deletion](app/Actions/DeleteCommit.php).
 
+### Related commits
+
+Under a commit, /dev/null shows up to three commits from other threads that say something close to it. It only suggests commits you could find through search: nothing from people you block, who block you or whom you mute, and nothing still pending. When nothing is close, nothing shows.
+
+Sources: [related commits query](app/Queries/RelatedCommitsQuery.php), [related commits rail](resources/js/components/related-commits.tsx).
+
 ### Quoting commits
 
 Quote a visible commit to publish a new top-level commit with your own commentary and an embedded card pointing to the original. Quotes follow normal commit limits, image support, and moderation. The quote composer has a preview and link-card choice, but no poll or scheduling. Quoting is different from replying: the quote appears in logs as its own post, and it is also listed among the original's replies and counted in its reply count. The original author is notified after approval, except for self-quotes.

app/Http/Controllers/Api/RelatedCommitController.php

@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Http\Controllers\Api;
+
+use App\Http\Requests\ListApiRelatedCommitsRequest;
+use App\Http\Resources\CommitResource;
+use App\Models\Commit;
+use App\Models\User;
+use App\Queries\RelatedCommitsQuery;
+use Illuminate\Container\Attributes\CurrentUser;
+use Illuminate\Http\Resources\Json\AnonymousResourceCollection;
+
+final readonly class RelatedCommitController
+{
+    public function index(ListApiRelatedCommitsRequest $request, #[CurrentUser] ?User $user, Commit $commit, RelatedCommitsQuery $related): AnonymousResourceCollection
+    {
+        return CommitResource::collection($related->get($commit, $user));
+    }
+}

app/Http/Controllers/RelatedCommitController.php

@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Http\Controllers;
+
+use App\Http\Requests\ListRelatedCommitsRequest;
+use App\Models\Commit;
+use App\Models\User;
+use App\Queries\RelatedCommitsQuery;
+use Illuminate\Container\Attributes\CurrentUser;
+use Illuminate\Http\JsonResponse;
+
+final readonly class RelatedCommitController
+{
+    public function index(ListRelatedCommitsRequest $request, #[CurrentUser] ?User $user, Commit $commit, RelatedCommitsQuery $related): JsonResponse
+    {
+        return response()->json(['commits' => $related->get($commit, $user)]);
+    }
+}

app/Http/Requests/ListApiRelatedCommitsRequest.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Http\Requests;
+
+use App\Models\Commit;
+use Illuminate\Auth\Access\Response;
+use Illuminate\Foundation\Http\FormRequest;
+
+final class ListApiRelatedCommitsRequest extends FormRequest
+{
+    public function authorize(): Response
+    {
+        $commit = $this->route('commit');
+        assert($commit instanceof Commit);
+
+        return $commit->isVisibleTo($this->user()) ? Response::allow() : Response::denyAsNotFound();
+    }
+
+    /**
+     * @return array{}
+     */
+    public function rules(): array
+    {
+        return [];
+    }
+}

app/Http/Requests/ListRelatedCommitsRequest.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Http\Requests;
+
+use App\Models\Commit;
+use Illuminate\Auth\Access\Response;
+use Illuminate\Foundation\Http\FormRequest;
+
+final class ListRelatedCommitsRequest extends FormRequest
+{
+    public function authorize(): Response
+    {
+        $commit = $this->route('commit');
+        assert($commit instanceof Commit);
+
+        return $commit->isVisibleTo($this->user()) ? Response::allow() : Response::denyAsNotFound();
+    }
+
+    /**
+     * @return array{}
+     */
+    public function rules(): array
+    {
+        return [];
+    }
+}

app/Providers/AppServiceProvider.php

@@ -153,6 +153,11 @@ private function bootRateLimitingDefaults(): void
                 $headers,
             )));
 
+        // Every commit page asks for its related commits, a vector index scan with no outside call...
+        RateLimiter::for('related-commits', fn (Request $request): Limit => Limit::perMinute(60)->by(
+            $request->user() instanceof User ? $this->userId($request) : 'ip:'.$request->ip(),
+        ));
+
         // Exports stream every commit a user ever made...
         RateLimiter::for('exports', fn (Request $request): Limit => Limit::perHour(3)->by($this->userId($request)));
     }

app/Queries/RelatedCommitsQuery.php

@@ -0,0 +1,62 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Queries;
+
+use App\Models\Commit;
+use App\Models\User;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\DB;
+
+final readonly class RelatedCommitsQuery
+{
+    /**
+     * Cosine distance up to which two text-embedding-3-small vectors say about the same thing. Measured on 281 real
+     * posts: pairs under 0.45 are near duplicates, 0.45 to 0.60 share a topic, and past 0.60 the pairs come apart.
+     */
+    public const float MAX_DISTANCE = 0.60;
+
+    public const int LIMIT = 3;
+
+    /**
+     * The top-level commits of other threads closest in meaning to the commit, closest first, loaded as the feeds load
+     * them so the same card renders them. Only commits the viewer could find through search are suggested, and none
+     * further than MAX_DISTANCE, so a commit with nothing close, or without an embedding, gets an empty list.
+     *
+     * The HNSW index hands back a fixed number of candidates before the filters run, so closer replies or hidden
+     * commits could use them all up. An iterative scan keeps reading the index until LIMIT commits pass the filters.
+     * The cutoff is applied to those afterwards, or a commit with nothing close would scan the whole index.
+     *
+     * @return Collection<int, Commit>
+     */
+    public function get(Commit $commit, ?User $viewer): Collection
+    {
+        $nearest = Commit::query()
+            ->select('commits.id')
+            ->selectRaw(
+                'commits.embedding <=> (select source.embedding from commits as source where source.id = ?) as distance',
+                [$commit->id],
+            )
+            ->topLevel()
+            ->whereKeyNot($commit->threadRootId())
+            ->visibleTo($viewer)
+            ->withoutMutedAuthorsFor($viewer)
+            ->orderBy('distance')
+            ->limit(self::LIMIT);
+
+        return DB::transaction(function () use ($nearest, $viewer): Collection {
+            DB::statement('set local hnsw.iterative_scan = strict_order');
+
+            return Commit::query()
+                ->joinSub($nearest, 'nearest', 'nearest.id', '=', 'commits.id')
+                ->where('nearest.distance', '<=', self::MAX_DISTANCE)
+                ->withVisibleReplyCount($viewer)
+                ->withContinuationId($viewer)
+                ->withCardDataFor($viewer)
+                ->orderBy('nearest.distance')
+                ->get()
+                ->toBase();
+        });
+    }
+}

resources/js/components/related-commits.tsx

@@ -0,0 +1,31 @@
+import { CommitCard } from '@/components/commit-card';
+import { useRelatedCommits } from '@/hooks/use-related-commits';
+import { laneOf } from '@/lib/utils';
+
+export function RelatedCommits({ commitId }: { commitId: string }) {
+    const commits = useRelatedCommits(commitId);
+
+    if (commits.length === 0) {
+        return null;
+    }
+
+    return (
+        <section aria-labelledby="related-commits" className="border-t">
+            <div className="flex h-11 items-center border-b px-4 sm:px-6">
+                <h2 id="related-commits" className="font-semibold">
+                    Related commits
+                </h2>
+            </div>
+
+            {commits.map((commit, index) => (
+                <CommitCard
+                    key={commit.id}
+                    commit={commit}
+                    previousLane={
+                        index === 0 ? undefined : laneOf(commits[index - 1])
+                    }
+                />
+            ))}
+        </section>
+    );
+}

resources/js/hooks/use-related-commits.ts

@@ -0,0 +1,46 @@
+import { router } from '@inertiajs/react';
+import { useEffect, useState } from 'react';
+import RelatedCommitController from '@/actions/App/Http/Controllers/RelatedCommitController';
+import { fetchRelatedCommits, hiddenAfterFlash } from '@/lib/related-commits';
+import type { Commit } from '@/types';
+
+export function useRelatedCommits(commitId: string): Commit[] {
+    const [loaded, setLoaded] = useState<{
+        id: string;
+        commits: Commit[];
+        hidden: ReadonlySet<string>;
+    }>({ id: '', commits: [], hidden: new Set() });
+
+    useEffect(() => {
+        const controller = new AbortController();
+
+        fetchRelatedCommits(
+            RelatedCommitController.index(commitId).url,
+            controller.signal,
+        )
+            .then((commits) =>
+                setLoaded({ id: commitId, commits, hidden: new Set() }),
+            )
+            .catch(() => {});
+
+        return () => controller.abort();
+    }, [commitId]);
+
+    useEffect(
+        () =>
+            router.on('flash', (event) =>
+                setLoaded((current) => ({
+                    ...current,
+                    hidden: hiddenAfterFlash(
+                        current.hidden,
+                        event.detail.flash,
+                    ),
+                })),
+            ),
+        [],
+    );
+
+    return loaded.id === commitId
+        ? loaded.commits.filter((commit) => !loaded.hidden.has(commit.id))
+        : [];
+}

resources/js/lib/related-commits.ts

@@ -0,0 +1,54 @@
+import type { Commit, RelatedCommits } from '@/types';
+
+/**
+ * Takes the url rather than the commit id so the node test runner, which cannot resolve Wayfinder's imports, can load it.
+ */
+export function fetchRelatedCommits(
+    url: string,
+    signal: AbortSignal,
+): Promise<Commit[]> {
+    return fetch(url, {
+        headers: {
+            Accept: 'application/json',
+            'X-Requested-With': 'XMLHttpRequest',
+        },
+        signal,
+    })
+        .then((response) =>
+            response.ok
+                ? response.json().then((data: RelatedCommits) => data.commits)
+                : [],
+        )
+        .catch((error: unknown) => {
+            if (signal.aborted) {
+                throw error;
+            }
+
+            return [];
+        });
+}
+
+/**
+ * The rail keeps the order it was fetched in, so a deleted commit's card is hidden rather than dropped, and a restore
+ * mounts it afresh as the feeds do.
+ */
+export function hiddenAfterFlash(
+    hidden: ReadonlySet<string>,
+    {
+        deleted,
+        restored,
+    }: { deleted?: { id: string }; restored?: { id: string } },
+): ReadonlySet<string> {
+    if (deleted) {
+        return new Set(hidden).add(deleted.id);
+    }
+
+    if (restored && hidden.has(restored.id)) {
+        const shown = new Set(hidden);
+        shown.delete(restored.id);
+
+        return shown;
+    }
+
+    return hidden;
+}

resources/js/pages/commit/show.tsx

@@ -4,6 +4,7 @@ import LogController from '@/actions/App/Http/Controllers/LogController';
 import { CommitCard, CommitTombstone } from '@/components/commit-card';
 import { CopyLinkButton } from '@/components/copy-link-button';
 import { GraphCanvas } from '@/components/graph-canvas';
+import { RelatedCommits } from '@/components/related-commits';
 import { useContinuation } from '@/hooks/use-continuation';
 import AppLayout from '@/layouts/app-layout';
 import { isDeleted, laneOf, shortHash } from '@/lib/utils';
@@ -164,6 +165,8 @@ export default function Show({
                     </div>
                 )}
             </GraphCanvas>
+
+            {!isDeleted(commit) && <RelatedCommits commitId={commit.id} />}
         </>
     );
 }

resources/js/types/commit.ts

@@ -132,6 +132,10 @@ export type ThreadContinuation = {
     next_id: string | null;
 };
 
+export type RelatedCommits = {
+    commits: Commit[];
+};
+
 export type CursorPaginated<T> = {
     data: T[];
     next_cursor: string | null;

routes/api.php

@@ -40,6 +40,7 @@
 use App\Http\Controllers\Api\OrgPingController;
 use App\Http\Controllers\Api\OrgReadController;
 use App\Http\Controllers\Api\OrgTypingController;
+use App\Http\Controllers\Api\RelatedCommitController;
 use App\Http\Controllers\Api\RemoteCandidateController;
 use App\Http\Controllers\Api\StarredCommitController;
 use App\Http\Controllers\Api\SuggestedUserController;
@@ -89,6 +90,8 @@
             ->name('api-user-following.index');
         Route::get('commits/{commit}', [ApiCommitController::class, 'show'])
             ->name('api-commit.show');
+        Route::get('commits/{commit}/related', [RelatedCommitController::class, 'index'])
+            ->name('api-related-commit.index');
     });
 
     // The Mac app's latest release, which its update check compares with, unsigned so it sends nothing about anyone...

routes/web.php

@@ -85,6 +85,7 @@
 use App\Http\Controllers\PrivacyController;
 use App\Http\Controllers\ProductMetricsController;
 use App\Http\Controllers\PushSubscriptionController;
+use App\Http\Controllers\RelatedCommitController;
 use App\Http\Controllers\RemoteCandidateController;
 use App\Http\Controllers\ReportedCommitController;
 use App\Http\Controllers\SafetyController;
@@ -254,6 +255,9 @@
     ->name('commit-reply.index');
 Route::get('commits/{commit}/continuation', [CommitContinuationController::class, 'index'])
     ->name('commit-continuation.index');
+Route::get('commits/{commit}/related', [RelatedCommitController::class, 'index'])
+    ->middleware('throttle:related-commits')
+    ->name('related-commit.index');
 
 // Search...
 Route::get('search', [SearchController::class, 'index'])