diff --git a/app/Models/Notification.php b/app/Models/Notification.php index da4f9dfe6..7934de900 100644 --- a/app/Models/Notification.php +++ b/app/Models/Notification.php @@ -34,6 +34,7 @@ * @property-read User $actor * @property-read Commit|null $commit * @property-read string $group_id + * @property-read CarbonInterface $group_started_at * @property-read int $event_count * @property-read int $actors_count * @property-read int $unread_count @@ -44,6 +45,7 @@ #[Hidden([ 'key', 'group_rank', + 'group_started_at', 'actor_position', 'second_actor_id', 'third_actor_id', @@ -73,6 +75,7 @@ public function casts(): array 'level' => 'integer', 'read_at' => 'datetime', 'created_at' => 'datetime', + 'group_started_at' => 'datetime', 'updated_at' => 'datetime', 'actors_count' => 'integer', 'unread_count' => 'integer', diff --git a/app/Queries/NotificationGroupsQuery.php b/app/Queries/NotificationGroupsQuery.php index 927458fb8..75c677708 100644 --- a/app/Queries/NotificationGroupsQuery.php +++ b/app/Queries/NotificationGroupsQuery.php @@ -14,6 +14,8 @@ use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Database\Query\Builder as QueryBuilder; +use Illuminate\Pagination\Cursor; +use Illuminate\Pagination\CursorPaginator as Paginator; use Illuminate\Support\Facades\DB; final readonly class NotificationGroupsQuery @@ -22,28 +24,22 @@ private const int STACKED_ACTORS = 3; + private const int WINDOW_ROWS_PER_GROUP = 32; + + private const int WINDOW_ATTEMPTS = 4; + + private const int MIN_WINDOW_GROWTH = 2; + + private const int MAX_WINDOW_GROWTH = 8; + /** @return CursorPaginator */ public function paginate(User $user, string $type = 'all', ?CarbonImmutable $through = null): CursorPaginator { - $order = 'PARTITION BY group_id ORDER BY CASE WHEN actor_position = 1 THEN 0 ELSE 1 END, created_at DESC, id DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'; - $events = $this->events($user, $type)->select('notifications.*') - ->selectRaw("NTH_VALUE(actor_id, 2) OVER ({$order}) AS second_actor_id") - ->selectRaw("NTH_VALUE(actor_id, 3) OVER ({$order}) AS third_actor_id"); - - $query = Notification::query()->fromSub($events, 'notifications') - ->where('group_rank', 1) - ->with([ - 'actor' => fn (BelongsTo $query): BelongsTo => $query - ->withExists(['followers as is_followed' => fn (Builder $query): Builder => $query->whereKey($user->id)]), - 'commit', - ]) - ->latest()->latest('id'); $perPage = $through instanceof CarbonImmutable ? max(self::PER_PAGE, $this->groupsSince($user, $through, $type)->where('group_rank', 1)->count()) : self::PER_PAGE; - $groups = $query->cursorPaginate($perPage)->appends(request()->except('through')); + $groups = $this->groups($user, $type, $perPage, Paginator::resolveCurrentCursor())->appends(request()->except('through')); $this->loadStackedActors($groups->items()); @@ -109,6 +105,160 @@ public function events(User $user, string $type = 'all'): NotificationBuilder return $this->group($this->visible($user), $type); } + /** + * Grouping the whole history costs time in proportion to its size for a page of groups, so the page is grouped + * from a window of recent notifications, widened until it provably holds the same groups as the whole history would. + * + * @return CursorPaginator + */ + private function groups(User $user, string $type, int $perPage, ?Cursor $cursor): CursorPaginator + { + $until = $this->windowEnd($user, $cursor); + $rows = $perPage * self::WINDOW_ROWS_PER_GROUP; + + $sparse = $type !== 'all' && $this->below($user, $cursor)->whereIn('type', $this->kinds($type))->offset($rows - 1)->doesntExist(); + + for ($round = 0; ! $sparse && $round < self::WINDOW_ATTEMPTS; $round++) { + $since = $this->windowStart($user, $cursor, $rows); + + if (! $since instanceof CarbonInterface) { + break; + } + + $groups = $this->page($this->window($user, $since, $until), $user, $type, $perPage); + + if ($this->windowSuffices($groups, $since)) { + return $groups; + } + + $rows = $this->widened($rows, $perPage, count($groups->items())); + } + + return $this->page($this->window($user, null, $until), $user, $type, $perPage); + } + + /** @return CursorPaginator */ + private function page(NotificationBuilder $visible, User $user, string $type, int $perPage): CursorPaginator + { + $order = 'PARTITION BY group_id ORDER BY CASE WHEN actor_position = 1 THEN 0 ELSE 1 END, created_at DESC, id DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'; + $events = $this->group($visible, $type)->select('notifications.*') + ->selectRaw("NTH_VALUE(actor_id, 2) OVER ({$order}) AS second_actor_id") + ->selectRaw("NTH_VALUE(actor_id, 3) OVER ({$order}) AS third_actor_id"); + + return Notification::query()->fromSub($events, 'notifications') + ->where('group_rank', 1) + ->with([ + 'actor' => fn (BelongsTo $query): BelongsTo => $query + ->withExists(['followers as is_followed' => fn (Builder $query): Builder => $query->whereKey($user->id)]), + 'commit', + ]) + ->latest()->latest('id') + ->cursorPaginate($perPage); + } + + /** + * The rows for the next window, scaled by how many groups this one found so it would hold twice the page and the + * one group past it, but never less than double nor more than eight times as many, so one sparse window cannot + * blow the next up to the whole history. + */ + private function widened(int $rows, int $perPage, int $found): int + { + return (int) ceil($rows * min(self::MAX_WINDOW_GROWTH, max(self::MIN_WINDOW_GROWTH, 2 * ($perPage + 1) / max(1, $found)))); + } + + /** + * Whether fewer than twice the window's rows lie below the cursor, so the window would take at least half of them + * and grouping them all costs little more. + */ + private function windowCoversHalf(NotificationBuilder $below, int $rows): bool + { + return $below->clone()->offset(2 * $rows - 1)->doesntExist(); + } + + /** + * Where a window of the given rows below the cursor begins: the oldest notification the inbox shows from the time + * of the row that many back. Null when the window would not be worth it, or no notification it shows lies there, + * and the page is grouped from the whole history instead. + */ + private function windowStart(User $user, ?Cursor $cursor, int $rows): ?CarbonInterface + { + $below = $this->below($user, $cursor)->latest()->latest('id'); + + if ($this->windowCoversHalf($below, $rows)) { + return null; + } + + return $this->below($user, $cursor)->relevant() + ->where('created_at', '>=', $below->offset($rows - 1)->value('created_at')) + ->oldest()->oldest('id')->first()?->created_at; + } + + private function below(User $user, ?Cursor $cursor): NotificationBuilder + { + return Notification::query()->where('recipient_id', $user->id) + ->when($cursor, fn (NotificationBuilder $query, Cursor $cursor): NotificationBuilder => $query + ->whereRowValues(['created_at', 'id'], '<', [$cursor->parameter('created_at'), $cursor->parameter('id')])); + } + + /** + * A page past a cursor needs the notifications above it only to tell the groups reaching back from above the cursor + * apart from those ending below it. Such a group skips the hour past the first notification at or above the cursor + * only by continuing a session over a gap longer than an hour, which it cannot, as that notification came between. + * So the window ends an hour after that notification, and a page before a cursor keeps all that lies above it. + */ + private function windowEnd(User $user, ?Cursor $cursor): ?CarbonInterface + { + if (! $cursor instanceof Cursor || ! $cursor->pointsToNextItems()) { + return null; + } + + return $this->visible($user) + ->whereRowValues(['created_at', 'id'], '>=', [$cursor->parameter('created_at'), $cursor->parameter('id')]) + ->oldest()->oldest('id')->first()?->created_at->addHour(); + } + + /** + * Grouping the window splits groups like the whole history does, apart from cutting off the oldest group of each + * topic (see groupsSince) and the groups going on past the window's end. Either bound may be left off: a window + * without a start takes the whole history below its end. Stars never split, so a star group spans a + * commit's whole history: taking along every star of each commit starred within the window keeps those groups + * whole, and the extra stars cannot change how anything else groups, as a notification only looks back past its + * own topic's previous one. + */ + private function window(User $user, ?CarbonInterface $since, ?CarbonInterface $until): NotificationBuilder + { + if (! $since instanceof CarbonInterface && ! $until instanceof CarbonInterface) { + return $this->visible($user); + } + + $within = fn (NotificationBuilder $query): NotificationBuilder => $query + ->when($since, fn (NotificationBuilder $query, CarbonInterface $since): NotificationBuilder => $query->where('created_at', '>=', $since)) + ->when($until, fn (NotificationBuilder $query, CarbonInterface $until): NotificationBuilder => $query->where('created_at', '<=', $until)); + $starredWithin = $within($this->visible($user)->select('commit_id')->where('type', 'starred')); + + return $within($this->visible($user))->unionAll( + $this->visible($user)->where('type', 'starred')->whereNot($within)->whereIn('commit_id', $starredWithin) + ); + } + + /** + * A page from the window matches one from the whole history when the window holds more groups than the page and no + * group on it was cut off. A session continues past its topic's previous notification only when that one is at most + * an hour older, so a group that starts over an hour after the window began has nothing before the window: the row + * just before its first member would have been another member. Groups starting within that first hour may go on + * below, and the window widens until none is on the page. + * + * @param CursorPaginator $groups + */ + private function windowSuffices(CursorPaginator $groups, CarbonInterface $since): bool + { + $horizon = $since->toImmutable()->addHour(); + + return $groups->hasMorePages() && array_all($groups->items(), fn (Notification $group): bool => $group->type === NotificationType::Starred + || $group->group_started_at->isAfter($horizon)); + } + /** * Grouping only the notifications from some point onwards puts every one of them in the same group as grouping all * of them does, apart from the oldest group of each topic missing what came before that point. So it finds just as @@ -124,23 +274,29 @@ private function visible(User $user): NotificationBuilder return Notification::query()->where('recipient_id', $user->id)->relevant()->select('notifications.*'); } + /** @return list */ + private function kinds(string $type): array + { + return array_merge(...array_map(fn (string $category): array => match ($category) { + 'replies' => ['replied'], + 'quotes' => ['quoted'], + 'stars' => ['starred'], + 'mentions' => ['mentioned'], + 'follows' => ['followed'], + 'moderation' => ['approved', 'rejected'], + default => [], + }, explode(',', $type))); + } + private function group(NotificationBuilder $visible, string $type = 'all', ?Notification $member = null): NotificationBuilder { - $filtered = Notification::query()->from('visible')->select('visible.*', 'identified.group_id') + $filtered = Notification::query()->from('visible')->select('visible.*', 'identified.group_id', 'identified.group_started_at') ->join('identified', 'identified.id', '=', 'visible.id') ->when($member, fn (NotificationBuilder $query, Notification $member): NotificationBuilder => $query->where( 'identified.group_id', fn (QueryBuilder $query): QueryBuilder => $query->from('identified')->where('id', $member->id)->select('group_id'), )) - ->when($type !== 'all', fn (NotificationBuilder $query): NotificationBuilder => $query->whereIn('type', array_merge(...array_map(fn (string $category): array => match ($category) { - 'replies' => ['replied'], - 'quotes' => ['quoted'], - 'stars' => ['starred'], - 'mentions' => ['mentioned'], - 'follows' => ['followed'], - 'moderation' => ['approved', 'rejected'], - default => [], - }, explode(',', $type))))); + ->when($type !== 'all', fn (NotificationBuilder $query): NotificationBuilder => $query->whereIn('type', $this->kinds($type))); $gap = match (DB::connection($visible->getModel()->getConnectionName())->getDriverName()) { 'mysql', 'mariadb' => 'TIMESTAMPDIFF(SECOND, previous_match_at, created_at)', @@ -197,7 +353,8 @@ private function group(NotificationBuilder $visible, string $type = 'all', ?Noti ), identified AS ( SELECT id, - FIRST_VALUE(id) OVER (PARTITION BY topic_kind, topic_id, topic_day, session_number ORDER BY created_at, id) AS group_id + FIRST_VALUE(id) OVER (PARTITION BY topic_kind, topic_id, topic_day, session_number ORDER BY created_at, id) AS group_id, + FIRST_VALUE(created_at) OVER (PARTITION BY topic_kind, topic_id, topic_day, session_number ORDER BY created_at, id) AS group_started_at FROM sessions ), filtered AS ( diff --git a/tests/Unit/Queries/NotificationGroupsQueryTest.php b/tests/Unit/Queries/NotificationGroupsQueryTest.php index ec1ce6fc2..06c1b409f 100644 --- a/tests/Unit/Queries/NotificationGroupsQueryTest.php +++ b/tests/Unit/Queries/NotificationGroupsQueryTest.php @@ -7,13 +7,18 @@ use App\Actions\CreateUserFollow; use App\Actions\DeleteCommit; use App\Actions\DeleteUser; +use App\Enums\CommitStatus; use App\Enums\NotificationType; +use App\Enums\ProfileType; use App\Models\Commit; use App\Models\Notification; +use App\Models\Profile; use App\Models\User; use App\Queries\NotificationGroupsQuery; +use Carbon\CarbonInterface; use Illuminate\Support\Facades\Date; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Str; function starAt(User $user, Commit $commit, int $minutesAgo): void { @@ -36,6 +41,115 @@ function followAt(User $follower, User $user, int $minutesAgo): void Date::withTestNow(now()->subMinutes($minutesAgo), fn () => resolve(CreateUserFollow::class)->handle($follower, $user)); } +/** + * @return list + */ +function every(int $minutes, int $count): array +{ + return array_map(fn (int $step): CarbonInterface => now()->subMinutes($minutes * $step), range(1, $count)); +} + +/** + * @param list> $rows + */ +function insertAnalyzed(User $user, NotificationType $type, array $rows): void +{ + Notification::query()->insert(array_map(fn (array $row): array => [ + 'id' => (string) Str::uuid7(), + 'recipient_id' => $user->id, + 'type' => $type->value, + 'updated_at' => $row['created_at'], + ...$row, + ], $rows)); + DB::statement('ANALYZE'); +} + +/** + * @param list $moments + */ +function starsAt(User $user, array $moments): void +{ + $stargazer = User::factory()->create(); + $commits = array_map(fn (CarbonInterface $at): array => [ + 'id' => (string) Str::uuid7(), 'user_id' => $user->id, 'body' => 'Starred', 'status' => CommitStatus::Approved->value, 'created_at' => $at, 'updated_at' => $at, + ], $moments); + Commit::query()->insert($commits); + + insertAnalyzed($user, NotificationType::Starred, array_map(fn (array $commit): array => [ + 'actor_id' => $stargazer->id, 'commit_id' => $commit['id'], 'subject_id' => $commit['id'], 'key' => 'starred:'.$commit['id'].':'.$stargazer->id, 'created_at' => $commit['created_at'], + ], $commits)); +} + +/** + * @param list $mentioners + * @param list $moments + */ +function mentionsAt(User $user, Commit $commit, array $mentioners, array $moments): void +{ + $replies = array_map(fn (CarbonInterface $at, int $index): array => [ + 'id' => (string) Str::uuid7(), 'user_id' => $mentioners[$index % count($mentioners)]->id, 'parent_id' => $commit->id, 'root_id' => $commit->threadRootId(), 'body' => 'Mention', 'status' => CommitStatus::Approved->value, 'created_at' => $at, 'updated_at' => $at, + ], $moments, array_keys($moments)); + Commit::query()->insert($replies); + + insertAnalyzed($user, NotificationType::Mentioned, array_map(fn (array $reply): array => [ + 'actor_id' => $reply['user_id'], 'commit_id' => $reply['id'], 'key' => 'mentioned:'.$reply['id'], 'created_at' => $reply['created_at'], + ], $replies)); +} + +/** + * @param list $moments + */ +function followsAt(User $user, array $moments): void +{ + $followers = array_map(fn (CarbonInterface $at): array => [ + 'id' => $id = (string) Str::uuid7(), 'github_id' => $id, 'created_at' => $at, 'updated_at' => $at, + ], $moments); + Profile::query()->insert(array_map(fn (array $follower): array => [ + 'id' => (string) Str::uuid7(), 'type' => ProfileType::User, 'github_id' => $follower['github_id'], 'handle' => $follower['id'], 'name' => 'Follower', + 'created_at' => $follower['created_at'], 'updated_at' => $follower['created_at'], + ], $followers)); + User::query()->insert($followers); + DB::table('follows')->insert(array_map(fn (array $follower): array => [ + 'follower_id' => $follower['id'], 'followed_id' => $user->id, 'created_at' => $follower['created_at'], 'updated_at' => $follower['created_at'], + ], $followers)); + + insertAnalyzed($user, NotificationType::Followed, array_map(fn (array $follower): array => [ + 'actor_id' => $follower['id'], 'key' => 'followed:'.$follower['id'], 'created_at' => $follower['created_at'], + ], $followers)); +} + +/** + * @return list + */ +function scrolled(User $user): array +{ + $groups = []; + + do { + $page = resolve(NotificationGroupsQuery::class)->paginate($user); + array_push($groups, ...$page->items()); + request()->merge(['cursor' => $page->nextCursor()?->encode()]); + } while ($page->hasMorePages()); + + return $groups; +} + +/** + * @return list + */ +function wholeHistory(User $user): array +{ + return resolve(NotificationGroupsQuery::class)->events($user)->where('group_rank', 1)->latest()->latest('id')->get()->map(groupIdentity(...))->all(); +} + +/** + * @return array{string, int, int, int, string} + */ +function groupIdentity(Notification $group): array +{ + return [$group->group_id, $group->event_count, $group->actors_count, $group->unread_count, $group->actor_id]; +} + it('groups stars per commit, newest stargazer first', function (): void { $commit = Commit::factory()->create(); $stargazers = User::factory()->count(5)->create(); @@ -190,7 +304,7 @@ function followAt(User $follower, User $user, int $minutesAgo): void resolve(NotificationGroupsQuery::class)->paginate($user); - expect(DB::getQueryLog())->toHaveCount(6); + expect(DB::getQueryLog())->toHaveCount(7); })->with([1, 5]); it('skips loading stacked actors when nothing is grouped', function (): void { @@ -202,7 +316,7 @@ function followAt(User $follower, User $user, int $minutesAgo): void $group = resolve(NotificationGroupsQuery::class)->paginate($user)->items()[0]; - expect(DB::getQueryLog())->toHaveCount(3) + expect(DB::getQueryLog())->toHaveCount(4) ->and($group->actors->sole()->is($group->actor))->toBeTrue(); }); @@ -308,6 +422,65 @@ function followAt(User $follower, User $user, int $minutesAgo): void expect($query->unreadCount($user))->toBe(3)->toBe($wholeHistory); }); +it('keeps a conversation whole when its mentions straddle the grouping window', function (): void { + $user = User::factory()->create(); + $root = Commit::factory()->create(); + [$bob, $carol] = User::factory()->count(2)->create()->all(); + + starsAt($user, every(30, 1000)); + mentionsAt($user, $root, [$bob, $carol], every(50, 600)); + + $groups = resolve(NotificationGroupsQuery::class)->paginate($user)->items(); + + expect($groups[1]->type)->toBe(NotificationType::Mentioned) + ->and($groups[1]->event_count)->toBe(600) + ->and($groups[1]->unread_count)->toBe(600) + ->and($groups[1]->actors_count)->toBe(2) + ->and($groups[1]->actors->pluck('id')->all())->toBe([$bob->id, $carol->id]) + ->and(array_map(groupIdentity(...), $groups))->toBe(array_slice(wholeHistory($user), 0, NotificationGroupsQuery::PER_PAGE)); +}); + +it('keeps the older stars of a commit the grouping window begins after', function (): void { + $user = User::factory()->create(); + $viral = Commit::factory()->for($user)->create(); + $stargazers = User::factory()->count(4)->create(); + + starsAt($user, every(30, 1300)); + foreach ($stargazers->take(3) as $index => $stargazer) { + starAt($stargazer, $viral, 40003 - $index); + } + starAt($stargazers[3], $viral, 1); + + $groups = resolve(NotificationGroupsQuery::class)->paginate($user)->items(); + + expect($groups[0]->commit_id)->toBe($viral->id) + ->and($groups[0]->event_count)->toBe(4) + ->and($groups[0]->actors_count)->toBe(4) + ->and($groups[0]->actors->pluck('id')->all())->toBe([$stargazers[3]->id, $stargazers[2]->id, $stargazers[1]->id]) + ->and(array_map(groupIdentity(...), $groups))->toBe(array_slice(wholeHistory($user), 0, NotificationGroupsQuery::PER_PAGE)); +}); + +it('splits a day of follows at its hour gap like grouping the whole history does, on every page', function (): void { + $user = User::factory()->create(); + $days = range(1, 3); + $at = fn (int $day, string $time): CarbonInterface => today()->subDays($day)->setTimeFromTimeString($time); + $span = fn (int $day, string $from, string $to, int $seconds): array => array_map( + fn (int $offset): CarbonInterface => $at($day, $from)->addSeconds($offset), + range(0, (int) $at($day, $from)->diffInSeconds($at($day, $to)), $seconds), + ); + + followsAt($user, array_merge(...array_map(fn (int $day): array => [...$span($day, '09:00', '11:00', 10), ...$span($day, '13:00', '15:00', 10)], $days))); + starsAt($user, array_merge(...array_map(fn (int $day): array => $span($day, '09:01:30', '14:58:30', 180), $days))); + + $groups = scrolled($user); + $sessions = array_values(array_filter($groups, fn (Notification $group): bool => $group->type === NotificationType::Followed)); + + expect($groups[0]->type)->toBe(NotificationType::Followed) + ->and($groups[0]->event_count)->toBe(721) + ->and(array_map(fn (Notification $session): int => $session->actors_count, $sessions))->toBe(array_fill(0, 6, 721)) + ->and(array_map(groupIdentity(...), $groups))->toBe(wholeHistory($user))->toHaveCount(366); +}); + it('compiles the grouping window for each database dialect with bound recipient and filter values', function (string $driver, string $gap): void { $user = User::factory()->make(['id' => '019c1234-5678-7000-8000-123456789abc']); $original = DB::getDefaultConnection();