feat: import every user's GitHub follows from a queued job, reading their stored handle

app/Actions/CreateUserGitHubFollows.php

@@ -40,10 +40,13 @@ public function handle(User $user, bool $notify = true): void
             return;
         }
 
-        $login = $this->github->login($user->github_id);
-        $followingIds = $login === null ? ['users' => [], 'organizations' => []] : $this->github->followingIds($login, self::MaxPages);
+        // The handle is the GitHub login as of the user's last sign-in, which saves looking it up by ID: a sign-in
+        // updates it right before importing. An account renamed since then isn't found, or in the rare case someone took
+        // its old handle, has theirs read instead. The bots' handles belong to strangers on GitHub, so they import nothing.
+        $isBot = in_array($user->github_id, User::BOT_GITHUB_IDS, true);
+        $followingIds = $isBot ? ['users' => [], 'organizations' => []] : $this->github->followingIds($user->handle, self::MaxPages);
         $following = $importsPeople ? $this->users($user, $followingIds['users']) : new Collection();
-        $followers = $importsPeople && $login !== null ? $this->users($user, $this->github->followerIds($login, self::MaxPages)) : new Collection();
+        $followers = $importsPeople && ! $isBot ? $this->users($user, $this->github->followerIds($user->handle, self::MaxPages)) : new Collection();
         $organizationIds = collect($followingIds['organizations'])->unique()->values();
 
         DB::transaction(function () use ($user, $notify, $following, $followers, $organizationIds, $importsPeople, $importsOrganizations): void {

app/Console/Commands/ImportGitHubFollowsCommand.php

@@ -4,80 +4,21 @@
 
 namespace App\Console\Commands;
 
-use App\Actions\CreateUserGitHubFollows;
-use App\Contracts\Services\GitHubProvider;
-use App\Models\User;
+use App\Jobs\ImportAllGitHubFollowsJob;
 use Illuminate\Console\Attributes\Description;
 use Illuminate\Console\Attributes\Signature;
 use Illuminate\Console\Command;
-use Illuminate\Database\Eloquent\Builder;
-use Illuminate\Http\Client\ConnectionException;
-use Illuminate\Http\Client\RequestException;
 
 #[Signature('follows:import')]
-#[Description('Follow, for every user not imported yet, who and which organizations they follow and who follows them on GitHub, without notifying anyone')]
+#[Description('Queue the import of who and which organizations every user not imported yet follows, and who follows them, on GitHub, without notifying anyone')]
 final class ImportGitHubFollowsCommand extends Command
 {
-    /**
-     * Below this many requests left, the command stops, leaving the rest to the app until the rate limit resets: above
-     * the floor where looking up mentioned organizations pauses, so they carry on meanwhile.
-     */
-    private const int RateLimitFloor = 1500;
-
-    public function handle(CreateUserGitHubFollows $createUserGitHubFollows, GitHubProvider $github): int
+    public function handle(): int
     {
-        $imported = 0;
-        $failed = 0;
-
-        foreach ($this->users()->lazyById() as $user) {
-            $rateLimit = $github->rateLimit();
-
-            // A user takes one request for their login, and up to the maximum of pages each way.
-            if ($rateLimit !== null && $rateLimit['remaining'] < self::RateLimitFloor + 1 + 2 * CreateUserGitHubFollows::MaxPages) {
-                $this->components->warn("Stopped after {$imported} users, to keep ".self::RateLimitFloor." GitHub requests for the rest of the app: {$rateLimit['remaining']} left.");
-                $this->components->info("{$this->users()->count()} users left. Run the command again after {$rateLimit['resets_at']->utc()->toDateTimeString()} UTC ({$rateLimit['resets_at']->diffForHumans()}), when the GitHub rate limit resets.");
-
-                return self::FAILURE;
-            }
-
-            try {
-                $createUserGitHubFollows->handle($user, notify: false);
-
-                $imported++;
-            } catch (RequestException $exception) {
-                if (in_array($exception->response->status(), [403, 429], true)) {
-                    $this->components->error("GitHub rate limit reached after {$imported} users. Run the command again later.");
-
-                    return self::FAILURE;
-                }
-
-                $this->failed($user, $exception);
-
-                $failed++;
-            } catch (ConnectionException $exception) {
-                $this->failed($user, $exception);
+        dispatch(new ImportAllGitHubFollowsJob());
 
-                $failed++;
-            }
-        }
-
-        $this->components->info("Imported the GitHub follows of {$imported} users.");
-
-        return $failed === 0 ? self::SUCCESS : self::FAILURE;
-    }
-
-    /**
-     * @return Builder<User>
-     */
-    private function users(): Builder
-    {
-        return User::query()->where(fn (Builder $query): Builder => $query->whereNull('github_follows_imported_at')->orWhereNull('github_organization_follows_imported_at'));
-    }
-
-    private function failed(User $user, RequestException|ConnectionException $exception): void
-    {
-        report($exception);
+        $this->components->info('Queued the import, '.ImportAllGitHubFollowsJob::BatchSize.' users at a time. It waits for the GitHub rate limit to reset by itself, keeping '.ImportAllGitHubFollowsJob::Reserve.' requests for the rest of the app, and logs when every user is imported.');
 
-        $this->components->warn("Could not import the GitHub follows of {$user->mention()}.");
+        return self::SUCCESS;
     }
 }

app/Jobs/ImportAllGitHubFollowsJob.php

@@ -0,0 +1,124 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Jobs;
+
+use App\Actions\CreateUserGitHubFollows;
+use App\Contracts\Services\GitHubProvider;
+use App\Models\User;
+use Carbon\CarbonInterface;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Http\Client\ConnectionException;
+use Illuminate\Http\Client\RequestException;
+use Illuminate\Http\Client\Response;
+use Illuminate\Queue\Attributes\Timeout;
+use Illuminate\Support\Facades\Log;
+
+/**
+ * Imports the GitHub follows of every user not imported yet, a few users at a time, without notifying anyone: each job
+ * queues the next one, after the users it read, so a run takes no worker for long, survives deploys and waits for the
+ * GitHub rate limit to reset by itself. It is unique, so launching it twice doesn't run it twice.
+ *
+ * Its timeout leaves room for the budget plus the longest a single user can take, under the queue's retry_after.
+ */
+#[Timeout(300)]
+final class ImportAllGitHubFollowsJob implements ShouldBeUniqueUntilProcessing, ShouldQueue
+{
+    use Queueable;
+
+    public const int BatchSize = 25;
+
+    /**
+     * The requests left to the rest of the app, which shares the rate limit: above the floor where looking up mentioned
+     * organizations pauses, so they carry on meanwhile.
+     */
+    public const int Reserve = 1500;
+
+    /**
+     * The seconds a job spends importing before it leaves the rest to the next one, so it ends well within the worker's
+     * timeout: a user rarely takes a second, but one following thousands takes up to twenty requests.
+     */
+    public const int Budget = 30;
+
+    public function __construct(public ?string $after = null)
+    {
+        //
+    }
+
+    public function handle(GitHubProvider $github, CreateUserGitHubFollows $createUserGitHubFollows): void
+    {
+        $rateLimit = $github->rateLimit();
+
+        // A user takes up to the maximum of pages each way.
+        if ($rateLimit !== null && $rateLimit['remaining'] < self::Reserve + self::BatchSize * 2 * CreateUserGitHubFollows::MaxPages) {
+            // A minute past the reset, so GitHub counts from the new hour by then.
+            $this->continueAt($rateLimit['resets_at']->addMinute());
+
+            return;
+        }
+
+        $users = User::query()
+            ->where(fn (Builder $query): Builder => $query->whereNull('github_follows_imported_at')->orWhereNull('github_organization_follows_imported_at'))
+            ->when($this->after !== null, fn (Builder $query): Builder => $query->where('id', '>', $this->after))
+            ->orderBy('id')
+            ->limit(self::BatchSize)
+            ->get();
+
+        $deadline = now()->addSeconds(self::Budget);
+
+        foreach ($users as $user) {
+            try {
+                $createUserGitHubFollows->handle($user, notify: false);
+            } catch (RequestException $exception) {
+                if ($this->isRateLimited($exception->response)) {
+                    $retryAfter = $exception->response->header('Retry-After');
+                    $this->continueAt(is_numeric($retryAfter) ? now()->addSeconds((int) $retryAfter) : now()->addMinutes(15));
+
+                    return;
+                }
+
+                // An account renamed since its last sign-in isn't found under its handle: it is imported at its next one.
+                report_if($exception->response->status() !== 404, $exception);
+            } catch (ConnectionException $exception) {
+                report($exception);
+            }
+
+            $this->after = $user->id;
+
+            if (now()->greaterThanOrEqualTo($deadline)) {
+                $this->continueAt(now());
+
+                return;
+            }
+        }
+
+        if ($users->count() < self::BatchSize) {
+            Log::info('Went through every user whose GitHub follows were not imported yet.');
+
+            return;
+        }
+
+        $this->continueAt(now());
+    }
+
+    /**
+     * GitHub answers a rate limit, primary or secondary, with a 429 or a 403 that says so. Any other 403 won't go away by
+     * waiting, so that user is skipped rather than holding up everyone after them.
+     */
+    private function isRateLimited(Response $response): bool
+    {
+        return $response->status() === 429 || ($response->status() === 403 && str_contains(mb_strtolower($response->body()), 'rate limit'));
+    }
+
+    /**
+     * Some queues can't hold a job back more than 15 minutes, so a longer wait reads the rate limit again by then.
+     */
+    private function continueAt(CarbonInterface $at): void
+    {
+        dispatch(new self($this->after))->delay($at->min(now()->addMinutes(15)));
+    }
+}

tests/Feature/Commands/ImportGitHubFollowsTest.php

@@ -2,120 +2,15 @@
 
 declare(strict_types=1);
 
-use App\Contracts\Services\GitHubProvider;
-use App\Models\Notification;
-use App\Models\Organization;
-use App\Models\User;
-use Carbon\CarbonImmutable;
-use GuzzleHttp\Psr7\Response as Psr7Response;
-use Illuminate\Http\Client\ConnectionException;
-use Illuminate\Http\Client\RequestException;
-use Illuminate\Http\Client\Response;
-use Illuminate\Support\Facades\Exceptions;
+use App\Jobs\ImportAllGitHubFollowsJob;
+use Illuminate\Support\Facades\Queue;
 
-it('imports the github follows of every user not imported yet, without notifying anyone', function (): void {
-    $alice = User::factory()->create();
-    $bob = User::factory()->create();
-    $imported = User::factory()->create(['github_follows_imported_at' => now()->subDay(), 'github_organization_follows_imported_at' => now()->subDay()]);
-
-    $github = $this->mock(GitHubProvider::class);
-    $github->shouldReceive('rateLimit')->andReturnNull();
-    $github->shouldReceive('login')->with($alice->github_id)->andReturn($alice->handle);
-    $github->shouldReceive('followingIds')->with($alice->handle, 10)->andReturn(['users' => [$bob->github_id, $imported->github_id], 'organizations' => []]);
-    $github->shouldReceive('followerIds')->with($alice->handle, 10)->andReturn([]);
-    $github->shouldReceive('login')->with($bob->github_id)->andReturn($bob->handle);
-    $github->shouldReceive('followingIds')->with($bob->handle, 10)->andReturn(['users' => [], 'organizations' => []]);
-    $github->shouldReceive('followerIds')->with($bob->handle, 10)->andReturn([$alice->github_id]);
-
-    $this->artisan('follows:import')
-        ->expectsOutputToContain('Imported the GitHub follows of 2 users.')
-        ->assertSuccessful();
-
-    expect($alice->isFollowing($bob))->toBeTrue()
-        ->and($alice->isFollowing($imported))->toBeTrue()
-        ->and($bob->refresh()->github_follows_imported_at)->not->toBeNull()
-        ->and(Notification::query()->count())->toBe(0);
-
-    $github->shouldNotHaveReceived('login', [$imported->github_id]);
-});
-
-it('keeps going when the follows of a user cannot be imported', function (string $exception, Closure $throwable): void {
-    Exceptions::fake();
-
-    $failing = User::factory()->create(['handle' => 'failing']);
-    $user = User::factory()->create();
-
-    $github = $this->mock(GitHubProvider::class);
-    $github->shouldReceive('rateLimit')->andReturnNull();
-    $github->shouldReceive('login')->with($failing->github_id)->andThrow($throwable());
-    $github->shouldReceive('login')->with($user->github_id)->andReturn($user->handle);
-    $github->shouldReceive('followingIds')->with($user->handle, 10)->andReturn(['users' => [], 'organizations' => []]);
-    $github->shouldReceive('followerIds')->with($user->handle, 10)->andReturn([]);
-
-    $this->artisan('follows:import')
-        ->expectsOutputToContain('Could not import the GitHub follows of @failing.')
-        ->expectsOutputToContain('Imported the GitHub follows of 1 users.')
-        ->assertFailed();
-
-    expect($failing->refresh()->github_follows_imported_at)->toBeNull()
-        ->and($user->refresh()->github_follows_imported_at)->not->toBeNull();
-
-    Exceptions::assertReported($exception);
-})->with([
-    'server error' => [RequestException::class, fn (): RequestException => new RequestException(new Response(new Psr7Response(500)))],
-    'connection error' => [ConnectionException::class, fn (): ConnectionException => new ConnectionException()],
-]);
-
-it('stops when github rate limits the requests', function (int $status): void {
-    $user = User::factory()->create();
-    User::factory()->create();
-
-    $github = $this->mock(GitHubProvider::class);
-    $github->shouldReceive('rateLimit')->andReturnNull();
-    $github->shouldReceive('login')->with($user->github_id)->once()->andThrow(new RequestException(new Response(new Psr7Response($status))));
-
-    $this->artisan('follows:import')
-        ->expectsOutputToContain('GitHub rate limit reached after 0 users. Run the command again later.')
-        ->assertFailed();
-
-    $github->shouldHaveReceived('login')->once();
-})->with([403, 429]);
-
-it('imports the organizations followed by the users whose people were imported before', function (): void {
-    $user = User::factory()->create(['github_follows_imported_at' => now()->subDay()]);
-    $organization = Organization::factory()->create();
-
-    $github = $this->mock(GitHubProvider::class);
-    $github->shouldReceive('rateLimit')->andReturnNull();
-    $github->shouldReceive('login')->with($user->github_id)->andReturn($user->handle);
-    $github->shouldReceive('followingIds')->with($user->handle, 10)->andReturn(['users' => [], 'organizations' => [$organization->profile->github_id]]);
+it('queues the import of every user not imported yet', function (): void {
+    Queue::fake();
 
     $this->artisan('follows:import')
-        ->expectsOutputToContain('Imported the GitHub follows of 1 users.')
+        ->expectsOutputToContain('Queued the import, 25 users at a time. It waits for the GitHub rate limit to reset by itself, keeping 1500 requests for the rest of the app')
         ->assertSuccessful();
 
-    expect($user->followedOrganizations()->pluck('organizations.id')->all())->toBe([$organization->id]);
-});
-
-it('stops before using up the github requests the rest of the app needs, and says when to run it again', function (): void {
-    $this->travelTo(CarbonImmutable::parse('2026-11-01 10:00:00', 'UTC'));
-
-    $imported = User::factory()->create();
-    User::factory()->count(2)->create();
-
-    $github = $this->mock(GitHubProvider::class);
-    $github->shouldReceive('rateLimit')->andReturn(
-        ['remaining' => 1521, 'limit' => 5000, 'resets_at' => CarbonImmutable::parse('2026-11-01 10:42:00', 'UTC')],
-        ['remaining' => 1520, 'limit' => 5000, 'resets_at' => CarbonImmutable::parse('2026-11-01 10:42:00', 'UTC')],
-    );
-    $github->shouldReceive('login')->with($imported->github_id)->once()->andReturn($imported->handle);
-    $github->shouldReceive('followingIds')->with($imported->handle, 10)->andReturn(['users' => [], 'organizations' => []]);
-    $github->shouldReceive('followerIds')->with($imported->handle, 10)->andReturn([]);
-
-    $this->artisan('follows:import')
-        ->expectsOutputToContain('Stopped after 1 users, to keep 1500 GitHub requests for the rest of the app: 1520 left.')
-        ->expectsOutputToContain('2 users left. Run the command again after 2026-11-01 10:42:00 UTC (42 minutes from now), when the GitHub rate limit resets.')
-        ->assertFailed();
-
-    $github->shouldHaveReceived('login')->once();
+    Queue::assertPushed(ImportAllGitHubFollowsJob::class, fn (ImportAllGitHubFollowsJob $job): bool => $job->after === null);
 });