From 856a5c9f0d5d09785ad97b8f4d1a43b27c827ecb Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Thu, 29 Oct 2026 11:57:20 +0100 Subject: [PATCH] feat: list organization pages in the sitemap Organization pages are public and indexable but no sitemap listed them. They are now listed by the month their page was made, dated by the last change of their GitHub profile or of the approved commits that mention them. Placeholder handles of profiles renamed away on GitHub are left out. --- .../OrganizationSitemapController.php | 22 +++++ app/Http/Controllers/SitemapController.php | 4 + .../ShowOrganizationSitemapRequest.php | 46 ++++++++++ app/Queries/SitemapQuery.php | 92 ++++++++++++++++++- routes/web.php | 4 + .../OrganizationSitemapControllerTest.php | 60 ++++++++++++ .../Controllers/SitemapControllerTest.php | 12 ++- 7 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 app/Http/Controllers/OrganizationSitemapController.php create mode 100644 app/Http/Requests/ShowOrganizationSitemapRequest.php create mode 100644 tests/Feature/Controllers/OrganizationSitemapControllerTest.php diff --git a/app/Http/Controllers/OrganizationSitemapController.php b/app/Http/Controllers/OrganizationSitemapController.php new file mode 100644 index 000000000..1a13566ed --- /dev/null +++ b/app/Http/Controllers/OrganizationSitemapController.php @@ -0,0 +1,22 @@ +view('sitemaps.urls', [ + 'urls' => array_map(fn (array $organization): array => [ + 'loc' => route('user.show', $organization['handle']), + 'lastmod' => $organization['updated_at']->toAtomString(), + ], $sitemaps->organizationsIn($request->month())), + ], headers: ['Content-Type' => 'application/xml; charset=UTF-8']); + } +} diff --git a/app/Http/Controllers/SitemapController.php b/app/Http/Controllers/SitemapController.php index 076e5ffcd..9b412e4b6 100644 --- a/app/Http/Controllers/SitemapController.php +++ b/app/Http/Controllers/SitemapController.php @@ -17,6 +17,10 @@ public function index(SitemapQuery $sitemaps): Response 'loc' => route('user-sitemap.show', $month['month']), 'lastmod' => $month['updated_at']->toAtomString(), ], $sitemaps->userMonths()), + ...array_map(fn (array $month): array => [ + 'loc' => route('organization-sitemap.show', $month['month']), + 'lastmod' => $month['updated_at']->toAtomString(), + ], $sitemaps->organizationMonths()), ...array_map(fn (array $day): array => [ 'loc' => route('commit-sitemap.show', $day['day']), 'lastmod' => $day['updated_at']->toAtomString(), diff --git a/app/Http/Requests/ShowOrganizationSitemapRequest.php b/app/Http/Requests/ShowOrganizationSitemapRequest.php new file mode 100644 index 000000000..77b5bd649 --- /dev/null +++ b/app/Http/Requests/ShowOrganizationSitemapRequest.php @@ -0,0 +1,46 @@ +month()->format('Y-m') === $this->route('month') && resolve(SitemapQuery::class)->hasOrganizationsIn($this->month()) + ? Response::allow() + : Response::denyAsNotFound(); + } + + /** + * @return array{} + */ + public function rules(): array + { + return []; + } + + /** + * The start of the month whose new organizations the sitemap lists, in UTC. + */ + public function month(): CarbonImmutable + { + $month = $this->route('month'); + assert(is_string($month)); + + $start = CarbonImmutable::createFromFormat('!Y-m', $month); + assert($start instanceof CarbonImmutable); + + return $start; + } +} diff --git a/app/Queries/SitemapQuery.php b/app/Queries/SitemapQuery.php index 6b0a8ec02..b2319fc8f 100644 --- a/app/Queries/SitemapQuery.php +++ b/app/Queries/SitemapQuery.php @@ -4,16 +4,19 @@ namespace App\Queries; +use App\Enums\ProfileType; +use App\Enums\TopicType; use App\Models\Commit; +use App\Models\Profile; use App\Models\User; use Carbon\CarbonImmutable; use Illuminate\Database\Eloquent\Builder; use Illuminate\Support\Facades\DB; /** - * Profiles are listed by the month they signed up and commits by the day they were written, in UTC, so a URL never - * moves to another sitemap and the older sitemaps rarely change. A sitemap may list 50,000 URLs, far more than a month - * of sign-ups or a day of commits. + * Profiles are listed by the month they signed up, organizations by the month their page was made, and commits by the + * day they were written, in UTC, so a URL never moves to another sitemap and the older sitemaps rarely change. A + * sitemap may list 50,000 URLs, far more than a month of sign-ups or a day of commits. */ final readonly class SitemapQuery { @@ -40,6 +43,29 @@ public function userMonths(): array ], $months); } + /** + * The months, as "2026-10", in which the organizations' pages were made, oldest first, with when the last of them + * changed. + * + * @return list + */ + public function organizationMonths(): array + { + /** @var list $months */ + $months = DB::query() + ->fromSub($this->organizations(), 'organizations') + ->selectRaw("to_char(created_at, 'YYYY-MM') as month, max(updated_at) as updated_at") + ->groupBy('month') + ->orderBy('month') + ->get() + ->all(); + + return array_map(fn (object $month): array => [ + 'month' => $month->month, + 'updated_at' => CarbonImmutable::parse($month->updated_at), + ], $months); + } + /** * The days, as "2026-10-29", on which approved commits were written, oldest first, with when the last of them * changed, so search engines only read again the sitemaps that changed. @@ -68,6 +94,11 @@ public function hasUsersIn(CarbonImmutable $month): bool return $this->usersSignedUpIn($month)->exists(); } + public function hasOrganizationsIn(CarbonImmutable $month): bool + { + return $this->organizationsMadeIn($month)->exists(); + } + public function hasCommitsOn(CarbonImmutable $day): bool { return $this->commitsWrittenOn($day)->exists(); @@ -93,6 +124,26 @@ public function usersIn(CarbonImmutable $month): array ], $users); } + /** + * Read as plain rows rather than models, like the profiles. + * + * @return list + */ + public function organizationsIn(CarbonImmutable $month): array + { + /** @var list $organizations */ + $organizations = $this->organizationsMadeIn($month) + ->orderBy('profiles.id') + ->toBase() + ->get() + ->all(); + + return array_map(fn (object $organization): array => [ + 'handle' => $organization->handle, + 'updated_at' => CarbonImmutable::parse($organization->updated_at), + ], $organizations); + } + /** * Read as plain rows rather than models, with the author and the images in the same query, since a day lists * thousands of them. @@ -163,6 +214,41 @@ private function usersSignedUpIn(CarbonImmutable $month): Builder ->where('users.created_at', '<', $month->addMonth()); } + /** + * Every organization has a page, but not the placeholder handle of a profile whose handle was taken on GitHub, + * which its next sync replaces. An organization's page changes with its GitHub profile and the approved commits + * that mention it. + * + * @return Builder + */ + private function organizations(): Builder + { + $lastMentions = $this->commits() + ->toBase() + ->join('commit_topics', 'commit_topics.commit_id', '=', 'commits.id') + ->where('commit_topics.topic_type', TopicType::Organization) + ->select('commit_topics.topic_id') + ->selectRaw('max(commits.updated_at) as updated_at') + ->groupBy('commit_topics.topic_id'); + + return Profile::query() + ->leftJoinSub($lastMentions, 'last_mentions', 'last_mentions.topic_id', '=', 'profiles.id') + ->where('profiles.type', ProfileType::Organization) + ->where('profiles.handle', 'not like', '\_%') + ->select(['profiles.handle', 'profiles.created_at']) + ->selectRaw('greatest(profiles.updated_at, last_mentions.updated_at) as updated_at'); + } + + /** + * @return Builder + */ + private function organizationsMadeIn(CarbonImmutable $month): Builder + { + return $this->organizations() + ->where('profiles.created_at', '>=', $month) + ->where('profiles.created_at', '<', $month->addMonth()); + } + private function commits(): CommitBuilder { return Commit::query()->approved(); diff --git a/routes/web.php b/routes/web.php index e5bb2d572..f611835f4 100644 --- a/routes/web.php +++ b/routes/web.php @@ -76,6 +76,7 @@ use App\Http\Controllers\OperationsEventController; use App\Http\Controllers\OrganizationController; use App\Http\Controllers\OrganizationMentionController; +use App\Http\Controllers\OrganizationSitemapController; use App\Http\Controllers\OrgArchiveController; use App\Http\Controllers\OrgController; use App\Http\Controllers\OrgMemberController; @@ -237,6 +238,9 @@ Route::get('sitemaps/users-{month}.xml', [UserSitemapController::class, 'show']) ->where('month', '[0-9]{4}-[0-9]{2}') ->name('user-sitemap.show'); + Route::get('sitemaps/organizations-{month}.xml', [OrganizationSitemapController::class, 'show']) + ->where('month', '[0-9]{4}-[0-9]{2}') + ->name('organization-sitemap.show'); Route::get('sitemaps/commits-{day}.xml', [CommitSitemapController::class, 'show']) ->where('day', '[0-9]{4}-[0-9]{2}-[0-9]{2}') ->name('commit-sitemap.show'); diff --git a/tests/Feature/Controllers/OrganizationSitemapControllerTest.php b/tests/Feature/Controllers/OrganizationSitemapControllerTest.php new file mode 100644 index 000000000..9df57d3e7 --- /dev/null +++ b/tests/Feature/Controllers/OrganizationSitemapControllerTest.php @@ -0,0 +1,60 @@ +organization()->create(['handle' => 'laravel', 'created_at' => '2026-10-01 00:00:00']); + Profile::factory()->organization()->create(['handle' => 'symfony', 'created_at' => '2026-10-31 23:59:59']); + resolve(CreateCommitTopics::class)->handle(Commit::factory()->create(['body' => 'Thanks @laravel']), ['laravel' => $laravel->id]); + Profile::factory()->organization()->create(['handle' => 'last-month', 'created_at' => '2026-09-30 23:59:59']); + Profile::factory()->organization()->create(['handle' => 'next-month', 'created_at' => '2026-11-01 00:00:00']); + Profile::factory()->organization()->create(['handle' => '_renamed-123', 'created_at' => '2026-10-15 00:00:00']); + Commit::factory()->for(User::factory()->state(['handle' => 'octocat', 'created_at' => '2026-10-15 00:00:00']))->create(); + + $response = $this->get(route('organization-sitemap.show', '2026-10')); + + $response->assertOk() + ->assertHeader('Content-Type', 'application/xml; charset=UTF-8') + ->assertSeeInOrder([ + ''.route('user.show', 'laravel').'', + ''.route('user.show', 'symfony').'', + ], false) + ->assertDontSee('last-month') + ->assertDontSee('next-month') + ->assertDontSee('_renamed-123') + ->assertDontSee('octocat'); + + expect(mb_substr_count($response->getContent(), ''))->toBe(2); +}); + +it('dates an organization by the last change of its GitHub profile or the approved commits that mention it', function (string $profile, string $mention, string $lastmod): void { + $this->travelTo('2026-10-01 00:00:00'); + $laravel = Profile::factory()->organization()->create(['handle' => 'laravel']); + Profile::query()->whereKey($laravel->id)->toBase()->update(['updated_at' => $profile]); + resolve(CreateCommitTopics::class)->handle(Commit::factory()->create(['body' => '@laravel', 'updated_at' => $mention]), ['laravel' => $laravel->id]); + resolve(CreateCommitTopics::class)->handle(Commit::factory()->pending()->create(['body' => '@laravel', 'updated_at' => '2026-10-29 00:00:00']), ['laravel' => $laravel->id]); + resolve(CreateCommitTopics::class)->handle(Commit::factory()->create(['body' => '@symfony', 'updated_at' => '2026-10-28 00:00:00']), ['symfony' => Profile::factory()->organization()->create()->id]); + + $this->get(route('organization-sitemap.show', '2026-10')) + ->assertOk() + ->assertSee(''.route('user.show', 'laravel')."\n {$lastmod}", false); +})->with([ + 'its GitHub profile' => ['2026-10-20 10:00:00', '2026-10-10 10:00:00', '2026-10-20T10:00:00+00:00'], + 'a commit that mentions it' => ['2026-10-10 10:00:00', '2026-10-20 10:00:00', '2026-10-20T10:00:00+00:00'], +]); + +it('has no sitemap for a month without organizations', function (): void { + Profile::factory()->organization()->create(['created_at' => '2026-09-15 00:00:00']); + Profile::factory()->organization()->create(['handle' => '_renamed-123', 'created_at' => '2026-10-15 00:00:00']); + + $this->get(route('organization-sitemap.show', '2026-10'))->assertNotFound(); +}); + +it('has no sitemap for a month that does not exist', function (string $month): void { + $this->get("/sitemaps/organizations-{$month}.xml")->assertNotFound(); +})->with(['2026-13', '2026-00', '2026-1', 'abc']); diff --git a/tests/Feature/Controllers/SitemapControllerTest.php b/tests/Feature/Controllers/SitemapControllerTest.php index 1459aa621..48da94841 100644 --- a/tests/Feature/Controllers/SitemapControllerTest.php +++ b/tests/Feature/Controllers/SitemapControllerTest.php @@ -3,9 +3,11 @@ declare(strict_types=1); use App\Models\Commit; +use App\Models\Profile; use App\Models\User; -it('lists a sitemap for every month of sign-ups and every day of commits', function (): void { +it('lists a sitemap for every month of sign-ups, every month of organizations and every day of commits', function (): void { + Profile::factory()->organization()->create(['created_at' => '2026-08-15 00:00:00']); $user = User::factory()->create(['created_at' => '2026-09-30 23:59:59']); Commit::factory()->for($user)->create(['created_at' => '2026-10-15 08:00:00']); Commit::factory()->for($user)->create(['created_at' => '2026-10-15 23:59:59']); @@ -19,11 +21,12 @@ ->assertSeeInOrder([ ''.route('user-sitemap.show', '2026-09').'', ''.route('user-sitemap.show', now()->format('Y-m')).'', + ''.route('organization-sitemap.show', '2026-08').'', ''.route('commit-sitemap.show', '2026-10-15').'', ''.route('commit-sitemap.show', '2026-10-29').'', ], false); - expect(mb_substr_count($response->getContent(), ''))->toBe(4); + expect(mb_substr_count($response->getContent(), ''))->toBe(5); })->freezeTime(); it('leaves out the days and months with nothing approved', function (): void { @@ -39,17 +42,18 @@ it('serves every sitemap without cookies, so a cache can share it', function (string $route): void { Commit::factory()->create(); + Profile::factory()->organization()->create(); $response = $this->get(route($route, match ($route) { 'sitemap.index' => [], - 'user-sitemap.show' => now()->format('Y-m'), + 'user-sitemap.show', 'organization-sitemap.show' => now()->format('Y-m'), 'commit-sitemap.show' => now()->toDateString(), })); $response->assertOk() ->assertHeader('Cache-Control', 'max-age=3600, public') ->assertHeaderMissing('Set-Cookie'); -})->with(['sitemap.index', 'user-sitemap.show', 'commit-sitemap.show'])->freezeTime(); +})->with(['sitemap.index', 'user-sitemap.show', 'organization-sitemap.show', 'commit-sitemap.show'])->freezeTime(); it('dates every sitemap by the last change of what it lists', function (): void { $this->travelTo('2026-10-01 00:00:00');