From a8da950b82d1c1aa11f6efd6feec36eac9d77251 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 30 Aug 2026 03:45:35 +0000 Subject: [PATCH] site: security headers, entity schema, and the files AI crawlers look for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a 15-engine AEO audit, limited to what was verified against the live site by hand. Several of the engines' findings were wrong and are not acted on here: one reported no robots.txt, no sitemap and no JSON-LD when all three are served; five called the sitemap lastmod "future-dated" when the date was that day; and the "unrendered ${userInput} template literal" is a TypeScript sample in a block on the security page, showing the injection DiskPush deliberately does not do. What was real: - No security headers at all. Only cache-control was set — no HSTS, CSP, X-Frame-Options, nosniff, Referrer-Policy or Permissions-Policy. The CSP keeps 'unsafe-inline' for script and style because Next inlines its hydration payload and the app's styles; scripts are otherwise same-origin, and object-src/frame-ancestors 'none' close the usual bypasses. - Every sitemap URL carried one build timestamp. A lastmod that is identical site-wide and changes on every deploy tells a crawler nothing, so it gets discounted — worse than sending none. Dates now come from the files each page is built from; doc pages use their own markdown's mtime, which is exactly when that page last changed. 18 URLs, 18 distinct dates. - No Organization. The schema described an application that nothing published: the only trace of who makes DiskPush was a GitHub URL and the domain on the security address. Organization and WebSite are now in the graph, and SoftwareApplication names them as publisher and author. - /llms.txt and /.well-known/security.txt 404'd. Both are generated rather than written: a hand-kept copy of the docs index goes stale, and a stale llms.txt is worse than none because models quote it confidently. The security.txt Expires is computed a year out for the same reason — a hard-coded date silently invalidates the file on a day nobody has diarised. - /security and /docs/security were the same markdown at two URLs, with the same title. The promoted page is now canonical for both. robots.txt names the AI crawlers explicitly. The wildcard already allowed them, so this grants nothing new; it states a position rather than leaving it to be inferred, and makes narrowing it later an edit rather than an unrecorded decision. Left alone deliberately, because they are content decisions rather than bugs: about/team page, general contact route, on-site changelog, comparison pages, testimonials. Also outside the repo: _dmarc.diskpush.com is a CNAME to uixie.porkbun.com, which serves no TXT record, so DMARC resolves to nothing. Verified by building and serving the site: six headers present, llms.txt and security.txt served, 18 distinct sitemap dates, /docs/security canonical to /security, and Organization/WebSite in the graph. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Y5jnkZKX4AdPgBMzMosxE7 --- .../web/app/.well-known/security.txt/route.ts | 35 +++++++++++ apps/web/app/docs/[slug]/page.tsx | 13 +++- apps/web/app/llms.txt/route.ts | 58 +++++++++++++++++ apps/web/app/page.tsx | 32 ++++++++++ apps/web/app/robots.ts | 30 ++++++++- apps/web/app/sitemap.ts | 63 ++++++++++++++++--- apps/web/next.config.mjs | 39 ++++++++++++ 7 files changed, 259 insertions(+), 11 deletions(-) create mode 100644 apps/web/app/.well-known/security.txt/route.ts create mode 100644 apps/web/app/llms.txt/route.ts diff --git a/apps/web/app/.well-known/security.txt/route.ts b/apps/web/app/.well-known/security.txt/route.ts new file mode 100644 index 0000000..985afe1 --- /dev/null +++ b/apps/web/app/.well-known/security.txt/route.ts @@ -0,0 +1,35 @@ +import { SITE } from '@/lib/site' + +/** + * /.well-known/security.txt — RFC 9116. + * + * The disclosure address was already published in prose on /security; this is + * the same fact where a scanner looks for it. + * + * Expires is required by the RFC and must be in the future, so it is computed + * rather than written down: a hard-coded date is a file that silently becomes + * invalid on a day nobody has in their calendar. A year out, recomputed on each + * deploy, means it stays valid as long as the site is maintained — and goes + * stale only once the site itself has been abandoned, which is exactly the + * signal the field is for. + */ +export const dynamic = 'force-dynamic' + +export async function GET() { + const expires = new Date() + expires.setUTCFullYear(expires.getUTCFullYear() + 1) + + const body = `Contact: mailto:security@profullstack.com +Expires: ${expires.toISOString()} +Preferred-Languages: en +Canonical: ${SITE.url}/.well-known/security.txt +Policy: ${SITE.url}/security +` + + return new Response(body, { + headers: { + 'content-type': 'text/plain; charset=utf-8', + 'cache-control': 'public, max-age=0, s-maxage=86400', + }, + }) +} diff --git a/apps/web/app/docs/[slug]/page.tsx b/apps/web/app/docs/[slug]/page.tsx index cb75cb6..9b8e24e 100644 --- a/apps/web/app/docs/[slug]/page.tsx +++ b/apps/web/app/docs/[slug]/page.tsx @@ -6,6 +6,12 @@ import { listDocs, readDoc } from '@/lib/docs' type Params = { params: Promise<{ slug: string }> } +/** + * Docs that are also published at a shorter, promoted URL. The promoted page is + * canonical; this one keeps its place in the docs tree without competing with it. + */ +const PROMOTED: Record = { security: '/security' } + /** Pre-rendered at build time: the docs are static files, so the pages are too. */ export async function generateStaticParams() { const docs = await listDocs() @@ -20,7 +26,12 @@ export async function generateMetadata({ params }: Params): Promise { return { title: doc.title, description: meta?.description || `${doc.title} — DiskPush documentation.`, - alternates: { canonical: `/docs/${slug}` }, + // A doc with a promoted page of its own is the same bytes at two URLs — + // same markdown, same rendered HTML, same title. Point the canonical at the + // promoted one rather than letting a crawler pick, and rather than dropping + // either URL: /security is linked from every footer, and /docs/security is + // where the docs sidebar goes. + alternates: { canonical: PROMOTED[slug] ?? `/docs/${slug}` }, } } diff --git a/apps/web/app/llms.txt/route.ts b/apps/web/app/llms.txt/route.ts new file mode 100644 index 0000000..63f3327 --- /dev/null +++ b/apps/web/app/llms.txt/route.ts @@ -0,0 +1,58 @@ +import { listDocs } from '@/lib/docs' +import { SITE } from '@/lib/site' + +/** + * /llms.txt — the llmstxt.org convention. + * + * Generated rather than written, for the same reason the docs pages are: a + * hand-kept copy of the docs index is a copy that goes stale, and a stale + * llms.txt is worse than none because models quote it confidently. + * + * Deliberately not a copy of the homepage. It answers, in the order a model + * needs them, the questions a marketing page buries: what this is, what it + * costs, who makes it, and which page to read next. + */ +export const dynamic = 'force-static' + +export async function GET() { + const docs = await listDocs() + + const body = `# ${SITE.name} + +> ${SITE.description} + +${SITE.name} is free and open source under the MIT licence. There is no account, +no paid tier, and no ${SITE.name} server in any transfer path — the desktop app +and the CLI speak SSH and rsync directly to your own hosts. Linux first, macOS +supported. Maintained by Profullstack, Inc. + +## Start here + +- [Home](${SITE.url}): What it is, how it works, and the ten most common questions. +- [Download](${SITE.url}/download): Current release for Linux and macOS, plus the one-line installer. +- [Documentation](${SITE.url}/docs): Every guide, rendered from the repository's own docs directory. + +## Documentation + +${docs.map((doc) => `- [${doc.title}](${SITE.url}/docs/${doc.slug}): ${doc.description}`).join('\n')} + +## Project + +- [Source](${SITE.repo}): MIT licensed, on GitHub. +- [Releases](${SITE.releases}): Version history and build artifacts. +- [Security model](${SITE.url}/security): Threat model, argument handling, host keys, credentials and destructive-operation guards. +- [Privacy](${SITE.url}/privacy): What the app and the site do and do not collect. + +## Contact + +- Security reports: security@profullstack.com +- Everything else: GitHub issues at ${SITE.repo}/issues +` + + return new Response(body, { + headers: { + 'content-type': 'text/plain; charset=utf-8', + 'cache-control': 'public, max-age=0, s-maxage=3600', + }, + }) +} diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx index 62694fc..7686ed0 100644 --- a/apps/web/app/page.tsx +++ b/apps/web/app/page.tsx @@ -54,6 +54,33 @@ export default function HomePage() { __html: JSON.stringify({ '@context': 'https://schema.org', '@graph': [ + // The maker, as an entity a knowledge graph can resolve. Without + // this the schema described an application that nothing published: + // the only trace of who builds DiskPush was a GitHub URL and the + // domain on the security address. + { + '@type': 'Organization', + '@id': `${SITE.url}/#organization`, + name: 'Profullstack, Inc.', + url: 'https://profullstack.com', + logo: `${SITE.url}/logo.png`, + sameAs: ['https://github.com/profullstack'], + contactPoint: { + '@type': 'ContactPoint', + contactType: 'security', + email: 'security@profullstack.com', + url: `${SITE.url}/security`, + }, + }, + { + '@type': 'WebSite', + '@id': `${SITE.url}/#website`, + name: SITE.name, + url: SITE.url, + description: SITE.description, + publisher: { '@id': `${SITE.url}/#organization` }, + inLanguage: 'en', + }, { '@type': 'SoftwareApplication', name: 'DiskPush', @@ -61,6 +88,11 @@ export default function HomePage() { operatingSystem: 'Linux, macOS', description: SITE.description, url: SITE.url, + downloadUrl: `${SITE.url}/download`, + license: 'https://opensource.org/licenses/MIT', + isAccessibleForFree: true, + publisher: { '@id': `${SITE.url}/#organization` }, + author: { '@id': `${SITE.url}/#organization` }, offers: { '@type': 'Offer', price: '0', priceCurrency: 'USD' }, }, { diff --git a/apps/web/app/robots.ts b/apps/web/app/robots.ts index 91868db..a4c15ee 100644 --- a/apps/web/app/robots.ts +++ b/apps/web/app/robots.ts @@ -1,9 +1,37 @@ import type { MetadataRoute } from 'next' import { SITE } from '@/lib/site' +/** + * The wildcard already allows everything, so these named rules grant no access + * the crawlers did not have. They are here because the policy is worth stating + * rather than inferring: DiskPush is MIT-licensed software whose documentation + * exists to be read, and an answer engine that summarises it accurately is + * doing the thing the docs are for. + * + * Named explicitly so the position is legible to a person reading the file, and + * so that narrowing it later is an edit rather than a decision nobody recorded. + */ +const AI_CRAWLERS = [ + 'GPTBot', + 'OAI-SearchBot', + 'ChatGPT-User', + 'ClaudeBot', + 'Claude-User', + 'PerplexityBot', + 'Google-Extended', + 'Applebot-Extended', + 'CCBot', + 'meta-externalagent', + 'Bytespider', +] + export default function robots(): MetadataRoute.Robots { return { - rules: [{ userAgent: '*', allow: '/' }], + rules: [ + { userAgent: '*', allow: '/' }, + ...AI_CRAWLERS.map((userAgent) => ({ userAgent, allow: '/' })), + ], sitemap: `${SITE.url}/sitemap.xml`, + host: SITE.url, } } diff --git a/apps/web/app/sitemap.ts b/apps/web/app/sitemap.ts index 8e93ac8..3050860 100644 --- a/apps/web/app/sitemap.ts +++ b/apps/web/app/sitemap.ts @@ -1,21 +1,66 @@ +import { stat } from 'node:fs/promises' +import { join } from 'node:path' import type { MetadataRoute } from 'next' import { listDocs } from '@/lib/docs' import { SITE } from '@/lib/site' +/** + * Every URL used to carry `new Date()`, so all eighteen shared one build + * timestamp. A lastmod that changes on every deploy and is identical across the + * site says nothing about what actually changed, and crawlers discount it — + * which is worse than sending none, because it costs a field and buys nothing. + * + * The real date is on disk. Doc pages are rendered from the repository's own + * `docs/*.md`, so that file's mtime *is* when the page last changed; the rest + * are their own source files. Falls back to the build time only when a stat + * fails, which should not happen but must not break the sitemap if it does. + */ +const ROOT = join(process.cwd(), '..', '..') +const buildTime = new Date() + +async function modified(...relative: string[]): Promise { + const times = await Promise.all( + relative.map(async (path) => { + try { + return (await stat(join(ROOT, path))).mtime + } catch { + return null + } + }), + ) + const known = times.filter((time): time is Date => time !== null) + if (known.length === 0) return buildTime + // The newest of the sources a page is built from. + return new Date(Math.max(...known.map((time) => time.getTime()))) +} + export default async function sitemap(): Promise { const docs = await listDocs() - const now = new Date() - return [ - { url: SITE.url, lastModified: now, changeFrequency: 'weekly', priority: 1 }, - { url: `${SITE.url}/download`, lastModified: now, changeFrequency: 'weekly', priority: 0.9 }, - { url: `${SITE.url}/docs`, lastModified: now, changeFrequency: 'weekly', priority: 0.8 }, - { url: `${SITE.url}/security`, lastModified: now, changeFrequency: 'monthly', priority: 0.5 }, - { url: `${SITE.url}/privacy`, lastModified: now, changeFrequency: 'yearly', priority: 0.3 }, - ...docs.map((doc) => ({ + const web = 'apps/web' + + const [home, download, docsIndex, security, privacy] = await Promise.all([ + modified(`${web}/app/page.tsx`, `${web}/app/layout.tsx`), + modified(`${web}/app/download/page.tsx`), + modified(`${web}/app/docs/page.tsx`), + modified(`${web}/app/security/page.tsx`), + modified(`${web}/app/privacy/page.tsx`), + ]) + + const docEntries = await Promise.all( + docs.map(async (doc) => ({ url: `${SITE.url}/docs/${doc.slug}`, - lastModified: now, + lastModified: await modified(`docs/${doc.slug}.md`), changeFrequency: 'monthly' as const, priority: 0.7, })), + ) + + return [ + { url: SITE.url, lastModified: home, changeFrequency: 'weekly', priority: 1 }, + { url: `${SITE.url}/download`, lastModified: download, changeFrequency: 'weekly', priority: 0.9 }, + { url: `${SITE.url}/docs`, lastModified: docsIndex, changeFrequency: 'weekly', priority: 0.8 }, + { url: `${SITE.url}/security`, lastModified: security, changeFrequency: 'monthly', priority: 0.5 }, + { url: `${SITE.url}/privacy`, lastModified: privacy, changeFrequency: 'yearly', priority: 0.3 }, + ...docEntries, ] } diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index 3dc2a3e..f2ca04b 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -1,4 +1,40 @@ /** @type {import('next').NextConfig} */ + +// Response headers the site was serving none of. +// +// The CSP is deliberately not `default-src 'self'` alone: Next inlines its +// hydration payload in a