From 271474d3e685e5232ac2034bb563d079d506abdf Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 12:46:46 -0400 Subject: [PATCH 01/34] feat(migrate): add migration command suite --- .gitignore | 2 + bun.lock | 6 +- packages/cli-core/package.json | 4 +- packages/cli-core/src/cli-program.ts | 2 + .../cli-core/src/commands/migrate/README.md | 742 +++++++++++++++++ .../src/commands/migrate/delete.test.ts | 422 ++++++++++ .../cli-core/src/commands/migrate/delete.ts | 336 ++++++++ .../src/commands/migrate/export/auth0.test.ts | 308 +++++++ .../src/commands/migrate/export/auth0.ts | 346 ++++++++ .../src/commands/migrate/export/authjs.ts | 141 ++++ .../src/commands/migrate/export/betterauth.ts | 191 +++++ .../src/commands/migrate/export/clerk.test.ts | 281 +++++++ .../src/commands/migrate/export/clerk.ts | 263 ++++++ .../migrate/export/db-exports.test.ts | 358 +++++++++ .../src/commands/migrate/export/db-options.ts | 111 +++ .../commands/migrate/export/firebase.test.ts | 451 +++++++++++ .../src/commands/migrate/export/firebase.ts | 458 +++++++++++ .../src/commands/migrate/export/index.ts | 181 +++++ .../commands/migrate/export/registry.test.ts | 39 + .../src/commands/migrate/export/registry.ts | 80 ++ .../src/commands/migrate/export/shared.ts | 85 ++ .../src/commands/migrate/export/supabase.ts | 138 ++++ .../src/commands/migrate/import-users.test.ts | 335 ++++++++ .../src/commands/migrate/import-users.ts | 347 ++++++++ .../src/commands/migrate/index.test.ts | 208 +++++ .../cli-core/src/commands/migrate/index.ts | 134 ++++ .../src/commands/migrate/lib/analysis.test.ts | 97 +++ .../src/commands/migrate/lib/analysis.ts | 80 ++ .../commands/migrate/lib/clerk-config.test.ts | 85 ++ .../src/commands/migrate/lib/clerk-config.ts | 119 +++ .../src/commands/migrate/lib/db.test.ts | 237 ++++++ .../cli-core/src/commands/migrate/lib/db.ts | 227 ++++++ .../src/commands/migrate/lib/instance.test.ts | 83 ++ .../src/commands/migrate/lib/instance.ts | 93 +++ .../commands/migrate/lib/log-files.test.ts | 200 +++++ .../src/commands/migrate/lib/log-files.ts | 141 ++++ .../src/commands/migrate/lib/logger.test.ts | 110 +++ .../src/commands/migrate/lib/logger.ts | 114 +++ .../commands/migrate/lib/readiness.test.ts | 322 ++++++++ .../src/commands/migrate/lib/readiness.ts | 238 ++++++ .../src/commands/migrate/lib/retry.test.ts | 156 ++++ .../src/commands/migrate/lib/retry.ts | 66 ++ .../commands/migrate/lib/scheduler.test.ts | 69 ++ .../src/commands/migrate/lib/scheduler.ts | 51 ++ .../src/commands/migrate/lib/settings.test.ts | 42 + .../src/commands/migrate/lib/settings.ts | 43 + .../migrate/lib/supabase-providers.test.ts | 143 ++++ .../migrate/lib/supabase-providers.ts | 145 ++++ .../commands/migrate/lib/transform.test.ts | 218 +++++ .../src/commands/migrate/lib/transform.ts | 469 +++++++++++ .../src/commands/migrate/logs/clean.ts | 69 ++ .../src/commands/migrate/logs/convert.ts | 121 +++ .../src/commands/migrate/logs/index.ts | 72 ++ .../src/commands/migrate/logs/list.ts | 64 ++ .../migrate/logs/logs-interactive.test.ts | 177 +++++ .../src/commands/migrate/logs/logs.test.ts | 219 +++++ .../src/commands/migrate/readme.test.ts | 124 +++ .../commands/migrate/run-interactive.test.ts | 301 +++++++ .../cli-core/src/commands/migrate/run.test.ts | 749 ++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 519 ++++++++++++ .../commands/migrate/transformers/auth0.ts | 43 + .../commands/migrate/transformers/authjs.ts | 38 + .../migrate/transformers/betterauth.ts | 46 ++ .../commands/migrate/transformers/clerk.ts | 45 ++ .../commands/migrate/transformers/firebase.ts | 121 +++ .../migrate/transformers/list.test.ts | 116 +++ .../src/commands/migrate/transformers/list.ts | 67 ++ .../migrate/transformers/load-custom.test.ts | 222 ++++++ .../migrate/transformers/load-custom.ts | 164 ++++ .../commands/migrate/transformers/registry.ts | 74 ++ .../commands/migrate/transformers/shared.ts | 93 +++ .../commands/migrate/transformers/supabase.ts | 70 ++ .../migrate/transformers/transformers.test.ts | 405 ++++++++++ .../cli-core/src/commands/migrate/types.ts | 193 +++++ .../src/commands/migrate/validator.test.ts | 80 ++ .../src/commands/migrate/validator.ts | 99 +++ .../src/commands/migrate/wizard.test.ts | 267 +++++++ .../cli-core/src/commands/migrate/wizard.ts | 167 ++++ 78 files changed, 14240 insertions(+), 2 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/README.md create mode 100644 packages/cli-core/src/commands/migrate/delete.test.ts create mode 100644 packages/cli-core/src/commands/migrate/delete.ts create mode 100644 packages/cli-core/src/commands/migrate/export/auth0.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/auth0.ts create mode 100644 packages/cli-core/src/commands/migrate/export/authjs.ts create mode 100644 packages/cli-core/src/commands/migrate/export/betterauth.ts create mode 100644 packages/cli-core/src/commands/migrate/export/clerk.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/clerk.ts create mode 100644 packages/cli-core/src/commands/migrate/export/db-exports.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/db-options.ts create mode 100644 packages/cli-core/src/commands/migrate/export/firebase.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/firebase.ts create mode 100644 packages/cli-core/src/commands/migrate/export/index.ts create mode 100644 packages/cli-core/src/commands/migrate/export/registry.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/registry.ts create mode 100644 packages/cli-core/src/commands/migrate/export/shared.ts create mode 100644 packages/cli-core/src/commands/migrate/export/supabase.ts create mode 100644 packages/cli-core/src/commands/migrate/import-users.test.ts create mode 100644 packages/cli-core/src/commands/migrate/import-users.ts create mode 100644 packages/cli-core/src/commands/migrate/index.test.ts create mode 100644 packages/cli-core/src/commands/migrate/index.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/analysis.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/analysis.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/clerk-config.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/db.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/db.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/instance.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/instance.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/log-files.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/log-files.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/logger.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/logger.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/readiness.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/readiness.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/retry.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/retry.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/scheduler.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/scheduler.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/settings.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/settings.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/supabase-providers.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/transform.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/transform.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/clean.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/convert.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/index.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/list.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts create mode 100644 packages/cli-core/src/commands/migrate/logs/logs.test.ts create mode 100644 packages/cli-core/src/commands/migrate/readme.test.ts create mode 100644 packages/cli-core/src/commands/migrate/run-interactive.test.ts create mode 100644 packages/cli-core/src/commands/migrate/run.test.ts create mode 100644 packages/cli-core/src/commands/migrate/run.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/auth0.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/authjs.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/betterauth.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/clerk.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/firebase.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/list.test.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/list.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/load-custom.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/registry.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/shared.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/supabase.ts create mode 100644 packages/cli-core/src/commands/migrate/transformers/transformers.test.ts create mode 100644 packages/cli-core/src/commands/migrate/types.ts create mode 100644 packages/cli-core/src/commands/migrate/validator.test.ts create mode 100644 packages/cli-core/src/commands/migrate/validator.ts create mode 100644 packages/cli-core/src/commands/migrate/wizard.test.ts create mode 100644 packages/cli-core/src/commands/migrate/wizard.ts diff --git a/.gitignore b/.gitignore index 91df74696..c5f69f393 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ coverage # logs logs +!packages/cli-core/src/commands/migrate/logs/ +!packages/cli-core/src/commands/migrate/logs/** _.log report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json diff --git a/bun.lock b/bun.lock index 6dd7ef10c..40357bdac 100644 --- a/bun.lock +++ b/bun.lock @@ -19,7 +19,7 @@ }, "packages/cli": { "name": "clerk", - "version": "2.3.0", + "version": "3.0.0", "bin": { "clerk": "./bin/clerk", }, @@ -36,11 +36,13 @@ "@commander-js/extra-typings": "^15.0.0", "@napi-rs/keyring": "^1.3.0", "commander": "^15.0.0", + "csv-parser": "^3.2.1", "env-paths": "^4.0.0", "external-editor": "^3.1.0", "magicast": "^0.5.3", "semver": "^7.8.5", "yaml": "^2.9.0", + "zod": "^4.4.3", }, "devDependencies": { "@clerk/shared": "^4.13.1", @@ -335,6 +337,8 @@ "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "csv-parser": ["csv-parser@3.2.1", "", { "bin": { "csv-parser": "bin/csv-parser" } }, "sha512-v8RPMSglouR9od735SnwSxLBbCJqEPSbgm1R5qfr8yIiMUCEFjox56kRZid0SvgHJEkxeIEu3+a9QS3YRh7CuA=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], diff --git a/packages/cli-core/package.json b/packages/cli-core/package.json index f6be6ee93..ffddc6db2 100644 --- a/packages/cli-core/package.json +++ b/packages/cli-core/package.json @@ -22,11 +22,13 @@ "@commander-js/extra-typings": "^15.0.0", "@napi-rs/keyring": "^1.3.0", "commander": "^15.0.0", + "csv-parser": "^3.2.1", "env-paths": "^4.0.0", "external-editor": "^3.1.0", "magicast": "^0.5.3", "semver": "^7.8.5", - "yaml": "^2.9.0" + "yaml": "^2.9.0", + "zod": "^4.4.3" }, "devDependencies": { "@clerk/shared": "^4.13.1", diff --git a/packages/cli-core/src/cli-program.ts b/packages/cli-core/src/cli-program.ts index 9ea89cb54..ca63b2212 100644 --- a/packages/cli-core/src/cli-program.ts +++ b/packages/cli-core/src/cli-program.ts @@ -22,6 +22,7 @@ import { registerCompletion } from "./commands/completion/index.ts"; import { registerUpdate } from "./commands/update/index.ts"; import { registerDeploy } from "./commands/deploy/index.ts"; import { registerWebhooks } from "./commands/webhooks/index.ts"; +import { registerMigrate } from "./commands/migrate/index.ts"; import { getEnvironment } from "./lib/config.ts"; import { setCurrentEnv, @@ -75,6 +76,7 @@ const registrants: CommandRegistrant[] = [ registerUpdate, registerDeploy, registerWebhooks, + registerMigrate, registerExtras, ]; diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md new file mode 100644 index 000000000..f18e38223 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/README.md @@ -0,0 +1,742 @@ +# `clerk migrate` + +Migrate users into a Clerk instance from another auth provider, or from another +Clerk instance. + +## Targeting And Auth + +`migrate run` resolves its Backend API key through the CLI's standard chain: + +| Flag | Description | +| ------------------------ | ---------------------------------------------------------------- | +| `--secret-key ` | Use a specific Backend API secret key directly | +| `--clerk-secret-key ` | **Deprecated** alias for `--secret-key`; warns and keeps working | +| `--app ` | Target an application directly, even outside a linked project | +| `--instance ` | Target `dev`, `prod`, or a full instance ID | + +Resolution order: `--secret-key` → `--app` + Platform API lookup → +`CLERK_SECRET_KEY` → the keyless project's own key → a linked project profile +from `clerk link`. + +The **instance type is read from the key**: `sk_live_…` is treated as +production, anything else as development. That choice drives the throughput +defaults and the hard development-instance cap below. + +## Commands + +### `clerk migrate` (interactive) + +Bare `clerk migrate` dispatches to `migrate run`, which walks a human through +the migration instead of demanding flags — mirroring how bare `clerk deploy` +dispatches to `deploy run`. + +```sh +clerk migrate +``` + +It picks the transformer from a list built off the registry, asks for the file, +collects Firebase's hash parameters when they are needed, and pre-fills every +answer from the last run's `.settings` so a repeat migration is mostly pressing +enter. Anything already passed as a flag is not asked for. + +Then it prints the [Migration Readiness report](#migration-readiness-report) +and waits for confirmation. Declining writes nothing to Clerk. + +**Agent mode never prompts.** `clerk migrate` with no flags exits with a usage +error naming exactly what to pass: + +``` +`clerk migrate` is interactive and cannot prompt in agent mode. +Pass --transformer and --file . +``` + +### `clerk migrate run` + +Reads an exported user file, maps it onto Clerk's user schema, validates every +record, and creates the users through the Backend API. + +```sh +clerk migrate run -y --transformer clerk --file users.json +``` + +| Flag | Description | +| --------------------------------------- | --------------------------------------------------------------- | +| `-t, --transformer ` | Source platform the file came from (see below) | +| `--transformer-file ` | A transformer you wrote, for a platform with no built-in | +| `-f, --file ` | Path to the export. `.json` or `.csv` | +| `-r, --resume-after ` | Skip every user up to and including this **source** ID | +| `--require-password` | Import only users that carry a password digest | +| `--skip-unsupported-providers` | Supabase: skip users whose only social provider is off in Clerk | +| `--firebase-signer-key ` | Firebase base64 signer key | +| `--firebase-salt-separator ` | Firebase base64 salt separator | +| `--firebase-rounds ` | Firebase scrypt rounds | +| `--firebase-mem-cost ` | Firebase scrypt memory cost | +| `-y, --yes` | Skip the confirmation prompt | + +Plus the targeting flags from the table above: `--secret-key`, +`--clerk-secret-key`, `--app` and `--instance`. + +`--transformer` and `--file` are required. Omitting either fails with a usage +error that names the valid values. + +Failures do not stop the run: each user's outcome is written to the log and the +import continues. A `429` backs off — honouring `Retry-After` when the response +carries it — and retries up to 5 times before the user is recorded as failed. +The command exits non-zero if any user failed. + +Two failures do abort the whole run, because continuing would produce a +corrupt instance: + +- An **unrecognized password hasher**, which would import credentials nobody can + sign in with. +- A **`--resume-after` ID that is not in the file**, which would otherwise + re-import every user the previous run already created. + +#### Additional identifiers + +Only the first verified email and phone go on `POST /v1/users`. Every +additional verified identifier, and every unverified one, is attached +afterwards with its own request. A failure there is logged and the user still +counts as imported — a duplicate secondary email should not undo an otherwise +successful user. + +#### Throughput + +Defaults follow Clerk's documented `POST /v1/users` limits: 100 req/s for +production instances, 10 req/s for development. Concurrency defaults to ~95% of +that, assuming ~100ms of API latency. Both are overridable: + +| Variable | Effect | +| --------------------------------- | ----------------------------- | +| `CLERK_MIGRATE_RATE_LIMIT` | Requests per second | +| `CLERK_MIGRATE_CONCURRENCY_LIMIT` | Concurrent in-flight requests | + +A non-numeric or non-positive value is ignored in favour of the default. + +**Development instances refuse imports over 500 users**, matching Clerk's own +limit — the run fails before any request is sent. + +### `clerk migrate export` + +Gets users **out** of a source platform, so there is something to feed +`migrate run`. + +```sh +clerk migrate export # pick a platform +clerk migrate export clerk --output users.json +clerk migrate export auth0 --domain my-tenant.us.auth0.com \ + --client-id … --client-secret … +``` + +The platform is an optional positional. Omitted, you get a picker built from +the registry; given, it runs directly. Each platform resolves its own flags — +what Auth0 needs (a tenant domain and M2M credentials) has nothing in common +with what a database export needs. + +| Platform | Source | Feeds | +| ------------ | -------------------------------- | -------------------------- | +| `clerk` | Clerk Backend API | `--transformer clerk` | +| `auth0` | Auth0 Management API | `--transformer auth0` | +| `supabase` | Supabase Postgres (`auth.users`) | `--transformer supabase` | +| `authjs` | Auth.js database | `--transformer authjs` | +| `betterauth` | Better Auth database | `--transformer betterauth` | +| `firebase` | Firebase Identity Toolkit | `--transformer firebase` | + +Exports land at `./exports/-export.json` unless `--output` says +otherwise. `--output` resolves against the **current directory**, like every +other path flag here. + +| Flag | Platforms | Description | +| -------------------------- | ---------------------------------- | -------------------------------------------- | +| `-o, --output ` | all | Where to write the export | +| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL or SQLite connection string | +| `--service-account ` | `firebase` | Path to a service account key JSON file | +| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | +| `--client-id ` | `auth0` | Machine-to-machine application client ID | +| `--client-secret ` | `auth0` | Machine-to-machine application client secret | + +`export clerk` also takes the targeting flags — it reads from a Clerk instance, +so it resolves a key exactly the way `migrate run` does. + +After each export you get a field-coverage table — which Clerk-relevant fields +were present on how many users — so you know the data is thin _before_ you +import it, not after: + +``` +Field coverage + ● 3/3 have an email address + ○ 0/3 have a phone number + ○ 1/3 have a username + ○ 2/3 have a password (not exportable — see below) + +Exported 3 user(s) to /project/exports/clerk-export.json +Next: clerk migrate run --transformer clerk --file exports/clerk-export.json +``` + +Every export also writes `logs/export-.log`, so `migrate logs list` +sees it alongside imports and deletions. + +#### Neither platform exports passwords + +- **Clerk** never returns password digests, TOTP secrets or backup codes over + the API — only the `*_enabled` booleans. Migrated users must reset their + password in the destination instance. +- **Auth0**'s Management API does not return password hashes either; they come + only from a support request. Add a `passwordHash` field to each user before + importing, or migrate without passwords. + +Both say so on every run. The coverage row counts users who _have_ a password, +so the size of the gap is visible up front. + +#### Database-backed exports (`supabase`, `authjs`, `betterauth`) + +These three read the database directly, over **`--db-url`**: + +```sh +clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" +clerk migrate export authjs --db-url "mysql://user:...@127.0.0.1:3306/authjs" +clerk migrate export betterauth --db-url "./db.sqlite" +``` + +Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`. Both are +built into the runtime, so nothing native ships in the binary — that is the +whole reason the `engines.bun` floor exists. Resolution is `--db-url`, then +`SUPABASE_DB_URL` / `AUTHJS_DB_URL` / `BETTERAUTH_DB_URL`, then a masked prompt, +since a connection string carries the password inline. + +**Connection strings are redacted everywhere.** Errors show +`postgres://***@host/db`, including when the password itself contains an +unencoded `@` — the most common mistake, and exactly when the string ends up in +an error message. + +Connection failures get a hint rather than a driver error. Bun reports both an +unreachable host and a closed port as "Connection closed", so: + +| Situation | What you are told | +| ------------------------ | ------------------------------------------------------------------------------------------------- | +| Host or port unreachable | Check the host and port. On Supabase: use the pooler connection string, or enable the IPv4 add-on | +| Credentials rejected | Check the user and password | +| Table missing | Check the database name and SELECT permission. On Supabase: enable Auth, connect as `postgres` | +| SQLite file missing | Check the path and that the file is readable | + +**`supabase` reads the database rather than the Admin API** because +`encrypted_password` exists only there. An API-based export would force every +user to reset their password; this one carries the bcrypt digests across. It +also keeps `raw_app_meta_data`, which is what `--skip-unsupported-providers` +reads at import time. + +**`authjs` tries `User`, then `user`, then `users`.** Auth.js has no single +schema — Prisma capitalizes the table, Drizzle does not, and Postgres treats +the difference as significant once quoted. The run reports which one it found. +Auth.js core stores no passwords, so its users arrive without credentials. + +**`betterauth` detects its plugin columns from the schema.** The username +plugin adds `username`, admin adds `banned`, phone-number adds `phoneNumber`, +and so on; selecting a column that is not there fails the whole query, and the +database answers the question better than the user can. Passwords come from a +`LEFT JOIN` onto the credential `account` row — left, not inner, so a user who +only ever signed in with OAuth is still exported. + +#### `firebase` + +```sh +clerk migrate export firebase --service-account ./service-account.json +``` + +Needs a service account key from **Project settings → Service accounts → +Generate new private key**, with the Firebase Authentication Admin role. The +file is validated before anything reaches the network, so downloading the web +app config by mistake fails in a second with the right console page named +rather than after an auth round-trip. Key material never appears in output. + +Firebase's scrypt is a modified variant, so a digest is worthless without the +project's four hash parameters. The export **reads them from the project** and +prints the exact import command: + +``` +Password hash parameters +Read from the project. Import with: + clerk migrate run -y --transformer firebase --file exports/firebase-export.json \ + --firebase-signer-key "…" --firebase-salt-separator "…" \ + --firebase-rounds 8 --firebase-mem-cost 14 +``` + +Reading the config needs a broader role than listing users, so if it is denied +the export still succeeds and points at **Authentication → Users → (⋮) → +Password hash parameters** instead. An export with no password hashes says so +and asks for nothing. + +A user whose hash is present but whose salt is not (or the reverse) has both +dropped: half a credential produces a user nobody can sign in as. + +`FIREBASE_AUTH_EMULATOR_HOST` is honoured, so this works against the local +Firebase emulator as well as production. + +**No `firebase-admin`.** The spike the plan called for was run and _passed_ — a +compiled binary can import the SDK and complete `listUsers`, so the known +Firestore-under-compile bug does not reach the Auth Admin surface. It was still +not adopted: the SDK is 74 MB across 158 packages, including Firestore and +Cloud Storage, which would roughly double the ~62 MB binary every user +downloads, to serve one subcommand. What it does here is two REST calls and an +RS256 JWT, and Bun's Web Crypto signs RS256 with no dependency at all. + +#### Auth0 credentials + +Needs a machine-to-machine application with the `read:users` scope +(Applications → APIs → Auth0 Management API → Machine to Machine +Applications). Resolved from flags, then `AUTH0_DOMAIN` / `AUTH0_CLIENT_ID` / +`AUTH0_CLIENT_SECRET`, then a prompt. In agent mode a prompt is impossible, so +it exits naming **every** missing credential at once rather than one per run. + +Auth0 pages this endpoint only through the first **1000** users. Past that the +export stops and says so, pointing at Auth0's bulk export job — silently +returning the first thousand would read as "that is everyone". + +### `clerk migrate delete` + +The undo for a bad migration. Deletes the users a previous `clerk migrate run` +created in this directory, matched by the `external_id` the import stamped on +each one. + +```sh +clerk migrate delete # confirms first +clerk migrate delete -y # non-interactive +``` + +Takes the same targeting flags as `migrate run` (`--secret-key`, `--app`, +`--instance`). + +Flat rather than under a noun group: it is the one command in this tree that +destroys data **in Clerk**, and is worth keeping short and prominent. (Contrast +`migrate logs clean`, which only removes local files.) + +#### What it will and will not touch + +`.settings` is the only record of what a run created, so that is what +identifies the migration being undone. Without it the command fails and +explains — deleting nothing silently would look like a successful undo. + +Users are found with `GET /v1/users?external_id=…`, 100 IDs per request. Only a +user Clerk itself reports as carrying one of _this_ migration's external IDs is +ever deleted; anything else in the instance is out of scope. IDs with no +matching user are skipped and reported, which is the normal case for a partial +migration or one already partly undone. + +It confirms before acting — defaulting to **no** — and requires `-y` in +non-interactive or agent mode. + +#### Failures + +Rate limiting and 429 retries are literally the same code path as the import +(`lib/retry.ts`), not a second implementation that drifts. + +A failure on one user is logged and the rest continue: a half-undone migration +with no record of which half is far worse than a reported failure. Every +attempt lands in a timestamped `logs/user-deletion-.log`, carrying +both the source ID and the Clerk ID. The command exits non-zero if any deletion +failed. + +### `clerk migrate logs` + +Everything that touches the local `./logs/` directory. Noun-verb like every +other group in the CLI (`config pull`, `users list`), rather than the standalone +tool's `clean-logs`/`convert-logs`, which were npm script names. + +Grouping also disambiguates the two deletes in this tree: `migrate logs clean` +removes **local files**, `migrate delete` removes **users from a Clerk +instance**. + +```sh +clerk migrate logs # defaults to list +clerk migrate logs list --json +clerk migrate logs clean -y +clerk migrate logs convert --all +clerk migrate logs convert migration-2026-01-01T12-00-00.log +``` + +| Subcommand | Takes | Description | +| -------------- | ------------------ | ----------------------------------------------- | +| `logs list` | `--json` | Type, timestamp, size and entry count per file | +| `logs clean` | `-y, --yes` | Delete the `.log` files in `./logs/` | +| `logs convert` | `[file…]`, `--all` | NDJSON → a JSON array, written as `.json` | + +All three read the directory through one shared enumerator, which is what makes +`logs list` nearly free. + +#### `logs list` + +The default, because listing is read-only and therefore safe to run by +accident. Reports each file's type, timestamp, size and entry count, newest +first; `--json` gives an agent the same data without parsing NDJSON. + +``` +TYPE TIMESTAMP SIZE ENTRIES +migration 2026-02-01T09-14-22 4.1 KB 120 +deletion 2026-01-30T17-02-51 612 B 18 +``` + +Says so plainly when `./logs/` is empty or absent. + +#### `logs clean` + +Destructive, so the confirmation is not optional: interactive runs prompt +(defaulting to **no**), and non-interactive or agent runs must pass `-y` rather +than being allowed to assume. Deletes `.log` files only — converted `.json` +output is left alone. + +#### `logs convert` + +Turns NDJSON into a JSON array for spreadsheet or database analysis, written +alongside the original as `.json`. The original is left in place. + +Takes file positionals or `--all`; given neither, an interactive terminal +offers a multiselect and an agent gets a usage error naming both alternatives. + +A malformed line is reported with its line number and skipped, and the +remaining entries still convert: + +``` +migration-2026-01-01T12-00-00.log:2 is not valid JSON and was skipped — … +1 malformed line skipped. +``` + +That beats failing the whole file: a run killed mid-write leaves one truncated +final line, and the hundreds of complete entries before it are still worth +having. It also beats dropping the line silently, which would leave a JSON +array that looks complete. + +## Transformers + +A transformer maps one platform's export onto Clerk's user schema. Adding a +platform is one file in `transformers/` plus one line in `transformers/registry.ts` — +`--transformer`'s accepted values and its tab-completion both read from that array. + +| Key | Source | Passwords | Notes | +| ------------ | ----------------------------- | ----------------- | ---------------------------------------------------------------- | +| `clerk` | Clerk Dashboard export | as exported | Instance to instance, e.g. development → production | +| `auth0` | Auth0 Export Users API | `bcrypt` | Hashes need a support request to Auth0; not in a standard export | +| `authjs` | Auth.js / NextAuth user table | none | Assumes `SELECT id, name, email, email_verified, created_at` | +| `betterauth` | Better Auth export | `bcrypt` | Reads the credential account's `password_hash` | +| `firebase` | `firebase auth:export` | `scrypt_firebase` | CSV or JSON; needs the four hash parameters below | +| `supabase` | Supabase `auth.users` export | `bcrypt` | Supports `--skip-unsupported-providers` | + +### `clerk migrate transformers list` + +Which mappings are available. New in the CLI: the standalone tool's interactive +picker was the only place these appeared, which was fine when the user had the +source tree to grep. A compiled binary's users have neither. + +```sh +clerk migrate transformers list +clerk migrate transformers list --json +clerk migrate transformers list --transformer-file ./my-transformer.ts +``` + +| Flag | Description | +| --------------------------- | --------------------------------- | +| `--json` | Output as JSON | +| `--transformer-file ` | Also list a transformer you wrote | + +`--json` gives an agent the same data, including which source field each +transformer maps to `userId`. + +### Custom transformers (`--transformer-file`) + +Migrating from a platform with no built-in, without recompiling the CLI: + +```sh +clerk migrate run --transformer-file ./my-platform.ts --file users.json +``` + +The file lives in **your** project, not in the CLI, and is imported at runtime. +It exports the same shape the built-ins use — plain data, no imports, since +there is nothing in a compiled binary for your file to import from: + +```ts +export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform's admin console.", + transformer: { + account_ref: "userId", // required: becomes the Clerk user's external_id + contact_email: "email", + given: "firstName", + family: "lastName", + pw_bcrypt: "password", + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { + if (!user.firstName) delete user.firstName; + }, +}; +``` + +TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, +`satisfies` and `as const` all work in a file the compiled binary imports. +Plain `.js` works too. + +`--transformer-file` and `--transformer` together is an error: both name a +transformer and there is no sensible precedence between the one you wrote and +the one we ship. + +#### Validation + +The file is code the CLI executes, so its shape is checked before use and +rejected with the specific problem rather than crashing mid-pipeline: + +| Problem | Message | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------- | +| Path does not exist | `No transformer file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in transformer` | +| A hook is not a function | `postTransform must be a function when present` | + +The `userId` check is the load-bearing one: without it the import would run to +completion and create every user with no `external_id`, which is what makes a +migration re-runnable and what `migrate delete` matches on. + +### Verified vs unverified identifiers + +Every platform records verification differently, and each transformer declares +which style it uses. An identifier the source never confirmed is routed to +`unverifiedEmailAddresses` / `unverifiedPhoneNumbers` rather than the primary +field, because Clerk creates primary identifiers **verified** — sending an +unconfirmed address there would silently promote it. + +- **Boolean** (`auth0`, `betterauth`, `firebase`): `true`/`false`. A CSV export + stringifies these, so `"false"` is read as false, not as a non-empty string. +- **Timestamp** (`authjs`, `supabase`): a nullable confirmation time. Any real + value means verified; `""`, `null` and `\N` do not. + +### Firebase hash parameters + +Firebase uses a modified scrypt, so Clerk needs the project's four parameters +alongside each digest. Find them in the Firebase console under +**Authentication → Users → (⋮) → Password hash parameters**. + +```sh +clerk migrate run -y -t firebase -f users.json \ + --firebase-signer-key --firebase-salt-separator \ + --firebase-rounds 8 --firebase-mem-cost 14 +``` + +All four are **required as a set** — supplying some but not all is a usage error +naming what is missing. A partial set produces a well-formed digest that +verifies against nothing, so users would import successfully and then be unable +to sign in. They are saved to `.settings` and reused on the next run. + +An export with no password hashes needs no parameters at all. + +### `--skip-unsupported-providers` (Supabase) + +Reads each user's `raw_app_meta_data.providers` and cross-references it against +the social providers the destination instance has enabled (via BAPI +`/v1/domains` → the instance's Frontend API `/v1/environment`). + +A user is skipped **only when every one of their providers is disabled**. Anyone +who can still sign in another way — email, phone, or an enabled social provider +— is imported. The number skipped is reported, broken down by provider. + +If the instance configuration cannot be read, nobody is skipped and a warning is +printed: a failed lookup must not be mistaken for "no providers are enabled". + +## Schema fields + +What a transformer maps _onto_. Every user is validated against this schema +before any request is made, so a field a transformer produces that is not listed +here is silently dropped — Zod strips unknown keys — and never reaches Clerk. +Writing a custom transformer means targeting these names exactly. + +The schema lives in `validator.ts`; adding a source platform means adding a +transformer, not editing it. + +**Required:** `userId` (`string`). It becomes the Clerk user's `external_id`, +which is what makes a migration re-runnable and what `migrate delete` matches on. + +**Identifiers.** At least one of these must be present, or the user is logged as +a validation failure and skipped. Each accepts a single value or an array. + +| Field | Type | Description | +| -------------------------- | -------------------- | ---------------------------------- | +| `email` | `string \| string[]` | Primary verified email address(es) | +| `emailAddresses` | `string \| string[]` | Additional verified emails | +| `unverifiedEmailAddresses` | `string \| string[]` | Unverified emails | +| `phone` | `string \| string[]` | Primary verified phone number(s) | +| `phoneNumbers` | `string \| string[]` | Additional verified phones | +| `unverifiedPhoneNumbers` | `string \| string[]` | Unverified phones | +| `username` | `string` | Username | + +**Profile, password and 2FA.** + +| Field | Type | Description | +| -------------------- | ---------- | --------------------------------------------------- | +| `firstName` | `string` | First name | +| `lastName` | `string` | Last name | +| `password` | `string` | The hashed password from the source platform | +| `passwordHasher` | `enum` | **Required whenever `password` is set** (see below) | +| `totpSecret` | `string` | TOTP secret | +| `backupCodesEnabled` | `boolean` | Whether backup codes are enabled | +| `backupCodes` | `string[]` | Backup codes | + +Clerk verifies the digest as-is, so `passwordHasher` must name the algorithm the +source actually used: + +`argon2i`, `argon2id`, `awscognito`, `bcrypt`, `bcrypt_peppered`, +`bcrypt_sha256_django`, `hmac_sha256_utf16_b64`, `ldap_ssha`, `md5`, +`md5_phpass`, `md5_salted`, `pbkdf2_sha1`, `pbkdf2_sha256`, +`pbkdf2_sha256_django`, `pbkdf2_sha512`, `pbkdf2_sha512_hex`, `scrypt_firebase`, +`scrypt_werkzeug`, `sha256`, `sha256_salted`, `sha512_symfony` + +An unrecognized hasher aborts the run rather than importing credentials nobody +can sign in with. + +**Metadata.** + +| Field | Type | Description | +| ----------------- | -------- | -------------------------------------------------------- | +| `unsafeMetadata` | `object` | Readable **and writable** by the client — never trust it | +| `publicMetadata` | `object` | Readable by the client, writable only server-side | +| `privateMetadata` | `object` | Server-side only | + +**Account state.** These are passed straight through to `POST /v1/users`, and +are how a Clerk-to-Clerk migration keeps original signup dates instead of +stamping every user with today's. + +| Field | Type | Description | +| --------------------------- | --------- | ----------------------------------------- | +| `createdAt` | `string` | Original creation timestamp | +| `legalAcceptedAt` | `string` | When legal terms were accepted | +| `banned` | `boolean` | Whether the user is banned | +| `bypassClientTrust` | `boolean` | Skip client trust verification | +| `createOrganizationEnabled` | `boolean` | Whether the user can create orgs | +| `createOrganizationsLimit` | `number` | Maximum orgs the user can create | +| `deleteSelfEnabled` | `boolean` | Whether the user can delete their account | +| `skipLegalChecks` | `boolean` | Skip legal acceptance checks | +| `skipPasswordChecks` | `boolean` | Skip password requirements on import | + +## Migration Readiness report + +Printed immediately before the confirmation prompt, so declining aborts with +nothing written to Clerk. Skipped only for `-y`, which says "don't ask, don't +lecture" and should not pay for the two extra round-trips. Agent runs without +`-y` still get it — an agent can act on it exactly as a human would. + +It cross-references the file against the destination instance's live settings +(BAPI `/v1/domains` → that instance's Frontend API `/v1/environment`) and +flags the two failure modes a migration otherwise discovers halfway through: + +- **Required in Clerk, missing from the file.** Those users fail one at a time, + mid-import, after earlier users already exist. +- **Present in the file, disabled in Clerk.** Social providers users actually + signed up with, or an identifier the instance has switched off. + +``` +Migration readiness + 120 users ready to import + 3 failed validation and will be skipped + +Identifiers + ⚠ Email — required in Clerk, but 12 users lack it — 108/120 users + ✓ Username — enabled in Clerk — all users + +Social connections + ✓ Google — enabled in Clerk — 40/120 users + ⚠ Discord — not enabled in Clerk — 12/120 users + +⚠ 2 settings need attention +``` + +If the instance settings cannot be read — the secret key is rejected, or FAPI +is unreachable — the report degrades to a coverage-only listing with a note. +Nothing is flagged in that case: "could not read" is not the same as "switched +off", and treating it as such would raise alarms about settings that are +perfectly fine. + +## Artifacts + +Both are written relative to the **current working directory**, not to the +CLI's config directory, because they describe "which file am I migrating" +rather than "which project is linked here". + +| Path | Contents | +| -------------------------------------- | --------------------------------------------------------------------- | +| `./logs/migration-.log` | NDJSON: one line per user, plus validation failures and retry notices | +| `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | +| `./logs/export-.log` | NDJSON: one line per exported user | +| `./exports/-export.json` | The export itself, unless `--output` says otherwise | +| `./.settings` | The transformer key and file path of the last run | + +`.settings` is what `migrate delete` reads to know which migration to undo, so +it is load-bearing rather than a convenience. + +Log writes are synchronous appends, so a run interrupted with Ctrl-C still +leaves a complete record of everything already processed. Use the last +successful `userId` in that log with `--resume-after` to continue. + +### Why the logs are NDJSON + +One JSON object per line, rather than one JSON array per file. A migration is a +long append-only stream, and that format is the one that survives it: + +- **Appendable.** Each entry is written as it happens, without rewriting the + file. A JSON array would have to be re-serialized on every user. +- **Crash-safe.** Kill the process at any point and every line already written + is still valid. A truncated array is not parseable at all. +- **Streamable.** `tail -f` shows a long import progressing live, and analysis + reads line by line instead of loading a million-user log into memory. + +Which is also why it greps usefully without any tooling: + +```sh +grep '"status":"success"' logs/migration-2026-01-01T12-00-00.log | wc -l +grep '"userId":"user_123"' logs/migration-2026-01-01T12-00-00.log +``` + +The trade-off is that spreadsheets, databases and most JSON tooling want an +array. That is what `clerk migrate logs convert` is for — convert when you need +to open a log in Excel or hand it to someone who should not have to know what +NDJSON is. The original `.log` stays put. + +## API Endpoints + +| Method | Path | Used by | +| -------- | -------------------------- | ------------------------------------------------------------------------------------ | +| `POST` | `/v1/users` | `migrate run` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate run` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate run` — attaches additional phones | +| `GET` | `/v1/users?external_id=…` | `migrate delete` — finds this migration's users, 100 IDs a call | +| `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | +| `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | +| `GET` | `/v1/domains` | Readiness report and `--skip-unsupported-providers` — resolves the Frontend API host | + +Two exports talk to their own platform rather than to Clerk: + +| Method | Path | Used by | +| ------ | ---------------------------------------------- | ---------------------------------------------------- | +| `POST` | `https:///oauth/token` | `export auth0` — Management API access token | +| `GET` | `https:///api/v2/users` | `export auth0` — 100 per page, 1000 users maximum | +| `POST` | `https://oauth2.googleapis.com/token` | `export firebase` — RS256 assertion → access token | +| `GET` | `…/v1/projects/{project_id}/accounts:batchGet` | `export firebase` — pages users, 1000 at a time | +| `GET` | `…/admin/v2/projects/{project_id}/config` | `export firebase` — reads the scrypt hash parameters | + +The two Identity Toolkit paths are on `identitytoolkit.googleapis.com`, or on +`FIREBASE_AUTH_EMULATOR_HOST` when that is set. + +The three database exports (`supabase`, `authjs`, `betterauth`) make no HTTP +calls at all — they connect over `--db-url`. + +The readiness report and `--skip-unsupported-providers` additionally read the +instance's Frontend API `GET /v1/environment` for its attributes and enabled +social providers. + +## Notes + +- `userId` in the source file becomes the Clerk user's `external_id`. That is + what makes a migration re-runnable and reversible. +- CSV input is coerced before validation: `a@x.dev,b@x.dev` and `["a@x.dev"]` + both become arrays, `"true"`/`1` become booleans, and JSON metadata columns + are parsed. An empty column is dropped rather than sent as null. +- A user must end up with at least one identifier (email, phone or username). + Users that do not are logged as validation failures and skipped. diff --git a/packages/cli-core/src/commands/migrate/delete.test.ts b/packages/cli-core/src/commands/migrate/delete.test.ts new file mode 100644 index 000000000..5336e2312 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/delete.test.ts @@ -0,0 +1,422 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../lib/errors.ts"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { + batch, + deleteMigration, + deleteMigratedUsers, + findMigratedUsers, + readMigratedExternalIds, + resolveMigrationToUndo, +} from "./delete.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import { getLogDir } from "./lib/logger.ts"; +import { saveSettings } from "./lib/settings.ts"; + +const captured = useCaptureLog(); + +const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurrencyLimit: 8 }; +const DATE_TIME = "2026-01-01T00:00:00"; + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { method: string; url: string }[]; + +const EXPORT = [ + { id: "legacy_a", primary_email_address: "a@x.dev" }, + { id: "legacy_b", primary_email_address: "b@x.dev" }, +]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-delete-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + process.exitCode = 0; +}); + +/** + * Stubs BAPI: `GET /v1/users` answers with whichever of `present` the request + * asked for, mirroring how Clerk ignores external IDs it does not find. + */ +function stubBapi(present: Record, onDelete?: (id: string) => Response) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ method: init?.method ?? "GET", url }); + + if (url.includes("/v1/users?")) { + const asked = new URL(url).searchParams.getAll("external_id"); + return Response.json( + asked + .filter((externalId) => externalId in present) + .map((externalId) => ({ id: present[externalId], external_id: externalId })), + ); + } + + const match = /\/v1\/users\/([^/?]+)/.exec(url); + if (init?.method === "DELETE" && match) { + return onDelete ? onDelete(match[1] as string) : Response.json({ deleted: true }); + } + return Response.json({}); + }) as unknown as typeof fetch; +} + +const logEntries = () => + fs + .readdirSync(getLogDir()) + .flatMap((name) => fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n")) + .map((line) => JSON.parse(line) as Record); + +const deleteCalls = () => requests.filter((r) => r.method === "DELETE").map((r) => r.url); + +describe("resolveMigrationToUndo", () => { + test("reads the file and transformer from .settings", () => { + saveSettings({ key: "clerk", file: "export.json" }); + expect(resolveMigrationToUndo()).toEqual({ file: "export.json", key: "clerk" }); + }); + + // Deleting nothing silently would look like a successful undo. + test("explains when there is no .settings at all", () => { + expect(() => resolveMigrationToUndo()).toThrow(/no `.settings` from a previous/); + }); + + test.each([ + ["no file", { key: "clerk" }], + ["no transformer", { file: "export.json" }], + ])("explains when .settings has %s", (_label, settings) => { + saveSettings(settings); + expect(() => resolveMigrationToUndo()).toThrow(CliError); + }); + + test("explains when the migration file has since been removed", () => { + saveSettings({ key: "clerk", file: "gone.json" }); + expect(() => resolveMigrationToUndo()).toThrow(/no longer there/); + }); +}); + +describe("readMigratedExternalIds", () => { + test("returns the source IDs the import stamped as external_id", async () => { + expect(await readMigratedExternalIds("export.json", "clerk")).toEqual(["legacy_a", "legacy_b"]); + }); + + test("uses each transformer's own id field", async () => { + fs.writeFileSync( + path.join(workDir, "auth0.json"), + JSON.stringify([{ user_id: "auth0|1", email: "a@x.dev" }]), + ); + expect(await readMigratedExternalIds("auth0.json", "auth0")).toEqual(["auth0|1"]); + }); + + // Firebase's postTransform demands the project's hash parameters; deleting + // must not require them, so only the field mapping runs. + test("reads a firebase export without needing its password hash parameters", async () => { + fs.writeFileSync( + path.join(workDir, "firebase.json"), + JSON.stringify({ + users: [{ localId: "fb1", email: "a@x.dev", passwordHash: "H", salt: "S" }], + }), + ); + expect(await readMigratedExternalIds("firebase.json", "firebase")).toEqual(["fb1"]); + }); + + test("dedupes repeated IDs", async () => { + fs.writeFileSync( + path.join(workDir, "dupes.json"), + JSON.stringify([{ id: "legacy_a" }, { id: "legacy_a" }]), + ); + expect(await readMigratedExternalIds("dupes.json", "clerk")).toEqual(["legacy_a"]); + }); + + test("skips rows with no ID rather than matching on an empty string", async () => { + fs.writeFileSync( + path.join(workDir, "partial.json"), + JSON.stringify([{ id: "legacy_a" }, { primary_email_address: "b@x.dev" }, { id: "" }]), + ); + expect(await readMigratedExternalIds("partial.json", "clerk")).toEqual(["legacy_a"]); + }); +}); + +describe("batch", () => { + test.each([ + [0, 0], + [1, 1], + [100, 1], + [101, 2], + [250, 3], + ])("%i ids become %i request(s)", (count, expected) => { + const ids = Array.from({ length: count }, (_, i) => `u${i}`); + expect(batch(ids, 100)).toHaveLength(expected); + }); + + test("keeps every item, in order", () => { + expect(batch([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]); + }); +}); + +describe("findMigratedUsers", () => { + test("queries by external_id instead of listing the instance", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + const found = await findMigratedUsers({ + externalIds: ["legacy_a", "legacy_b"], + secretKey: "sk_test_x", + }); + + expect(found).toEqual([ + { id: "user_1", externalId: "legacy_a" }, + { id: "user_2", externalId: "legacy_b" }, + ]); + expect(requests).toHaveLength(1); + expect(requests[0]?.url).toContain("external_id=legacy_a"); + }); + + test("omits IDs the instance does not have", async () => { + stubBapi({ legacy_a: "user_1" }); + + const found = await findMigratedUsers({ + externalIds: ["legacy_a", "legacy_b"], + secretKey: "sk_test_x", + }); + + expect(found).toEqual([{ id: "user_1", externalId: "legacy_a" }]); + }); + + test("pages in batches of 100, the BAPI limit", async () => { + const ids = Array.from({ length: 150 }, (_, i) => `legacy_${i}`); + stubBapi(Object.fromEntries(ids.map((id, i) => [id, `user_${i}`]))); + + const found = await findMigratedUsers({ externalIds: ids, secretKey: "sk_test_x" }); + + expect(found).toHaveLength(150); + expect(requests).toHaveLength(2); + }); + + test("asks for a page large enough to hold the whole batch", async () => { + stubBapi({ legacy_a: "user_1" }); + await findMigratedUsers({ externalIds: ["legacy_a"], secretKey: "sk_test_x" }); + expect(requests[0]?.url).toContain("limit=100"); + }); + + // The guard that keeps this command from touching anything it did not create. + test("ignores a user whose external_id was not asked for", async () => { + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push({ method: "GET", url: input.toString() }); + return Response.json([ + { id: "user_1", external_id: "legacy_a" }, + { id: "user_999", external_id: "somebody_else" }, + { id: "user_888" }, + ]); + }) as unknown as typeof fetch; + + const found = await findMigratedUsers({ externalIds: ["legacy_a"], secretKey: "sk_test_x" }); + + expect(found).toEqual([{ id: "user_1", externalId: "legacy_a" }]); + }); +}); + +describe("deleteMigratedUsers", () => { + const users = [ + { id: "user_1", externalId: "legacy_a" }, + { id: "user_2", externalId: "legacy_b" }, + ]; + + test("deletes each user and logs the outcome", async () => { + stubBapi({}); + + const summary = await deleteMigratedUsers({ + users, + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ deleted: 2, failed: 0 }); + expect(deleteCalls()).toHaveLength(2); + expect(logEntries().filter((e) => e.status === "success")).toHaveLength(2); + }); + + test("records the source ID alongside the Clerk ID in the log", async () => { + stubBapi({}); + + await deleteMigratedUsers({ + users: [users[0] as (typeof users)[0]], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(logEntries()[0]).toMatchObject({ + userId: "legacy_a", + clerkUserId: "user_1", + status: "success", + }); + }); + + // A half-undone migration with no record of which half is worse than a + // reported failure. + test("keeps going after one user fails", async () => { + stubBapi({}, (id) => + id === "user_1" + ? new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { + status: 422, + }) + : Response.json({ deleted: true }), + ); + + const summary = await deleteMigratedUsers({ + users, + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ deleted: 1, failed: 1 }); + expect(deleteCalls()).toHaveLength(2); + expect(logEntries().some((e) => e.status === "error" && e.code === "422")).toBe(true); + }); + + test("retries a 429 and logs the attempt", async () => { + const attempts = new Map(); + stubBapi({}, (id) => { + const attempt = (attempts.get(id) ?? 0) + 1; + attempts.set(id, attempt); + return attempt === 1 + ? new Response(JSON.stringify({ errors: [{ code: "e", message: "slow down" }] }), { + status: 429, + headers: { "retry-after": "1" }, + }) + : Response.json({ deleted: true }); + }); + + const summary = await deleteMigratedUsers({ + users: [users[0] as (typeof users)[0]], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ deleted: 1, failed: 0 }); + expect(deleteCalls()).toHaveLength(2); + expect(logEntries().some((e) => e.status === "429_retry")).toBe(true); + }); + + test("groups identical failures in the breakdown", async () => { + stubBapi( + {}, + () => + new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { + status: 422, + }), + ); + + const summary = await deleteMigratedUsers({ + users, + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect([...summary.errorBreakdown.values()]).toEqual([2]); + }); +}); + +describe("deleteMigration", () => { + const baseOptions = { yes: true, secretKey: "sk_test_x" }; + + beforeEach(() => { + saveSettings({ key: "clerk", file: "export.json" }); + }); + + test("deletes the users the last run created", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + await deleteMigration(baseOptions); + + expect(deleteCalls()).toEqual([ + expect.stringContaining("/v1/users/user_1"), + expect.stringContaining("/v1/users/user_2"), + ]); + expect(captured.err).toContain("Deleted:"); + }); + + test("writes a timestamped NDJSON deletion log", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + await deleteMigration(baseOptions); + + const logs = fs.readdirSync(getLogDir()); + expect(logs).toHaveLength(1); + expect(logs[0]).toMatch(/^user-deletion-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + }); + + test("leaves users the migration did not create alone", async () => { + stubBapi({ legacy_a: "user_1" }); + + await deleteMigration(baseOptions); + + expect(deleteCalls()).toEqual([expect.stringContaining("/v1/users/user_1")]); + expect(captured.err).toContain("1 of the file's user(s) are not in this instance"); + }); + + test("does nothing when none of the migration's users are present", async () => { + stubBapi({}); + + await deleteMigration(baseOptions); + + expect(deleteCalls()).toHaveLength(0); + expect(captured.err).toContain("Nothing to delete"); + }); + + // Tests run non-TTY, which is the same signal an agent gives. + test("refuses without -y when it cannot prompt, and says how many are at stake", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + await expect(deleteMigration({ secretKey: "sk_test_x" })).rejects.toThrow( + /permanently deletes 2 user\(s\) and cannot prompt here/, + ); + expect(deleteCalls()).toHaveLength(0); + }); + + test("fails before any API call when there is no .settings", async () => { + fs.rmSync(path.join(workDir, ".settings"), { force: true }); + stubBapi({ legacy_a: "user_1" }); + + await expect(deleteMigration(baseOptions)).rejects.toThrow(CliError); + expect(requests).toHaveLength(0); + }); + + test("exits non-zero when a deletion failed", async () => { + stubBapi( + { legacy_a: "user_1" }, + () => + new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { + status: 422, + }), + ); + + await deleteMigration(baseOptions); + + expect(process.exitCode).toBe(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts new file mode 100644 index 000000000..5cc78dbef --- /dev/null +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -0,0 +1,336 @@ +/** + * `clerk migrate delete` — undo a migration. + * + * Ported from the standalone migration-tool's `src/delete/index.ts`, with two + * substantive changes: + * + * - **Users are looked up by `external_id`, not by downloading the instance.** + * The original paged through every user in the instance 500 at a time and + * intersected client-side, which on a large instance means fetching hundreds + * of thousands of users to delete a few hundred. `GET /v1/users` filters on + * up to 100 `external_id`s per call and ignores IDs it does not find, so the + * work is proportional to the migration rather than to the instance. + * - **IDs come from the existing transform pipeline.** The original + * re-implemented per-format ID extraction with its own Firebase CSV header + * list and a chain of `userId`/`user_id`/`localId`/`id` fallbacks. The + * transformer already declares which source field becomes `userId`. + * + * This is the one command in the `migrate` tree that destroys data in Clerk, so + * it stays flat and prominent rather than buried under a noun group, and it + * confirms before acting. + */ + +import { bapiRequest } from "../../lib/bapi.ts"; +import { bold, dim, green, red } from "../../lib/color.ts"; +import { + BapiError, + CliError, + ERROR_CODE, + throwUsageError, + throwUserAbort, +} from "../../lib/errors.ts"; +import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; +import { log } from "../../lib/log.ts"; +import { confirm } from "../../lib/prompts.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../mode.ts"; +import { normalizeErrorMessage } from "./import-users.ts"; +import { resolveLimits, type ResolvedLimits } from "./lib/instance.ts"; +import { deleteErrorLogger, deleteLogger, getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; +import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; +import { createApiScheduler } from "./lib/scheduler.ts"; +import { loadSettings } from "./lib/settings.ts"; +import { fileExists, readRawUsers, transformKeys } from "./lib/transform.ts"; +import { getTransformer } from "./transformers/registry.ts"; + +/** BAPI accepts at most 100 `external_id` values per `GET /v1/users` call. */ +const EXTERNAL_ID_BATCH = 100; + +export type MigrateDeleteOptions = { + yes?: boolean; + secretKey?: string; + clerkSecretKey?: string; + app?: string; + instance?: string; +}; + +export type MigratedUser = { + /** The Clerk user ID to delete. */ + id: string; + /** The source platform's ID, stamped on the user as `external_id`. */ + externalId: string; +}; + +/** + * Resolves which migration is being undone. + * + * `.settings` is the only record of that — this command has no independent way + * to know what a previous run created, which is why it is coupled to `run`. + */ +export function resolveMigrationToUndo(): { file: string; key: string } { + const settings = loadSettings(); + + if (!settings.file || !settings.key) { + throw new CliError( + "No migration to undo: this directory has no `.settings` from a previous `clerk migrate run`.\n" + + "Run `clerk migrate delete` from the directory you migrated from.", + { code: ERROR_CODE.FILE_NOT_FOUND }, + ); + } + + if (!fileExists(settings.file)) { + throw new CliError( + `The migration file ${settings.file} named in .settings is no longer there, so the users it created cannot be identified.`, + { code: ERROR_CODE.FILE_NOT_FOUND }, + ); + } + + return { file: settings.file, key: settings.key }; +} + +/** + * The source IDs a migration stamped onto Clerk users as `external_id`. + * + * Runs the transformer's field mapping but not its `postTransform`: only the + * ID matters here, and Firebase's post-transform would demand the project's + * password hash parameters to rebuild digests nobody is importing. + */ +export async function readMigratedExternalIds(file: string, key: string): Promise { + const transformer = getTransformer(key); + const rows = await readRawUsers(file, key); + + const ids = new Set(); + for (const row of rows) { + const userId = transformKeys(row, transformer).userId; + if (typeof userId === "string" && userId.length > 0) ids.add(userId); + } + return [...ids]; +} + +/** Splits `items` into chunks of at most `size`. */ +export function batch(items: T[], size: number): T[][] { + const batches: T[][] = []; + for (let i = 0; i < items.length; i += size) batches.push(items.slice(i, i + size)); + return batches; +} + +/** + * Finds the Clerk users a migration created, by `external_id`. + * + * IDs with no matching user are simply absent from the result — a partial + * migration, or one already partly undone, is the normal case. + */ +export async function findMigratedUsers(options: { + externalIds: string[]; + secretKey: string; + spinner?: SpinnerControls; +}): Promise { + const found: MigratedUser[] = []; + const batches = batch(options.externalIds, EXTERNAL_ID_BATCH); + + for (const [index, ids] of batches.entries()) { + options.spinner?.update(`Finding migrated users: batch ${index + 1}/${batches.length}`); + + const params = new URLSearchParams(); + params.set("limit", String(EXTERNAL_ID_BATCH)); + for (const id of ids) params.append("external_id", id); + + const response = await retryOn429(() => + bapiRequest({ + method: "GET", + path: `/v1/users?${params.toString()}`, + secretKey: options.secretKey, + }), + ); + + const users = (response.body ?? []) as { id?: string; external_id?: string }[]; + for (const user of Array.isArray(users) ? users : []) { + // Never delete on a partial match: only a user Clerk itself reports as + // carrying one of this migration's external IDs is in scope. + if (user.id && user.external_id && ids.includes(user.external_id)) { + found.push({ id: user.id, externalId: user.external_id }); + } + } + } + + return found; +} + +export type DeleteSummary = { + deleted: number; + failed: number; + errorBreakdown: Map; +}; + +/** Deletes each user, rate-limited and 429-retried exactly as the import is. */ +export async function deleteMigratedUsers(options: { + users: MigratedUser[]; + secretKey: string; + limits: ResolvedLimits; + dateTime: string; + spinner?: SpinnerControls; +}): Promise { + const { users, secretKey, limits, dateTime, spinner } = options; + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + const errorBreakdown = new Map(); + + let processed = 0; + let deleted = 0; + let failed = 0; + + const progress = () => + spinner?.update( + `Deleting users: [${processed}/${users.length}] (${deleted} deleted, ${failed} failed)`, + ); + + // A failure on one user must not abort the rest: a half-undone migration + // with no record of which half is far worse than a reported failure. + const recordFailure = (user: MigratedUser, message: string, code: string) => { + failed++; + processed++; + const normalized = normalizeErrorMessage(message); + errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); + deleteLogger( + { userId: user.externalId, clerkUserId: user.id, status: "error", error: message, code }, + dateTime, + ); + progress(); + }; + + const deleteOne = async (user: MigratedUser): Promise => { + try { + await retryOn429( + () => + schedule(() => + bapiRequest({ method: "DELETE", path: `/v1/users/${user.id}`, secretKey }), + ), + { + onRetry: ({ message }) => + deleteErrorLogger( + { + userId: user.externalId, + status: "429_retry", + errors: [{ code: "rate_limit_retry", message, longMessage: message }], + }, + dateTime, + ), + }, + ); + + deleted++; + processed++; + deleteLogger({ userId: user.externalId, clerkUserId: user.id, status: "success" }, dateTime); + progress(); + } catch (error) { + if (error instanceof RateLimitExceededError) { + recordFailure(user, error.message, "429"); + return; + } + const apiError = error as BapiError; + const message = apiError.longMessage ?? apiError.message ?? "Unknown error"; + recordFailure(user, message, String(apiError.status ?? "unknown")); + } + }; + + progress(); + await Promise.all(users.map(deleteOne)); + + return { deleted, failed, errorBreakdown }; +} + +function formatSummary(summary: DeleteSummary, logFile: string): string { + const lines = [ + `${bold("Deleted:")} ${green(String(summary.deleted))}`, + `${bold("Failed:")} ${red(String(summary.failed))}`, + ]; + + if (summary.errorBreakdown.size > 0) { + lines.push("", bold("Error breakdown:")); + for (const [error, count] of summary.errorBreakdown) { + lines.push(` ${count} user${count === 1 ? "" : "s"}: ${error}`); + } + } + lines.push("", dim(`Log: ${logFile}`)); + + return lines.join("\n"); +} + +export async function deleteMigration(options: MigrateDeleteOptions): Promise { + if (options.clerkSecretKey) { + log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); + } + const secretKeyOption = options.secretKey ?? options.clerkSecretKey; + + const { file, key } = resolveMigrationToUndo(); + + await withGutter("Undoing a migration", async () => { + const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const limits = resolveLimits(secretKey); + const dateTime = getDateTimeStamp(); + const logFile = getLogFilePath("user-deletion", dateTime); + + const externalIds = await readMigratedExternalIds(file, key); + if (externalIds.length === 0) { + log.warn(`No user IDs found in ${file}; nothing to undo.`); + return; + } + + const users = await withSpinner( + "Finding migrated users", + (spinner) => findMigratedUsers({ externalIds, secretKey, spinner }), + "Search complete", + ); + + if (users.length === 0) { + log.info( + `None of the ${externalIds.length} user(s) in ${file} are in ${target ?? "this instance"}. Nothing to delete.`, + ); + return; + } + + log.warn( + `About to delete ${users.length} user${users.length === 1 ? "" : "s"} from ` + + `${target ?? "the resolved instance"}, matched to ${file} by external ID.`, + ); + if (users.length < externalIds.length) { + log.info( + dim( + `${externalIds.length - users.length} of the file's user(s) are not in this instance and will be left alone.`, + ), + ); + } + + if (!options.yes) { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate delete\` permanently deletes ${users.length} user(s) and cannot prompt here. Pass -y to confirm.`, + undefined, + undefined, + [ + { + command: "clerk migrate delete -y", + description: "Delete the migrated users without prompting", + }, + ], + ); + } + + const proceed = await confirm({ + message: `Permanently delete ${users.length} user${users.length === 1 ? "" : "s"}?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + const summary = await withSpinner( + `Deleting users: [0/${users.length}]`, + (spinner) => deleteMigratedUsers({ users, secretKey, limits, dateTime, spinner }), + "Deletion complete", + ); + + log.raw(formatSummary(summary, logFile)); + + if (summary.failed > 0) process.exitCode = 1; + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts new file mode 100644 index 000000000..b48b89819 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -0,0 +1,308 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildAuth0Export, + exportAuth0, + fetchAllAuth0Users, + fetchAuth0Token, + mapAuth0UserToExport, + normalizeAuth0Domain, + resolveAuth0Credentials, +} from "./auth0.ts"; + +const captured = useCaptureLog(); + +const CREDENTIALS = { domain: "t.auth0.com", clientId: "cid", clientSecret: "csec" }; + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { url: string; body: unknown }[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expauth0-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +const auth0User = (i: number, overrides: Record = {}) => ({ + user_id: `auth0|a${i}`, + email: `a${i}@x.dev`, + email_verified: true, + given_name: `Given${i}`, + family_name: `Family${i}`, + ...overrides, +}); + +/** Stubs the token exchange plus one page of users per entry in `pages`. */ +function stubAuth0(pages: Record[][], token: Response | null = null) { + let page = 0; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ url, body: init?.body ? JSON.parse(init.body as string) : null }); + + if (url.includes("/oauth/token")) { + return token ?? Response.json({ access_token: "tok" }); + } + return Response.json({ + users: pages[page++] ?? [], + total: pages.reduce((sum, p) => sum + p.length, 0), + }); + }) as unknown as typeof fetch; +} + +describe("normalizeAuth0Domain", () => { + test.each([ + ["t.auth0.com", "t.auth0.com"], + ["https://t.auth0.com", "t.auth0.com"], + ["http://t.auth0.com/", "t.auth0.com"], + [" t.auth0.com ", "t.auth0.com"], + ])("%s -> %s", (input, expected) => { + expect(normalizeAuth0Domain(input)).toBe(expected); + }); +}); + +describe("resolveAuth0Credentials", () => { + test("prefers flags", async () => { + const resolved = await resolveAuth0Credentials( + { domain: "flag.auth0.com", clientId: "f", clientSecret: "s" }, + { AUTH0_DOMAIN: "env.auth0.com" }, + ); + expect(resolved.domain).toBe("flag.auth0.com"); + }); + + test("falls back to the environment", async () => { + const resolved = await resolveAuth0Credentials( + {}, + { AUTH0_DOMAIN: "env.auth0.com", AUTH0_CLIENT_ID: "e", AUTH0_CLIENT_SECRET: "s" }, + ); + expect(resolved).toEqual({ domain: "env.auth0.com", clientId: "e", clientSecret: "s" }); + }); + + test("normalizes a domain that came with a scheme", async () => { + const resolved = await resolveAuth0Credentials( + { domain: "https://t.auth0.com/", clientId: "c", clientSecret: "s" }, + {}, + ); + expect(resolved.domain).toBe("t.auth0.com"); + }); + + // Tests run non-TTY, the same signal an agent gives. + test("names every missing credential at once rather than one at a time", async () => { + await expect(resolveAuth0Credentials({}, {})).rejects.toThrow( + /--domain \(or AUTH0_DOMAIN\), --client-id \(or AUTH0_CLIENT_ID\), --client-secret \(or AUTH0_CLIENT_SECRET\)/, + ); + }); + + test("names only what is actually missing", async () => { + await expect( + resolveAuth0Credentials({ domain: "t.auth0.com", clientId: "c" }, {}), + ).rejects.toThrow(/Missing: --client-secret \(or AUTH0_CLIENT_SECRET\)\./); + }); +}); + +describe("fetchAuth0Token", () => { + test("exchanges client credentials for the Management API audience", async () => { + stubAuth0([[]]); + + expect(await fetchAuth0Token(CREDENTIALS)).toBe("tok"); + expect(requests[0]?.url).toBe("https://t.auth0.com/oauth/token"); + expect(requests[0]?.body).toEqual({ + grant_type: "client_credentials", + client_id: "cid", + client_secret: "csec", + audience: "https://t.auth0.com/api/v2/", + }); + }); + + test("explains a rejection instead of surfacing a raw status", async () => { + stubAuth0( + [[]], + new Response(JSON.stringify({ error_description: "Wrong client secret" }), { status: 401 }), + ); + + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow( + /Auth0 rejected the credentials \(401\): Wrong client secret/, + ); + }); + + test("mentions the read:users scope, the usual cause", async () => { + stubAuth0([[]], new Response("{}", { status: 403 })); + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(/read:users/); + }); + + test("fails when a 200 carries no token", async () => { + stubAuth0([[]], Response.json({})); + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(CliError); + }); +}); + +describe("fetchAllAuth0Users", () => { + test("pages until a short page arrives", async () => { + stubAuth0([ + Array.from({ length: 100 }, (_, i) => auth0User(i)), + Array.from({ length: 4 }, (_, i) => auth0User(100 + i)), + ]); + + const all = await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + + expect(all).toHaveLength(104); + expect(requests[0]?.url).toContain("page=0"); + expect(requests[1]?.url).toContain("page=1"); + expect(requests).toHaveLength(2); + }); + + test("asks for totals and the documented page size", async () => { + stubAuth0([[]]); + await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + expect(requests[0]?.url).toContain("per_page=100"); + expect(requests[0]?.url).toContain("include_totals=true"); + }); + + // Auth0 caps offset pagination at 1000. Returning the first thousand quietly + // would read as "that is everyone". + test("stops at Auth0's 1000-record ceiling and says so", async () => { + stubAuth0( + Array.from({ length: 12 }, () => Array.from({ length: 100 }, (_, i) => auth0User(i))), + ); + + const all = await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + + expect(all).toHaveLength(1000); + expect(captured.err).toContain("only pages through the first 1000 users"); + expect(captured.err).toContain("bulk user export job"); + }); + + test("raises a clear error on a failed page request", async () => { + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + + await expect(fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" })).rejects.toThrow( + /Auth0 returned 500 listing users/, + ); + }); +}); + +describe("mapAuth0UserToExport", () => { + test("keeps the fields the auth0 transformer maps from", () => { + expect( + mapAuth0UserToExport(auth0User(0, { phone_number: "+1555", created_at: "2025-01-01" })), + ).toEqual({ + user_id: "auth0|a0", + email: "a0@x.dev", + given_name: "Given0", + family_name: "Family0", + phone_number: "+1555", + created_at: "2025-01-01", + email_verified: true, + }); + }); + + // Dropping a false flag would import an unconfirmed address as verified. + test.each([ + ["email_verified", false], + ["phone_verified", false], + ])("keeps %s when it is %p", (field, value) => { + const mapped = mapAuth0UserToExport(auth0User(0, { [field]: value })); + expect(mapped[field]).toBe(value); + }); + + test("drops tenant internals the import has no use for", () => { + const mapped = mapAuth0UserToExport( + auth0User(0, { + identities: [{ provider: "auth0" }], + logins_count: 42, + last_login: "2026-01-01", + multifactor: ["guardian"], + }), + ); + for (const noise of ["identities", "logins_count", "last_login", "multifactor"]) { + expect(noise in mapped).toBe(false); + } + }); + + test("omits empty metadata", () => { + const mapped = mapAuth0UserToExport( + auth0User(0, { user_metadata: {}, app_metadata: { plan: "pro" } }), + ); + expect("user_metadata" in mapped).toBe(false); + expect(mapped.app_metadata).toEqual({ plan: "pro" }); + }); +}); + +describe("buildAuth0Export", () => { + test("counts coverage and logs each user", () => { + const { users, coverage } = buildAuth0Export( + [auth0User(0), auth0User(1, { given_name: undefined })], + "2026-01-01T00:00:00", + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + + const logged = fs.readdirSync(getLogDir()); + expect(logged[0]).toMatch(/^export-/); + }); +}); + +describe("exportAuth0", () => { + test("writes the default path and reports coverage", async () => { + stubAuth0([[auth0User(0)], []]); + + await exportAuth0({ ...CREDENTIALS }); + + const written = JSON.parse( + fs.readFileSync(path.join(workDir, "exports", "auth0-export.json"), "utf-8"), + ) as Record[]; + expect(written[0]?.user_id).toBe("auth0|a0"); + expect(captured.err).toContain("Field coverage"); + }); + + test("names the command that consumes the file", async () => { + stubAuth0([[auth0User(0)], []]); + await exportAuth0({ ...CREDENTIALS }); + expect(captured.err).toContain( + "migrate run --transformer auth0 --file exports/auth0-export.json", + ); + }); + + test("--output controls the destination", async () => { + stubAuth0([[auth0User(0)], []]); + + await exportAuth0({ ...CREDENTIALS, output: "tenant.json" }); + + expect(fs.existsSync(path.join(workDir, "tenant.json"))).toBe(true); + }); + + // Auth0 only releases hashes through a support request; finding that out + // after the import means nobody can sign in. + test("says plainly that password hashes are not in the file", async () => { + stubAuth0([[auth0User(0)], []]); + await exportAuth0({ ...CREDENTIALS }); + expect(captured.err).toContain("does not return password hashes"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts new file mode 100644 index 000000000..0e94933b5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -0,0 +1,346 @@ +/** + * `clerk migrate export auth0` — pull users out of an Auth0 tenant. + * + * Ported from the standalone migration-tool's `src/export/auth0.ts`, but + * **without the `auth0` SDK**. The SDK is 28 MB across five transitive + * dependencies — including a bundled legacy copy of itself — to make two REST + * calls, and it does its own HTTP, so nothing it sends would appear under + * `--verbose`. `.claude/rules/debug-logging.md` requires library HTTP to go + * through `loggedFetch`; two direct calls satisfy that and ship nothing extra + * inside the compiled binary. + * + * **Passwords do not come out of the Management API.** Auth0 exports password + * hashes only via a support request. The coverage report says so rather than + * leaving it to be discovered when nobody can sign in. + */ + +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt, text } from "../../../lib/prompts.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; + +const PAGE_SIZE = 100; + +/** + * Auth0 caps offset pagination on `GET /api/v2/users` at 1000 records. + * Past that the tenant needs a bulk export job, so the run says so instead of + * quietly returning the first thousand as though that were everyone. + */ +const AUTH0_PAGINATION_CEILING = 1000; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/auth0"; + +export type ExportAuth0Options = { + domain?: string; + clientId?: string; + clientSecret?: string; + output?: string; +}; + +export type Auth0Credentials = { + domain: string; + clientId: string; + clientSecret: string; +}; + +/** Strips a scheme and trailing slash, so both forms of `--domain` work. */ +export function normalizeAuth0Domain(domain: string): string { + return domain + .trim() + .replace(/^https?:\/\//, "") + .replace(/\/+$/, ""); +} + +/** + * Resolves the tenant credentials: flags, then environment, then a prompt. + * + * @throws CliError in agent mode when anything is still missing, naming each + * absent flag rather than failing on the first one. + */ +export async function resolveAuth0Credentials( + options: ExportAuth0Options, + env: Record = process.env, +): Promise { + const resolved = { + domain: options.domain ?? env.AUTH0_DOMAIN, + clientId: options.clientId ?? env.AUTH0_CLIENT_ID, + clientSecret: options.clientSecret ?? env.AUTH0_CLIENT_SECRET, + }; + + const missing = ( + [ + ["domain", "--domain", "AUTH0_DOMAIN"], + ["clientId", "--client-id", "AUTH0_CLIENT_ID"], + ["clientSecret", "--client-secret", "AUTH0_CLIENT_SECRET"], + ] as const + ).filter(([key]) => !resolved[key]); + + if (missing.length === 0) { + return { + domain: normalizeAuth0Domain(resolved.domain as string), + clientId: resolved.clientId as string, + clientSecret: resolved.clientSecret as string, + }; + } + + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export auth0\` needs credentials for a machine-to-machine application and cannot prompt here.\n` + + `Missing: ${missing.map(([, flag, variable]) => `${flag} (or ${variable})`).join(", ")}.`, + DOCS_URL, + undefined, + [ + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export with explicit credentials", + }, + ], + ); + } + + log.info( + "Auth0 needs a machine-to-machine application with the `read:users` scope. Create one under Applications → APIs → Auth0 Management API → Machine to Machine Applications.", + ); + + const domain = + resolved.domain ?? + (await text({ + message: "Auth0 tenant domain (e.g. my-tenant.us.auth0.com)", + validate: (value) => (value?.trim() ? undefined : "A domain is required"), + })); + const clientId = + resolved.clientId ?? + (await text({ + message: "Machine-to-machine client ID", + validate: (value) => (value?.trim() ? undefined : "A client ID is required"), + })); + const clientSecret = + resolved.clientSecret ?? + (await passwordPrompt({ + message: "Machine-to-machine client secret", + validate: (value) => (value?.trim() ? undefined : "A client secret is required"), + })); + + return { + domain: normalizeAuth0Domain(domain), + clientId: clientId.trim(), + clientSecret: clientSecret.trim(), + }; +} + +/** Exchanges the client credentials for a Management API access token. */ +export async function fetchAuth0Token(credentials: Auth0Credentials): Promise { + const url = new URL(`https://${credentials.domain}/oauth/token`); + + const response = await loggedFetch(url, { + tag: "auth0", + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + grant_type: "client_credentials", + client_id: credentials.clientId, + client_secret: credentials.clientSecret, + audience: `https://${credentials.domain}/api/v2/`, + }), + }); + + const body = (await response.json().catch(() => ({}))) as { + access_token?: string; + error_description?: string; + error?: string; + }; + + if (!response.ok || !body.access_token) { + throw new CliError( + `Auth0 rejected the credentials (${response.status}): ${body.error_description ?? body.error ?? "no access token returned"}\n` + + "Check the domain, client ID and secret, and that the application is authorized for the Management API with the `read:users` scope.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + return body.access_token; +} + +type Auth0User = Record & { user_id?: string }; + +/** Fetches one page of users from the Management API. */ +async function fetchAuth0Page( + credentials: Auth0Credentials, + token: string, + page: number, +): Promise<{ users: Auth0User[]; total: number }> { + const url = new URL(`https://${credentials.domain}/api/v2/users`); + url.searchParams.set("page", String(page)); + url.searchParams.set("per_page", String(PAGE_SIZE)); + url.searchParams.set("include_totals", "true"); + + const response = await loggedFetch(url, { + tag: "auth0", + method: "GET", + headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, + }); + + if (!response.ok) { + const body = await response.text(); + throw new CliError(`Auth0 returned ${response.status} listing users: ${body}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + } + + const body = (await response.json()) as { users?: Auth0User[]; total?: number }; + return { users: body.users ?? [], total: body.total ?? 0 }; +} + +/** + * Pages through the tenant's users. + * + * Stops at Auth0's 1000-record ceiling with a warning naming the bulk export + * job — silently truncating would read as "that is everyone". + */ +export async function fetchAllAuth0Users(options: { + credentials: Auth0Credentials; + token: string; + spinner?: SpinnerControls; +}): Promise { + const all: Auth0User[] = []; + + for (let page = 0; ; page++) { + const { users, total } = await fetchAuth0Page(options.credentials, options.token, page); + all.push(...users); + options.spinner?.update(`Fetching users from Auth0: ${all.length} so far`); + + if (users.length < PAGE_SIZE) break; + + if (all.length >= AUTH0_PAGINATION_CEILING) { + log.warn( + `Auth0 only pages through the first ${AUTH0_PAGINATION_CEILING} users on this endpoint` + + (total > AUTH0_PAGINATION_CEILING ? `, and this tenant reports ${total}` : "") + + ". Exported what is reachable; use Auth0's bulk user export job for the rest.", + ); + break; + } + } + + return all; +} + +/** + * Keeps the fields the `auth0` transformer maps from. + * + * Deliberately a copy rather than the raw record: an Auth0 user carries + * identities, session counts and tenant internals that would bloat the export + * and mean nothing to the import. + */ +export function mapAuth0UserToExport(user: Auth0User): Record { + const exported: Record = {}; + + for (const field of [ + "user_id", + "email", + "username", + "given_name", + "family_name", + "phone_number", + "created_at", + ] as const) { + if (user[field]) exported[field] = user[field]; + } + + // Verification flags are meaningful when false, so they are copied on + // presence rather than on truthiness. + for (const field of ["email_verified", "phone_verified"] as const) { + if (user[field] !== undefined) exported[field] = user[field]; + } + + for (const field of ["user_metadata", "app_metadata"] as const) { + const value = user[field]; + if (value && typeof value === "object" && Object.keys(value).length > 0) { + exported[field] = value; + } + } + + return exported; +} + +export type Auth0ExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0ExportResult { + const exported: Record[] = []; + const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0 }; + + for (const user of users) { + const userId = String(user.user_id ?? ""); + try { + const mapped = mapAuth0UserToExport(user); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.username) counts.username++; + if (mapped.given_name) counts.firstName++; + if (mapped.family_name) counts.lastName++; + if (mapped.phone_number) counts.phone++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a phone number", count: counts.phone }, + { label: "have a username", count: counts.username }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + ], + }; +} + +export async function exportAuth0(options: ExportAuth0Options): Promise { + const credentials = await resolveAuth0Credentials(options); + + await withGutter("Exporting users from Auth0", async () => { + const dateTime = getDateTimeStamp(); + log.info(`Exporting from ${credentials.domain}.`); + + const token = await withSpinner("Authenticating with Auth0", () => + fetchAuth0Token(credentials), + ); + + const users = await withSpinner( + "Fetching users from Auth0", + (spinner) => fetchAllAuth0Users({ credentials, token, spinner }), + "Users fetched", + ); + + const { users: exported, coverage } = buildAuth0Export(users, dateTime); + const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("auth0")); + + reportExport({ + platform: "auth0", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "auth0", + }); + + if (exported.length > 0) { + log.warn( + "Auth0's Management API does not return password hashes. Request a password hash export from Auth0 support and add a `passwordHash` field to each user before importing, or migrate without passwords.", + ); + log.info(dim(`See ${DOCS_URL}`)); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts new file mode 100644 index 000000000..d8183d981 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -0,0 +1,141 @@ +/** + * `clerk migrate export authjs` — read users out of an Auth.js database. + * + * Ported from the standalone migration-tool's `src/export/authjs.ts`, on the + * `Bun.sql`/`bun:sqlite` client. + * + * Auth.js has no export tool and no single schema: the adapter decides the + * table name, and Prisma's `User` differs from Drizzle's `user` only in + * casing — which Postgres and SQLite treat as significant once quoted. The + * export tries the documented casing first and falls back rather than making + * the user find out from a driver error. + */ + +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import { log } from "../../../lib/log.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; + +/** Table names to try, in order. Prisma capitalizes; Drizzle does not. */ +const TABLE_CANDIDATES = ["User", "user", "users"] as const; + +type AuthJsRow = Record & { + id?: unknown; + name?: string | null; + email?: string | null; + email_verified?: unknown; +}; + +export function buildAuthJsQuery(client: DbClient, table: string): string { + const q = (identifier: string) => client.quote(identifier); + return ( + `SELECT ${q("id")}, ${q("name")}, ${q("email")}, ${q("emailVerified")} AS ${q("email_verified")} ` + + `FROM ${q(table)} ORDER BY ${q("id")} ASC` + ); +} + +/** True for an error that means "wrong table name", not "broken connection". */ +function isMissingTable(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /does not exist|no such table|doesn't exist|unknown table/i.test(message); +} + +/** + * Reads the user table, trying each casing until one answers. + * + * @returns The rows and the table they came from, so the run can say which. + */ +export async function fetchAuthJsUsers( + client: DbClient, +): Promise<{ rows: AuthJsRow[]; table: string }> { + let lastError: unknown; + + for (const table of TABLE_CANDIDATES) { + try { + return { rows: await client.query(buildAuthJsQuery(client, table)), table }; + } catch (error) { + if (!isMissingTable(error)) throw error; + lastError = error; + } + } + + throw lastError instanceof Error + ? new Error( + `No Auth.js user table found. Tried ${TABLE_CANDIDATES.join(", ")}. ${lastError.message}`, + ) + : new Error(`No Auth.js user table found. Tried ${TABLE_CANDIDATES.join(", ")}.`); +} + +export function buildAuthJsExport(rows: AuthJsRow[], dateTime: string) { + const users: Record[] = []; + const counts = { email: 0, emailVerified: 0, name: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + const user: Record = { id: userId }; + + if (row.name) { + user.name = row.name; + counts.name++; + } + if (row.email) { + user.email = row.email; + counts.email++; + } + // A nullable timestamp, not a boolean: the transformer reads presence. + if (row.email_verified) { + user.email_verified = + row.email_verified instanceof Date ? row.email_verified.toISOString() : row.email_verified; + counts.emailVerified++; + } + + users.push(user); + exportLogger({ userId, status: "success" }, dateTime); + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.emailVerified }, + { label: "have a name", count: counts.name }, + ], + }; +} + +export async function exportAuthJs(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", + hint: "Postgres, MySQL or a SQLite file — whichever your Auth.js adapter uses.", + }); + + await withGutter("Exporting users from Auth.js", async () => { + const dateTime = getDateTimeStamp(); + + const { rows, table } = await withSpinner("Reading the user table", () => + withDbClient(dbUrl, "authjs", fetchAuthJsUsers), + ); + log.info(`Read ${rows.length} row(s) from ${table}.`); + + const { users, coverage } = buildAuthJsExport(rows, dateTime); + const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("authjs")); + + reportExport({ + platform: "authjs", + userCount: users.length, + outputPath, + coverage, + transformerKey: "authjs", + }); + + if (users.length > 0) { + log.warn( + "Auth.js core stores no passwords — its users sign in with OAuth or email links, so they arrive without credentials and will use the same providers in Clerk.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts new file mode 100644 index 000000000..e2dcad93e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -0,0 +1,191 @@ +/** + * `clerk migrate export betterauth` — read users out of a Better Auth database. + * + * Ported from the standalone migration-tool's `src/export/betterauth.ts`, on + * the `Bun.sql`/`bun:sqlite` client. + * + * Better Auth's schema depends on which plugins are enabled, so the columns + * are **detected from the schema** rather than asked for: the username plugin + * adds `username`, admin adds `banned`, phone-number adds `phoneNumber`, and + * so on. Selecting a column that is not there fails the whole query, and + * asking the user which plugins they run is a question their database can + * already answer. + * + * Passwords live on the `account` row for the credential provider, not on the + * user, which is why the export joins. + */ + +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; + +/** Columns a Better Auth plugin adds to the user table. */ +export const PLUGIN_COLUMNS = [ + "username", + "displayUsername", + "phoneNumber", + "phoneNumberVerified", + "role", + "banned", + "banReason", + "banExpires", + "twoFactorEnabled", +] as const; + +export type PluginColumn = (typeof PLUGIN_COLUMNS)[number]; + +/** Columns every Better Auth install has. */ +const CORE_COLUMNS = ["id", "email", "emailVerified", "name", "createdAt", "updatedAt"] as const; + +/** + * Asks the schema which plugin columns exist. + * + * SQLite has no `information_schema`, so it goes through `PRAGMA` — and the + * PRAGMA takes the table name inline rather than as a bind parameter. + */ +export async function detectPluginColumns(client: DbClient): Promise> { + const present = new Set(); + + if (client.dbType === "sqlite") { + const rows = await client.query<{ name: string }>(`PRAGMA table_info(${client.quote("user")})`); + const columns = new Set(rows.map((row) => row.name)); + for (const column of PLUGIN_COLUMNS) { + if (columns.has(column)) present.add(column); + } + return present; + } + + const scope = client.dbType === "mysql" ? "DATABASE()" : "current_schema()"; + const placeholders = PLUGIN_COLUMNS.map((_, index) => client.placeholder(index + 1)).join(", "); + + const rows = await client.query<{ column_name?: string; COLUMN_NAME?: string }>( + `SELECT column_name FROM information_schema.columns + WHERE table_name = 'user' AND table_schema = ${scope} + AND column_name IN (${placeholders})`, + [...PLUGIN_COLUMNS], + ); + + for (const row of rows) { + // MySQL 8 answers with an upper-case column label. + const name = (row.column_name ?? row.COLUMN_NAME) as PluginColumn | undefined; + if (name && (PLUGIN_COLUMNS as readonly string[]).includes(name)) present.add(name); + } + + return present; +} + +/** + * Builds the SELECT, including only the plugin columns that exist. + * + * @param pluginColumns - From {@link detectPluginColumns}. + */ +export function buildBetterAuthQuery(client: DbClient, pluginColumns: Set): string { + const q = (identifier: string) => client.quote(identifier); + const selected = [ + ...CORE_COLUMNS.map((column) => `u.${q(column)}`), + ...PLUGIN_COLUMNS.filter((column) => pluginColumns.has(column)).map( + (column) => `u.${q(column)}`, + ), + ]; + + // LEFT JOIN, not INNER: a user who only ever signed in with OAuth has no + // credential account, and dropping them would silently shrink the export. + return ( + `SELECT ${selected.join(", ")}, a.${q("password")} AS ${q("password_hash")} ` + + `FROM ${q("user")} u ` + + `LEFT JOIN ${q("account")} a ON a.${q("userId")} = u.${q("id")} ` + + `AND a.${q("providerId")} = 'credential' ` + + `ORDER BY u.${q("id")} ASC` + ); +} + +type BetterAuthRow = Record & { id?: unknown }; + +/** Renames the schema's camelCase onto what the betterauth transformer reads. */ +const FIELD_ALIASES: Record = { + id: "user_id", + emailVerified: "email_verified", + phoneNumber: "phone_number", + phoneNumberVerified: "phone_number_verified", + displayUsername: "display_username", + createdAt: "created_at", + updatedAt: "updated_at", +}; + +export function buildBetterAuthExport(rows: BetterAuthRow[], dateTime: string) { + const users: Record[] = []; + const counts = { email: 0, emailVerified: 0, password: 0, name: 0, username: 0, phone: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + const user: Record = {}; + + for (const [key, value] of Object.entries(row)) { + if (value === null || value === undefined) continue; + user[FIELD_ALIASES[key] ?? key] = value instanceof Date ? value.toISOString() : value; + } + + if (row.email) counts.email++; + if (row.emailVerified) counts.emailVerified++; + if (row.password_hash) counts.password++; + if (row.name) counts.name++; + if (row.username) counts.username++; + if (row.phoneNumber) counts.phone++; + + users.push(user); + exportLogger({ userId, status: "success" }, dateTime); + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.emailVerified }, + { label: "have a password hash", count: counts.password }, + { label: "have a name", count: counts.name }, + { label: "have a username", count: counts.username }, + { label: "have a phone number", count: counts.phone }, + ], + }; +} + +export async function exportBetterAuth(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, { + platform: "betterauth", + envVar: "BETTERAUTH_DB_URL", + prompt: "Better Auth database connection string", + hint: "Postgres, MySQL or a SQLite file — whichever your Better Auth install uses.", + }); + + await withGutter("Exporting users from Better Auth", async () => { + const dateTime = getDateTimeStamp(); + + const { rows, plugins } = await withSpinner("Reading the user table", () => + withDbClient(dbUrl, "betterauth", async (client) => { + const plugins = await detectPluginColumns(client); + const rows = await client.query(buildBetterAuthQuery(client, plugins)); + return { rows, plugins }; + }), + ); + + log.info( + plugins.size > 0 + ? `Detected plugin columns: ${[...plugins].join(", ")}.` + : "No plugin columns detected; exporting the core user fields.", + ); + + const { users, coverage } = buildBetterAuthExport(rows, dateTime); + const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("betterauth")); + + reportExport({ + platform: "betterauth", + userCount: users.length, + outputPath, + coverage, + transformerKey: "betterauth", + }); + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts new file mode 100644 index 000000000..12a3062db --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -0,0 +1,281 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildClerkExport, + exportClerk, + fetchAllClerkUsers, + mapClerkUserToExport, +} from "./clerk.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: string[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expclerk-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +/** Answers `GET /v1/users` from `pages`, one page per call. */ +function stubPages(pages: unknown[][]) { + let call = 0; + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push(input.toString()); + return Response.json(pages[call++] ?? []); + }) as unknown as typeof fetch; +} + +const user = (overrides: Record = {}) => ({ + id: "user_1", + primary_email_address_id: "idn_1", + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + ], + phone_numbers: [], + ...overrides, +}); + +describe("mapClerkUserToExport", () => { + test("writes the field names the clerk transformer reads", () => { + expect( + mapClerkUserToExport(user({ first_name: "Ada", last_name: "L", username: "ada" })), + ).toMatchObject({ + id: "user_1", + primary_email_address: "a@x.dev", + first_name: "Ada", + last_name: "L", + username: "ada", + }); + }); + + // `migrate run` puts the first entry on POST /v1/users and attaches the rest + // afterwards, so a reordered list would change which address signs the user in. + test("keeps the primary identifier out of the additional list", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("a@x.dev"); + expect(mapped.verified_email_addresses).toEqual(["b@x.dev"]); + }); + + test("separates unverified identifiers", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "c@x.dev", verification: { status: "unverified" } }, + ], + }), + ); + expect(mapped.unverified_email_addresses).toEqual(["c@x.dev"]); + expect(mapped.verified_email_addresses).toBeUndefined(); + }); + + test("promotes the first verified address when none is flagged primary", () => { + const mapped = mapClerkUserToExport( + user({ + primary_email_address_id: null, + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("a@x.dev"); + expect(mapped.verified_email_addresses).toEqual(["b@x.dev"]); + }); + + test("maps phone numbers the same way", () => { + const mapped = mapClerkUserToExport( + user({ + primary_phone_number_id: "pn_1", + phone_numbers: [ + { id: "pn_1", phone_number: "+15555550100", verification: { status: "verified" } }, + { id: "pn_2", phone_number: "+15555550101", verification: { status: "unverified" } }, + ], + }), + ); + expect(mapped.primary_phone_number).toBe("+15555550100"); + expect(mapped.unverified_phone_numbers).toEqual(["+15555550101"]); + }); + + test("converts BAPI's Unix-millisecond timestamps to RFC3339", () => { + const mapped = mapClerkUserToExport(user({ created_at: 1704067200000 })); + expect(mapped.created_at).toBe("2024-01-01T00:00:00.000Z"); + }); + + test("omits empty metadata rather than writing empty objects", () => { + const mapped = mapClerkUserToExport( + user({ public_metadata: {}, private_metadata: { plan: "pro" } }), + ); + expect("public_metadata" in mapped).toBe(false); + expect(mapped.private_metadata).toEqual({ plan: "pro" }); + }); + + test("carries the account-state fields the import accepts", () => { + const mapped = mapClerkUserToExport( + user({ + banned: true, + create_organization_enabled: false, + create_organizations_limit: 3, + delete_self_enabled: true, + }), + ); + expect(mapped).toMatchObject({ + banned: true, + create_organization_enabled: false, + create_organizations_limit: 3, + delete_self_enabled: true, + }); + }); +}); + +describe("fetchAllClerkUsers", () => { + test("pages until a short page arrives", async () => { + stubPages([ + Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), + Array.from({ length: 12 }, (_, i) => user({ id: `v${i}` })), + ]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(512); + expect(requests).toHaveLength(2); + expect(requests[1]).toContain("offset=500"); + }); + + // A full final page must still trigger one more request, or an instance whose + // size is an exact multiple of the page size would look short by one page. + test("makes one more request when the last page is exactly full", async () => { + stubPages([Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), []]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(500); + expect(requests).toHaveLength(2); + }); + + test("asks for BAPI's maximum page size", async () => { + stubPages([[]]); + await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + expect(requests[0]).toContain("limit=500"); + }); + + test("copes with an instance that has no users", async () => { + stubPages([[]]); + expect(await fetchAllClerkUsers({ secretKey: "sk_test_x" })).toEqual([]); + }); +}); + +describe("buildClerkExport", () => { + test("counts coverage per field", () => { + const { coverage } = buildClerkExport( + [user({ id: "u1", first_name: "Ada", password_enabled: true }), user({ id: "u2" })], + "2026-01-01T00:00:00", + ); + + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + expect(byLabel["have a password (not exportable — see below)"]).toBe(1); + }); + + test("logs one NDJSON line per exported user", () => { + buildClerkExport([user({ id: "u1" }), user({ id: "u2" })], "2026-01-01T00:00:00"); + + const entries = fs + .readdirSync(getLogDir()) + .flatMap((name) => fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n")) + .map((line) => JSON.parse(line) as Record); + + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ userId: "u1", status: "success" }); + }); + + test("writes the export log where `logs list` will find it", () => { + buildClerkExport([user()], "2026-01-01T12:00:00"); + expect(fs.readdirSync(getLogDir())[0]).toBe("export-2026-01-01T12-00-00.log"); + }); +}); + +describe("exportClerk", () => { + test("writes the default path and reports coverage", async () => { + stubPages([[user({ id: "u1", first_name: "Ada" })], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + const written = JSON.parse( + fs.readFileSync(path.join(workDir, "exports", "clerk-export.json"), "utf-8"), + ) as Record[]; + expect(written).toHaveLength(1); + expect(written[0]?.id).toBe("u1"); + expect(captured.err).toContain("Field coverage"); + expect(captured.err).toContain("Exported 1 user(s)"); + }); + + test("names the command that consumes the file", async () => { + stubPages([[user()], []]); + await exportClerk({ secretKey: "sk_test_x" }); + expect(captured.err).toContain( + "migrate run --transformer clerk --file exports/clerk-export.json", + ); + }); + + test("--output controls the destination, relative to the working directory", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x", output: "somewhere/mine.json" }); + + expect(fs.existsSync(path.join(workDir, "somewhere", "mine.json"))).toBe(true); + expect(fs.existsSync(path.join(workDir, "exports", "clerk-export.json"))).toBe(false); + }); + + // Silence here would be the worst outcome: the operator finds out when + // nobody can sign in to the destination instance. + test("says plainly that passwords are not in the file", async () => { + stubPages([[user({ password_enabled: true })], []]); + await exportClerk({ secretKey: "sk_test_x" }); + expect(captured.err).toContain("never returns password digests"); + }); + + test("writes an empty file and says so when the instance has no users", async () => { + stubPages([[]]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("No users found to export"); + expect( + JSON.parse(fs.readFileSync(path.join(workDir, "exports", "clerk-export.json"), "utf-8")), + ).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts new file mode 100644 index 000000000..37c10d802 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -0,0 +1,263 @@ +/** + * `clerk migrate export clerk` — pull users out of a Clerk instance. + * + * Ported from the standalone migration-tool's `src/export/clerk.ts`, rewritten + * onto `bapiRequest` instead of `@clerk/backend` so it shares the CLI's auth + * resolution, `--verbose` request tracing and error taxonomy. + * + * The output feeds `clerk migrate run --transformer clerk` unedited, which is + * what makes development → production a two-command operation. + * + * **Passwords do not come out of this endpoint.** Clerk never returns password + * digests, TOTP secrets or backup codes over the API; only the `*_enabled` + * booleans. The coverage report says how many users *have* a password so the + * gap is visible before the import, not after. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { retryOn429 } from "../lib/retry.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; + +/** BAPI's maximum page size for `GET /v1/users`. */ +const PAGE_SIZE = 500; + +export type ExportClerkOptions = { + output?: string; + secretKey?: string; + clerkSecretKey?: string; + app?: string; + instance?: string; +}; + +type BapiIdentifier = { + email_address?: string; + phone_number?: string; + verification?: { status?: string } | null; +}; + +type BapiUser = { + id: string; + external_id?: string | null; + username?: string | null; + first_name?: string | null; + last_name?: string | null; + email_addresses?: BapiIdentifier[]; + phone_numbers?: BapiIdentifier[]; + primary_email_address_id?: string | null; + primary_phone_number_id?: string | null; + public_metadata?: Record; + private_metadata?: Record; + unsafe_metadata?: Record; + password_enabled?: boolean; + totp_enabled?: boolean; + banned?: boolean; + create_organization_enabled?: boolean; + create_organizations_limit?: number | null; + delete_self_enabled?: boolean; + created_at?: number; + legal_accepted_at?: number | null; +}; + +type IdentifierWithId = BapiIdentifier & { id?: string }; + +/** + * Splits identifiers into verified and unverified, primary first. + * + * The primary has to lead: `migrate run` puts the first entry on + * `POST /v1/users` and attaches the rest afterwards, so a reordered list would + * silently change which address the user signs in with. + */ +function splitIdentifiers( + entries: IdentifierWithId[] | undefined, + primaryId: string | null | undefined, + read: (entry: BapiIdentifier) => string | undefined, +): { primary?: string; verified: string[]; unverified: string[] } { + const verified: string[] = []; + const unverified: string[] = []; + let primary: string | undefined; + + for (const entry of entries ?? []) { + const value = read(entry); + if (!value) continue; + + if (entry.id && entry.id === primaryId) { + primary = value; + continue; + } + if (entry.verification?.status === "verified") verified.push(value); + else unverified.push(value); + } + + // No primary flagged: promote the first verified one so the export still has + // an identifier the import can lead with. + if (!primary && verified.length > 0) primary = verified.shift(); + + return { primary, verified, unverified }; +} + +/** Maps a BAPI user onto the shape the `clerk` transformer reads. */ +export function mapClerkUserToExport(user: BapiUser): Record { + const exported: Record = { id: user.id }; + + const emails = splitIdentifiers( + user.email_addresses, + user.primary_email_address_id, + (entry) => entry.email_address, + ); + if (emails.primary) exported.primary_email_address = emails.primary; + if (emails.verified.length > 0) exported.verified_email_addresses = emails.verified; + if (emails.unverified.length > 0) exported.unverified_email_addresses = emails.unverified; + + const phones = splitIdentifiers( + user.phone_numbers, + user.primary_phone_number_id, + (entry) => entry.phone_number, + ); + if (phones.primary) exported.primary_phone_number = phones.primary; + if (phones.verified.length > 0) exported.verified_phone_numbers = phones.verified; + if (phones.unverified.length > 0) exported.unverified_phone_numbers = phones.unverified; + + if (user.username) exported.username = user.username; + if (user.first_name) exported.first_name = user.first_name; + if (user.last_name) exported.last_name = user.last_name; + + for (const [source, target] of [ + ["public_metadata", "public_metadata"], + ["private_metadata", "private_metadata"], + ["unsafe_metadata", "unsafe_metadata"], + ] as const) { + const value = user[source]; + if (value && Object.keys(value).length > 0) exported[target] = value; + } + + if (user.banned) exported.banned = true; + if (user.create_organization_enabled !== undefined) { + exported.create_organization_enabled = user.create_organization_enabled; + } + if (user.create_organizations_limit !== null && user.create_organizations_limit !== undefined) { + exported.create_organizations_limit = user.create_organizations_limit; + } + if (user.delete_self_enabled !== undefined) { + exported.delete_self_enabled = user.delete_self_enabled; + } + + // BAPI reports timestamps as Unix milliseconds; the schema wants RFC3339. + if (user.created_at) exported.created_at = new Date(user.created_at).toISOString(); + if (user.legal_accepted_at) { + exported.legal_accepted_at = new Date(user.legal_accepted_at).toISOString(); + } + + return exported; +} + +/** Pages through every user in the instance. */ +export async function fetchAllClerkUsers(options: { + secretKey: string; + spinner?: SpinnerControls; +}): Promise { + const all: BapiUser[] = []; + + for (let offset = 0; ; offset += PAGE_SIZE) { + const response = await retryOn429(() => + bapiRequest({ + method: "GET", + path: `/v1/users?limit=${PAGE_SIZE}&offset=${offset}`, + secretKey: options.secretKey, + }), + ); + + const page = Array.isArray(response.body) ? (response.body as BapiUser[]) : []; + all.push(...page); + options.spinner?.update(`Fetching users from Clerk: ${all.length} so far`); + + // A short page means the end; anything else would loop forever on an + // instance whose size happens to be a multiple of the page size. + if (page.length < PAGE_SIZE) break; + } + + return all; +} + +export type ClerkExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +/** Maps every user and counts what the export actually contains. */ +export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExportResult { + const exported: Record[] = []; + const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0, password: 0 }; + + for (const user of users) { + try { + const mapped = mapClerkUserToExport(user); + exported.push(mapped); + + if (mapped.primary_email_address) counts.email++; + if (mapped.username) counts.username++; + if (mapped.first_name) counts.firstName++; + if (mapped.last_name) counts.lastName++; + if (mapped.primary_phone_number) counts.phone++; + if (user.password_enabled) counts.password++; + + exportLogger({ userId: user.id, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId: user.id, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a phone number", count: counts.phone }, + { label: "have a username", count: counts.username }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + { label: "have a password (not exportable — see below)", count: counts.password }, + ], + }; +} + +export async function exportClerk(options: ExportClerkOptions): Promise { + if (options.clerkSecretKey) { + log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); + } + const secretKeyOption = options.secretKey ?? options.clerkSecretKey; + + await withGutter("Exporting users from Clerk", async () => { + const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const dateTime = getDateTimeStamp(); + + log.info(`Exporting from ${target ?? "the resolved instance"}.`); + + const users = await withSpinner( + "Fetching users from Clerk", + (spinner) => fetchAllClerkUsers({ secretKey, spinner }), + "Users fetched", + ); + + const { users: exported, coverage } = buildClerkExport(users, dateTime); + const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("clerk")); + + reportExport({ + platform: "clerk", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "clerk", + }); + + if (exported.length > 0) { + log.warn( + "Clerk's API never returns password digests, TOTP secrets or backup codes, so they are not in this file. " + + "Users will need to reset their password in the destination instance.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts new file mode 100644 index 000000000..e7601df39 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -0,0 +1,358 @@ +/** + * The three database-backed exports, driven against a real SQLite database. + * + * SQLite because it is the one engine that needs no container, and it + * exercises the same client, the same query building and the same plugin + * detection path (via `PRAGMA` rather than `information_schema`). Postgres and + * MySQL are covered by the manual matrix run recorded in the ticket. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { createDbClient, type DbClient } from "../lib/db.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { buildAuthJsExport, buildAuthJsQuery, exportAuthJs, fetchAuthJsUsers } from "./authjs.ts"; +import { + buildBetterAuthExport, + buildBetterAuthQuery, + detectPluginColumns, + exportBetterAuth, + PLUGIN_COLUMNS, +} from "./betterauth.ts"; +import { buildSupabaseExport } from "./supabase.ts"; +import { looksLikeConnectionString, resolveDbUrl } from "./db-options.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-dbexp-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +/** Builds a fresh SQLite file so each test starts from a known schema. */ +function makeDb(build: (db: Database) => void): string { + const file = path.join(workDir, `db-${counter++}.sqlite`); + const db = new Database(file, { create: true }); + build(db); + db.close(); + return file; +} + +function betterAuthDb(pluginColumns: string[], rows: Record[] = []): string { + return makeDb((db) => { + const extra = pluginColumns.map((column) => `, "${column}" TEXT`).join(""); + db.run( + `CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER, name TEXT, + "createdAt" TEXT, "updatedAt" TEXT${extra})`, + ); + db.run(`CREATE TABLE "account" (id TEXT, "userId" TEXT, "providerId" TEXT, password TEXT)`); + for (const row of rows) { + const keys = Object.keys(row); + db.run( + `INSERT INTO "user" (${keys.map((k) => `"${k}"`).join(",")}) VALUES (${keys.map(() => "?").join(",")})`, + keys.map((k) => row[k]) as never[], + ); + } + }); +} + +async function withClient(file: string, work: (client: DbClient) => Promise): Promise { + const client = await createDbClient(file); + try { + return await work(client); + } finally { + await client.close(); + } +} + +describe("looksLikeConnectionString", () => { + test.each([ + ["postgres://u:p@h:5432/db", true], + ["mysql://u:p@h:3306/db", true], + ["./db.sqlite", true], + ["file:./db.sqlite", true], + ["/abs/app.db", true], + ["", false], + [" ", false], + ["just some words", false], + ["postgres://", false], + ])("%p -> %p", (input, expected) => { + expect(looksLikeConnectionString(input)).toBe(expected); + }); +}); + +describe("resolveDbUrl", () => { + const config = { platform: "authjs" as const, envVar: "AUTHJS_DB_URL", prompt: "url" }; + + test("prefers the flag", async () => { + const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, { + AUTHJS_DB_URL: "mysql://u:p@h/db", + }); + expect(url).toBe("postgres://u:p@h/db"); + }); + + test("falls back to the environment variable", async () => { + expect(await resolveDbUrl({}, config, { AUTHJS_DB_URL: "mysql://u:p@h/db" })).toBe( + "mysql://u:p@h/db", + ); + }); + + test("rejects a flag that is not a connection string, naming the encoding trap", async () => { + await expect(resolveDbUrl({ dbUrl: "not a url" }, config, {})).rejects.toThrow(/URL-encode it/); + }); + + test("warns and moves on when the environment variable is unusable", async () => { + // Tests run non-TTY, so it then hits the agent-mode branch. + await expect(resolveDbUrl({}, config, { AUTHJS_DB_URL: "garbage" })).rejects.toThrow( + /cannot prompt here/, + ); + expect(captured.err).toContain("AUTHJS_DB_URL is not a valid connection string"); + }); + + test("names both the flag and the variable when it cannot prompt", async () => { + await expect(resolveDbUrl({}, config, {})).rejects.toThrow(/--db-url.*AUTHJS_DB_URL/s); + }); +}); + +describe("authjs export", () => { + const authJsDb = (table: string) => + makeDb((db) => { + db.run( + `CREATE TABLE "${table}" (id TEXT PRIMARY KEY, name TEXT, email TEXT, "emailVerified" TEXT)`, + ); + db.run(`INSERT INTO "${table}" VALUES (?,?,?,?)`, [ + "aj1", + "Jane Doe", + "jane@x.dev", + "2024-01-15", + ]); + db.run(`INSERT INTO "${table}" VALUES (?,?,?,?)`, ["aj2", "John Smith", "john@x.dev", null]); + }); + + test("quotes identifiers for the dialect", async () => { + await withClient(authJsDb("User"), async (client) => { + expect(buildAuthJsQuery(client, "User")).toContain('"User"'); + expect(buildAuthJsQuery(client, "User")).toContain('"emailVerified" AS "email_verified"'); + }); + }); + + // Prisma capitalizes the table, Drizzle does not, and Auth.js has no single + // schema — so the export tries rather than making the user guess. + test.each([["User"], ["user"], ["users"]])("finds the %s table", async (table) => { + const { rows } = await withClient(authJsDb(table), fetchAuthJsUsers); + expect(rows).toHaveLength(2); + }); + + test("fails clearly when no candidate table exists", async () => { + const file = makeDb((db) => db.run(`CREATE TABLE unrelated (id TEXT)`)); + await expect(withClient(file, fetchAuthJsUsers)).rejects.toThrow( + /No Auth.js user table found. Tried User, user, users/, + ); + }); + + test("treats email_verified as a nullable timestamp, not a boolean", () => { + const { users } = buildAuthJsExport( + [ + { id: "a", email: "a@x.dev", email_verified: "2024-01-15" }, + { id: "b", email: "b@x.dev", email_verified: null }, + ], + "2026-01-01T00:00:00", + ); + expect(users[0]?.email_verified).toBe("2024-01-15"); + expect("email_verified" in (users[1] ?? {})).toBe(false); + }); + + test("counts coverage", () => { + const { coverage } = buildAuthJsExport( + [{ id: "a", email: "a@x.dev", name: "A", email_verified: "2024-01-01" }, { id: "b" }], + "2026-01-01T00:00:00", + ); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(1); + expect(byLabel["have a verified email"]).toBe(1); + }); + + test("exports end to end and says which table it read", async () => { + await exportAuthJs({ dbUrl: authJsDb("User"), output: "authjs.json" }); + + const written = JSON.parse(fs.readFileSync(path.join(workDir, "authjs.json"), "utf-8")); + expect(written).toHaveLength(2); + expect(captured.err).toContain("Read 2 row(s) from"); + expect(captured.err).toContain("stores no passwords"); + }); +}); + +describe("betterauth export", () => { + test("detects only the plugin columns that exist", async () => { + await withClient(betterAuthDb(["username", "banned"]), async (client) => { + expect([...(await detectPluginColumns(client))].sort()).toEqual(["banned", "username"]); + }); + }); + + test("detects nothing on a core-only schema", async () => { + await withClient(betterAuthDb([]), async (client) => { + expect((await detectPluginColumns(client)).size).toBe(0); + }); + }); + + test("detects every plugin column when all are present", async () => { + await withClient(betterAuthDb([...PLUGIN_COLUMNS]), async (client) => { + expect((await detectPluginColumns(client)).size).toBe(PLUGIN_COLUMNS.length); + }); + }); + + // Selecting a column that is not there fails the whole query, which is why + // the columns are detected rather than assumed. + test("selects only detected columns", async () => { + await withClient(betterAuthDb(["username"]), async (client) => { + const query = buildBetterAuthQuery(client, await detectPluginColumns(client)); + expect(query).toContain('"username"'); + expect(query).not.toContain('"twoFactorEnabled"'); + }); + }); + + test("the built query actually runs against the schema it was built for", async () => { + const file = betterAuthDb( + ["username", "role"], + [{ id: "u1", email: "a@x.dev", username: "a" }], + ); + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, await detectPluginColumns(client))), + ); + expect(rows).toHaveLength(1); + }); + + // A user who only ever signed in with OAuth has no credential account; + // an INNER JOIN would drop them and silently shrink the export. + test("keeps a user with no credential account", async () => { + const file = betterAuthDb( + [], + [ + { id: "u1", email: "a@x.dev" }, + { id: "u2", email: "b@x.dev" }, + ], + ); + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, new Set())), + ); + expect(rows).toHaveLength(2); + }); + + test("renames camelCase columns onto what the transformer reads", () => { + const { users } = buildBetterAuthExport( + [{ id: "u1", emailVerified: 1, phoneNumber: "+1555", createdAt: "2025-01-01" }], + "2026-01-01T00:00:00", + ); + expect(users[0]).toMatchObject({ + user_id: "u1", + email_verified: 1, + phone_number: "+1555", + created_at: "2025-01-01", + }); + }); + + test("exports end to end and reports the detected plugins", async () => { + const file = betterAuthDb(["username"], [{ id: "u1", email: "a@x.dev", username: "ada" }]); + + await exportBetterAuth({ dbUrl: file, output: "ba.json" }); + + expect(captured.err).toContain("Detected plugin columns: username"); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "ba.json"), "utf-8"))).toHaveLength(1); + }); + + test("says so plainly when no plugins are in use", async () => { + await exportBetterAuth({ dbUrl: betterAuthDb([]), output: "ba2.json" }); + expect(captured.err).toContain("No plugin columns detected"); + }); +}); + +describe("supabase export", () => { + test("serializes timestamps the transformer can parse", () => { + const { users } = buildSupabaseExport( + [{ id: "u1", email: "a@x.dev", created_at: new Date("2024-01-01T00:00:00Z") }], + "2026-01-01T00:00:00", + ); + expect(users[0]?.created_at).toBe("2024-01-01T00:00:00.000Z"); + }); + + test("omits null columns rather than exporting them", () => { + const { users } = buildSupabaseExport( + [{ id: "u1", email: "a@x.dev", phone: null, last_name: null }], + "2026-01-01T00:00:00", + ); + expect("phone" in (users[0] ?? {})).toBe(false); + expect("last_name" in (users[0] ?? {})).toBe(false); + }); + + test("counts the password hashes, the reason this reads the database", () => { + const { coverage } = buildSupabaseExport( + [ + { id: "u1", email: "a@x.dev", encrypted_password: "$2b$10$x" }, + { id: "u2", email: "b@x.dev" }, + ], + "2026-01-01T00:00:00", + ); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have a password hash"]).toBe(1); + }); + + test("keeps raw_app_meta_data, which --skip-unsupported-providers reads", () => { + const { users } = buildSupabaseExport( + [{ id: "u1", email: "a@x.dev", raw_app_meta_data: { providers: ["discord"] } }], + "2026-01-01T00:00:00", + ); + expect(users[0]?.raw_app_meta_data).toEqual({ providers: ["discord"] }); + }); + + test("logs one NDJSON line per exported user", () => { + buildSupabaseExport([{ id: "u1" }, { id: "u2" }], "2026-01-01T12:00:00"); + + const written = fs.readdirSync(getLogDir()); + expect(written[0]).toBe("export-2026-01-01T12-00-00.log"); + expect( + fs + .readFileSync(path.join(getLogDir(), written[0] as string), "utf-8") + .trim() + .split("\n"), + ).toHaveLength(2); + }); +}); + +describe("connection failures", () => { + afterEach(() => { + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); + }); + + test("a missing SQLite file fails before anything is written", async () => { + await expect(exportAuthJs({ dbUrl: "./definitely-not-here.sqlite" })).rejects.toThrow(CliError); + expect(fs.existsSync(path.join(workDir, "exports"))).toBe(false); + }); + + test("the failure never contains the password", async () => { + await expect( + exportBetterAuth({ dbUrl: "postgres://user:hunter2@127.0.0.1:1/db" }), + ).rejects.toThrow( + expect.objectContaining({ message: expect.not.stringContaining("hunter2") }) as Error, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts new file mode 100644 index 000000000..880e177da --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -0,0 +1,111 @@ +/** + * Resolving a `--db-url` for the database-backed exports. + * + * Shared by supabase, authjs and betterauth: all three take one connection + * string, from a flag, an environment variable, or a prompt. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { detectDbType, redactConnectionString, type DbPlatform } from "../lib/db.ts"; + +export type DbExportOptions = { + dbUrl?: string; + output?: string; +}; + +type ResolveConfig = { + platform: DbPlatform; + /** Environment variable checked when `--db-url` is absent. */ + envVar: string; + prompt: string; + /** Extra guidance shown before prompting. */ + hint?: string; +}; + +/** True for something that could plausibly be a connection string. */ +export function looksLikeConnectionString(value: string): boolean { + const trimmed = value.trim(); + if (!trimmed) return false; + + if (/^(postgresql|postgres|mysql|mysql2):\/\//i.test(trimmed)) { + try { + // A hostname is required: `postgres://` alone parses as a valid URL, and + // accepting it only defers the failure into the driver. + return new URL(trimmed).hostname.length > 0; + } catch { + // A password with an unencoded `@` or `#` is the usual cause, and it is + // worth saying so rather than failing later inside the driver. + return false; + } + } + + return ( + trimmed.startsWith("file:") || /\.(sqlite3?|db)$/i.test(trimmed) || trimmed.startsWith("./") + ); +} + +/** + * Resolves the connection string: flag, then environment, then a prompt. + * + * Prompted as a password so it is not echoed — a connection string carries the + * database password inline. + */ +export async function resolveDbUrl( + options: DbExportOptions, + config: ResolveConfig, + env: Record = process.env, +): Promise { + const fromFlag = options.dbUrl?.trim(); + if (fromFlag) { + if (!looksLikeConnectionString(fromFlag)) { + throwUsageError( + `--db-url does not look like a connection string. Expected postgres://…, mysql://… or a SQLite file path.\n` + + "If the password contains @, # or /, URL-encode it.", + ); + } + return fromFlag; + } + + const fromEnv = env[config.envVar]?.trim(); + if (fromEnv) { + if (looksLikeConnectionString(fromEnv)) return fromEnv; + // Falling through silently would make the prompt look unexplained. + log.warn(`${config.envVar} is not a valid connection string; ignoring it.`); + } + + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export ${config.platform}\` needs a database connection and cannot prompt here.\n` + + `Pass --db-url, or set ${config.envVar}.`, + undefined, + undefined, + [ + { + command: `clerk migrate export ${config.platform} --db-url "postgres://user:password@host:5432/db"`, + description: "Export from Postgres", + }, + ], + ); + } + + if (config.hint) log.info(dim(config.hint)); + + const answer = await passwordPrompt({ + message: config.prompt, + validate: (value) => + looksLikeConnectionString(value ?? "") + ? undefined + : "Expected postgres://…, mysql://… or a SQLite file path", + }); + + return answer.trim(); +} + +/** Describes the target for the run's opening line, credentials removed. */ +export function describeTarget(connectionString: string): string { + return `${detectDbType(connectionString)} at ${redactConnectionString(connectionString)}`; +} diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts new file mode 100644 index 000000000..3772e35de --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -0,0 +1,451 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildFirebaseExport, + exportFirebase, + fetchAccessToken, + fetchAllFirebaseUsers, + fetchHashConfig, + formatHashConfigGuidance, + mapFirebaseUserToExport, + readServiceAccount, + signServiceAccountJwt, + type ServiceAccount, +} from "./firebase.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { url: string; body: unknown }[]; +let account: ServiceAccount; + +beforeAll(async () => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-fb-"))); + process.chdir(workDir); + + // A real RSA key, so the signing path is genuinely exercised. + const pair = await crypto.subtle.generateKey( + { + name: "RSASSA-PKCS1-v1_5", + modulusLength: 2048, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, + true, + ["sign", "verify"], + ); + const pkcs8 = await crypto.subtle.exportKey("pkcs8", pair.privateKey); + const body = btoa(String.fromCharCode(...new Uint8Array(pkcs8))).replace(/(.{64})/g, "$1\n"); + + account = { + project_id: "demo-fb", + client_email: "exp@demo-fb.iam.gserviceaccount.com", + private_key: `-----BEGIN PRIVATE KEY-----\n${body}\n-----END PRIVATE KEY-----\n`, + }; + fs.writeFileSync( + path.join(workDir, "sa.json"), + JSON.stringify({ type: "service_account", ...account }), + ); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; +}); + +/** Answers the token exchange, then one page per entry in `pages`. */ +function stubFirebase(pages: Record[][], hashConfig?: unknown) { + let page = 0; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ url, body: init?.body ?? null }); + + if (url.includes("oauth2.googleapis.com/token")) { + return Response.json({ access_token: "tok" }); + } + if (url.includes("/config")) { + return hashConfig === undefined + ? new Response("forbidden", { status: 403 }) + : Response.json(hashConfig); + } + const current = pages[page++] ?? []; + const hasMore = page < pages.length; + return Response.json({ users: current, ...(hasMore ? { nextPageToken: `p${page}` } : {}) }); + }) as unknown as typeof fetch; +} + +const fbUser = (i: number, overrides: Record = {}) => ({ + localId: `fb${i}`, + email: `u${i}@fb.dev`, + emailVerified: true, + displayName: `User ${i}`, + passwordHash: `SGFzaA${i}`, + salt: `U2FsdA${i}`, + createdAt: "1704067200000", + ...overrides, +}); + +describe("readServiceAccount", () => { + test("reads a valid key file", () => { + expect(readServiceAccount("./sa.json").project_id).toBe("demo-fb"); + }); + + test("reports a path that is not there", () => { + expect(() => readServiceAccount("./nope.json")).toThrow(/No service account file at/); + }); + + test("reports a file that is not JSON", () => { + fs.writeFileSync(path.join(workDir, "bad.json"), "not json"); + expect(() => readServiceAccount("./bad.json")).toThrow(/is not valid JSON/); + }); + + // Downloading the web app config instead of a service account key is the + // usual mistake, and the two files look similar at a glance. + test("points at the right console page for a web app config", () => { + fs.writeFileSync(path.join(workDir, "web.json"), JSON.stringify({ apiKey: "x" })); + expect(() => readServiceAccount("./web.json")).toThrow(/"project_id" is missing/); + }); + + test("names a wrong type explicitly", () => { + fs.writeFileSync(path.join(workDir, "wrong.json"), JSON.stringify({ type: "authorized_user" })); + expect(() => readServiceAccount("./wrong.json")).toThrow( + /"type" is "authorized_user".*Generate new private key/s, + ); + }); + + test.each([["project_id"], ["client_email"], ["private_key"]])( + "reports a missing %s", + (field) => { + const partial: Record = { type: "service_account", ...account }; + delete partial[field]; + fs.writeFileSync(path.join(workDir, `no-${field}.json`), JSON.stringify(partial)); + expect(() => readServiceAccount(`./no-${field}.json`)).toThrow( + new RegExp(`"${field}" is missing`), + ); + }, + ); + + // Pasting a key through a form that eats newlines is common, and the failure + // would otherwise surface as an opaque crypto error. + test("catches a private key whose newlines were mangled", () => { + fs.writeFileSync( + path.join(workDir, "mangled.json"), + JSON.stringify({ type: "service_account", ...account, private_key: "mangled" }), + ); + expect(() => readServiceAccount("./mangled.json")).toThrow(/newlines survived copying/); + }); + + test("raises CliError so the global handler formats it", () => { + expect(() => readServiceAccount("./nope.json")).toThrow(CliError); + }); +}); + +describe("signServiceAccountJwt", () => { + test("produces a three-segment RS256 JWT", async () => { + const jwt = await signServiceAccountJwt(account); + expect(jwt.split(".")).toHaveLength(3); + }); + + test("claims the right issuer, audience and scopes", async () => { + const jwt = await signServiceAccountJwt(account, 1_700_000_000); + const claims = JSON.parse( + atob((jwt.split(".")[1] as string).replace(/-/g, "+").replace(/_/g, "/")), + ); + + expect(claims).toMatchObject({ + iss: "exp@demo-fb.iam.gserviceaccount.com", + aud: "https://oauth2.googleapis.com/token", + iat: 1_700_000_000, + exp: 1_700_003_600, + }); + expect(claims.scope).toContain("cloud-platform"); + }); + + test("declares RS256 in the header", async () => { + const jwt = await signServiceAccountJwt(account); + const header = JSON.parse( + atob((jwt.split(".")[0] as string).replace(/-/g, "+").replace(/_/g, "/")), + ); + expect(header).toEqual({ alg: "RS256", typ: "JWT" }); + }); + + test("rejects a private key that is not valid base64", async () => { + await expect( + signServiceAccountJwt({ + ...account, + private_key: "-----BEGIN PRIVATE KEY-----\n!!!\n-----END PRIVATE KEY-----", + }), + ).rejects.toThrow(CliError); + }); +}); + +describe("fetchAccessToken", () => { + test("exchanges the assertion for a token", async () => { + stubFirebase([[]]); + + expect(await fetchAccessToken(account)).toBe("tok"); + expect(String(requests[0]?.body)).toContain("grant-type%3Ajwt-bearer"); + }); + + test("explains a rejection rather than surfacing a raw status", async () => { + globalThis.fetch = (async () => + new Response(JSON.stringify({ error_description: "Invalid JWT Signature" }), { + status: 400, + })) as unknown as typeof fetch; + + await expect(fetchAccessToken(account)).rejects.toThrow( + /Google rejected the service account \(400\): Invalid JWT Signature/, + ); + }); + + test("names the role the service account usually lacks", async () => { + globalThis.fetch = (async () => new Response("{}", { status: 403 })) as unknown as typeof fetch; + await expect(fetchAccessToken(account)).rejects.toThrow(/Firebase Authentication Admin/); + }); + + // The emulator has no token endpoint; `firebase-admin` uses the same bearer. + test("skips the exchange entirely against the emulator", async () => { + process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; + globalThis.fetch = (async () => { + throw new Error("should not have been called"); + }) as unknown as typeof fetch; + + expect(await fetchAccessToken(account)).toBe("owner"); + }); +}); + +describe("fetchAllFirebaseUsers", () => { + test("follows nextPageToken until it stops coming", async () => { + stubFirebase([ + Array.from({ length: 1000 }, (_, i) => fbUser(i)), + Array.from({ length: 7 }, (_, i) => fbUser(1000 + i)), + ]); + + const all = await fetchAllFirebaseUsers({ account, token: "tok" }); + + expect(all).toHaveLength(1007); + expect(requests[1]?.url).toContain("nextPageToken=p1"); + }); + + test("asks for the endpoint's maximum page size", async () => { + stubFirebase([[]]); + await fetchAllFirebaseUsers({ account, token: "tok" }); + expect(requests[0]?.url).toContain("maxResults=1000"); + }); + + test("targets the project named in the key", async () => { + stubFirebase([[]]); + await fetchAllFirebaseUsers({ account, token: "tok" }); + expect(requests[0]?.url).toContain("/projects/demo-fb/accounts:batchGet"); + }); + + test("routes through the emulator when one is configured", async () => { + process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; + stubFirebase([[]]); + + await fetchAllFirebaseUsers({ account, token: "owner" }); + + expect(requests[0]?.url).toStartWith("http://127.0.0.1:9099/"); + }); + + test("raises a clear error on a failed page", async () => { + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + + await expect(fetchAllFirebaseUsers({ account, token: "tok" })).rejects.toThrow( + /Firebase returned 500 listing users/, + ); + }); +}); + +describe("mapFirebaseUserToExport", () => { + test("keeps the fields the firebase transformer maps from", () => { + expect(mapFirebaseUserToExport(fbUser(0))).toEqual({ + localId: "fb0", + email: "u0@fb.dev", + displayName: "User 0", + createdAt: "1704067200000", + emailVerified: true, + passwordHash: "SGFzaA0", + salt: "U2FsdA0", + }); + }); + + test("drops project internals the import has no use for", () => { + const mapped = mapFirebaseUserToExport( + fbUser(0, { + providerUserInfo: [{ providerId: "password" }], + lastLoginAt: "1704153600000", + customAttributes: '{"role":"x"}', + validSince: "1704067200", + }), + ); + for (const noise of ["providerUserInfo", "lastLoginAt", "customAttributes", "validSince"]) { + expect(noise in mapped).toBe(false); + } + }); + + // A digest without its salt cannot be verified, so exporting one alone would + // produce a user nobody can sign in as. + test.each([ + ["hash without salt", { passwordHash: "H", salt: undefined }], + ["salt without hash", { passwordHash: undefined, salt: "S" }], + ])("drops a %s", (_label, overrides) => { + const mapped = mapFirebaseUserToExport(fbUser(0, overrides)); + expect("passwordHash" in mapped).toBe(false); + expect("salt" in mapped).toBe(false); + }); + + test("keeps emailVerified when it is false", () => { + expect(mapFirebaseUserToExport(fbUser(0, { emailVerified: false })).emailVerified).toBe(false); + }); + + test("copes with a phone-only user", () => { + const mapped = mapFirebaseUserToExport({ localId: "fb9", phoneNumber: "+15555550100" }); + expect(mapped).toEqual({ localId: "fb9", phoneNumber: "+15555550100" }); + }); +}); + +describe("buildFirebaseExport", () => { + test("counts coverage and logs each user", () => { + const { users, coverage } = buildFirebaseExport( + [fbUser(0), { localId: "fb1", phoneNumber: "+1555" }], + "2026-01-01T12:00:00", + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have a password hash"]).toBe(1); + expect(byLabel["have a phone number"]).toBe(1); + expect(fs.readdirSync(getLogDir())[0]).toBe("export-2026-01-01T12-00-00.log"); + }); +}); + +describe("fetchHashConfig", () => { + test("reads the project's scrypt parameters", async () => { + stubFirebase([[]], { + signIn: { + hashConfig: { signerKey: "KEY==", saltSeparator: "Bw==", rounds: 8, memoryCost: 14 }, + }, + }); + + expect(await fetchHashConfig(account, "tok")).toEqual({ + signerKey: "KEY==", + saltSeparator: "Bw==", + rounds: 8, + memoryCost: 14, + }); + }); + + // Reading the config needs a broader role than listing users, so a project + // where it is denied must still export. + test("returns null rather than failing when the call is not permitted", async () => { + stubFirebase([[]]); + expect(await fetchHashConfig(account, "tok")).toBeNull(); + }); + + test("returns null when the response carries no hash config", async () => { + stubFirebase([[]], { signIn: {} }); + expect(await fetchHashConfig(account, "tok")).toBeNull(); + }); +}); + +describe("formatHashConfigGuidance", () => { + const config = { signerKey: "KEY==", saltSeparator: "Bw==", rounds: 8, memoryCost: 14 }; + + test("prints the exact import command when the parameters are known", () => { + const text = formatHashConfigGuidance(config, "exports/firebase-export.json", 3).join("\n"); + expect(text).toContain('--firebase-signer-key "KEY=="'); + expect(text).toContain('--firebase-salt-separator "Bw=="'); + expect(text).toContain("--firebase-rounds 8 --firebase-mem-cost 14"); + }); + + test("says where to find them when the project would not say", () => { + const text = formatHashConfigGuidance(null, "out.json", 3).join("\n"); + expect(text).toContain("Password hash parameters"); + expect(text).toContain("Authentication → Users"); + }); + + // Nothing to configure, so nothing to tell them to configure. + test("says nothing is needed when the export has no hashes", () => { + expect(formatHashConfigGuidance(null, "out.json", 0).join("\n")).toContain( + "no hash parameters are needed", + ); + }); +}); + +describe("exportFirebase", () => { + test("exports end to end and reports coverage", async () => { + stubFirebase([[fbUser(0), fbUser(1)]], { + signIn: { hashConfig: { signerKey: "K", saltSeparator: "S", rounds: 8, memoryCost: 14 } }, + }); + + await exportFirebase({ serviceAccount: "./sa.json" }); + + const written = JSON.parse( + fs.readFileSync(path.join(workDir, "exports", "firebase-export.json"), "utf-8"), + ) as Record[]; + expect(written).toHaveLength(2); + expect(captured.err).toContain("Field coverage"); + expect(captured.err).toContain("demo-fb project"); + }); + + test("names the command that consumes the file", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(captured.err).toContain( + "migrate run --transformer firebase --file exports/firebase-export.json", + ); + }); + + test("--output controls the destination", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json", output: "fb.json" }); + expect(fs.existsSync(path.join(workDir, "fb.json"))).toBe(true); + }); + + test("requires --service-account, before anything is read", async () => { + await expect(exportFirebase({})).rejects.toThrow(/needs a service account key file/); + }); + + test("validates the key file before making any request", async () => { + stubFirebase([[fbUser(0)]]); + await expect(exportFirebase({ serviceAccount: "./nope.json" })).rejects.toThrow(CliError); + expect(requests).toHaveLength(0); + }); + + test("never puts key material in the output", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(captured.err).not.toContain("BEGIN PRIVATE KEY"); + expect(captured.err).not.toContain(account.private_key.slice(40, 80)); + }); + + test("skips the hash-parameter section when nothing has a password", async () => { + stubFirebase([[{ localId: "fb9", phoneNumber: "+1555" }]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(captured.err).toContain("no hash parameters are needed"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts new file mode 100644 index 000000000..0491227ae --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -0,0 +1,458 @@ +/** + * `clerk migrate export firebase` — pull users out of Firebase Authentication. + * + * Ported from the standalone migration-tool's `src/export/firebase.ts`, but + * **without `firebase-admin`**. + * + * The spike the ticket asked for was run first, and it passed: a + * `bun build --compile` binary imports `firebase-admin`, initializes it, and + * completes `listUsers` against Identity Toolkit. The known Firestore-under- + * compile bug does not reach the Auth Admin surface. + * + * The SDK was still not adopted, on the second measurement: it is **74 MB + * across 158 packages**, including `@google-cloud/firestore` and + * `@google-cloud/storage`, neither of which this command touches. The compiled + * `clerk` binary is ~65 MB today, so that roughly doubles the artifact every + * user downloads — to serve one subcommand. + * + * What the SDK actually does here is two REST calls and an RS256 JWT, and Bun's + * Web Crypto signs RS256 with no dependency at all (verified compiled). So this + * adds **zero** packages, and its HTTP goes through `loggedFetch`, so a + * `--verbose` run shows the requests — which an SDK doing its own fetch would + * not. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { bold, dim } from "../../../lib/color.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; + +/** Identity Toolkit's maximum for `accounts:batchGet`. */ +const PAGE_SIZE = 1000; + +const TOKEN_URL = "https://oauth2.googleapis.com/token"; +const SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/firebase", +].join(" "); + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/firebase"; + +export type ExportFirebaseOptions = { + serviceAccount?: string; + output?: string; +}; + +export type ServiceAccount = { + project_id: string; + client_email: string; + private_key: string; +}; + +/** + * Reads and validates a service-account key file. + * + * Every failure names the field, because the usual causes are downloading the + * wrong JSON from the console (a web app config rather than a service account) + * or pasting a key with its newlines mangled. + */ +export function readServiceAccount(file: string): ServiceAccount { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No service account file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + + let parsed: unknown; + try { + parsed = JSON.parse(fs.readFileSync(resolved, "utf-8")); + } catch (error) { + throw new CliError(`${file} is not valid JSON: ${(error as Error).message}`, { + code: ERROR_CODE.INVALID_JSON, + docsUrl: DOCS_URL, + }); + } + + const account = parsed as Partial & { type?: string }; + const invalid = (problem: string): never => { + throw new CliError(`${file} is not a usable service account key: ${problem}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + }; + + if (account.type && account.type !== "service_account") { + invalid( + `its "type" is "${account.type}", not "service_account". Download a private key from ` + + "Project settings → Service accounts → Generate new private key.", + ); + } + for (const field of ["project_id", "client_email", "private_key"] as const) { + if (typeof account[field] !== "string" || account[field].length === 0) { + invalid(`"${field}" is missing`); + } + } + if (!account.private_key?.includes("PRIVATE KEY")) { + invalid('"private_key" does not look like a PEM key — check its newlines survived copying'); + } + + return account as ServiceAccount; +} + +function base64Url(input: string | Uint8Array): string { + const binary = + typeof input === "string" ? input : String.fromCharCode(...(input as unknown as number[])); + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +/** Imports the PEM private key for RS256 signing. */ +async function importPrivateKey(pem: string): Promise { + const body = pem.replace(/-----[^-]+-----/g, "").replace(/\s+/g, ""); + let der: Uint8Array; + try { + der = Uint8Array.from(atob(body), (character) => character.charCodeAt(0)); + } catch { + throw new CliError("The service account's private_key is not valid base64.", { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + } + + try { + return await crypto.subtle.importKey( + "pkcs8", + der, + { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, + false, + ["sign"], + ); + } catch (error) { + throw new CliError( + `The service account's private_key could not be read: ${(error as Error).message}`, + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } +} + +/** + * Signs the assertion Google exchanges for an access token. + * + * @param now - Seconds since the epoch; injectable so tests are not clock-bound. + */ +export async function signServiceAccountJwt( + account: ServiceAccount, + now: number = Math.floor(Date.now() / 1000), +): Promise { + const key = await importPrivateKey(account.private_key); + const claims = { + iss: account.client_email, + scope: SCOPES, + aud: TOKEN_URL, + iat: now, + exp: now + 3600, + }; + const body = `${base64Url(JSON.stringify({ alg: "RS256", typ: "JWT" }))}.${base64Url(JSON.stringify(claims))}`; + + const signature = await crypto.subtle.sign( + "RSASSA-PKCS1-v1_5", + key, + new TextEncoder().encode(body), + ); + + return `${body}.${base64Url(new Uint8Array(signature))}`; +} + +/** + * Exchanges the signed assertion for an Identity Toolkit access token. + * + * Against the emulator there is nothing to exchange with — Google's token + * endpoint is not part of it — so the run uses the `owner` bearer the emulator + * accepts, matching what `firebase-admin` does. + */ +export async function fetchAccessToken(account: ServiceAccount): Promise { + if (process.env.FIREBASE_AUTH_EMULATOR_HOST) return "owner"; + + const assertion = await signServiceAccountJwt(account); + + const response = await loggedFetch(new URL(TOKEN_URL), { + tag: "firebase", + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", + assertion, + }).toString(), + }); + + const body = (await response.json().catch(() => ({}))) as { + access_token?: string; + error_description?: string; + error?: string; + }; + + if (!response.ok || !body.access_token) { + throw new CliError( + `Google rejected the service account (${response.status}): ${body.error_description ?? body.error ?? "no access token returned"}\n` + + "Check the key has not been revoked, and that the service account has the Firebase Authentication Admin role.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + return body.access_token; +} + +/** + * Base URL for Identity Toolkit. + * + * Honours `FIREBASE_AUTH_EMULATOR_HOST`, the variable Firebase's own tooling + * uses, so this works against the local emulator as well as production. + */ +function identityToolkitBase(): string { + const emulator = process.env.FIREBASE_AUTH_EMULATOR_HOST; + return emulator + ? `http://${emulator}/identitytoolkit.googleapis.com` + : "https://identitytoolkit.googleapis.com"; +} + +export type FirebaseUser = Record & { localId?: string }; + +/** Pages through every user in the project. */ +export async function fetchAllFirebaseUsers(options: { + account: ServiceAccount; + token: string; + spinner?: SpinnerControls; +}): Promise { + const all: FirebaseUser[] = []; + let pageToken: string | undefined; + + do { + const url = new URL( + `${identityToolkitBase()}/v1/projects/${options.account.project_id}/accounts:batchGet`, + ); + url.searchParams.set("maxResults", String(PAGE_SIZE)); + if (pageToken) url.searchParams.set("nextPageToken", pageToken); + + const response = await loggedFetch(url, { + tag: "firebase", + method: "GET", + headers: { Authorization: `Bearer ${options.token}`, Accept: "application/json" }, + }); + + if (!response.ok) { + throw new CliError( + `Firebase returned ${response.status} listing users: ${await response.text()}`, + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + const body = (await response.json()) as { users?: FirebaseUser[]; nextPageToken?: string }; + all.push(...(body.users ?? [])); + options.spinner?.update(`Fetching users from Firebase: ${all.length} so far`); + pageToken = body.nextPageToken; + } while (pageToken); + + return all; +} + +export type HashConfig = { + signerKey: string; + saltSeparator: string; + rounds: number; + memoryCost: number; +}; + +/** + * Reads the project's scrypt parameters. + * + * These are the whole reason a Firebase migration keeps its passwords: without + * them Clerk cannot verify a single digest. Fetching them here saves the user + * hunting through the console — and if the call is not permitted, the run says + * exactly where to look instead. + * + * @returns `null` when the config could not be read. + */ +export async function fetchHashConfig( + account: ServiceAccount, + token: string, +): Promise { + try { + const url = new URL(`${identityToolkitBase()}/admin/v2/projects/${account.project_id}/config`); + const response = await loggedFetch(url, { + tag: "firebase", + method: "GET", + headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, + }); + if (!response.ok) { + log.debug(`firebase: ${response.status} reading the project config`); + return null; + } + + const body = (await response.json()) as { + signIn?: { hashConfig?: Partial & { algorithm?: string } }; + }; + const config = body.signIn?.hashConfig; + if (!config?.signerKey || !config.saltSeparator) return null; + + return { + signerKey: config.signerKey, + saltSeparator: config.saltSeparator, + rounds: Number(config.rounds ?? 8), + memoryCost: Number(config.memoryCost ?? 14), + }; + } catch (error) { + log.debug(`firebase: could not read the project config: ${String(error)}`); + return null; + } +} + +/** + * Keeps the fields the `firebase` transformer maps from. + * + * A Firebase user also carries provider records, custom claims and sign-in + * timestamps that would bloat the export and mean nothing to the import. + */ +export function mapFirebaseUserToExport(user: FirebaseUser): Record { + const exported: Record = {}; + + for (const field of ["localId", "email", "displayName", "phoneNumber", "createdAt"] as const) { + if (user[field]) exported[field] = user[field]; + } + // Meaningful when false, so copied on presence rather than truthiness. + if (user.emailVerified !== undefined) exported.emailVerified = user.emailVerified; + + // Both halves or neither: a digest without its salt cannot be verified. + if (user.passwordHash && user.salt) { + exported.passwordHash = user.passwordHash; + exported.salt = user.salt; + } + + return exported; +} + +export function buildFirebaseExport(users: FirebaseUser[], dateTime: string) { + const exported: Record[] = []; + const counts = { email: 0, verified: 0, password: 0, name: 0, phone: 0 }; + + for (const user of users) { + const userId = String(user.localId ?? ""); + try { + const mapped = mapFirebaseUserToExport(user); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.emailVerified) counts.verified++; + if (mapped.passwordHash) counts.password++; + if (mapped.displayName) counts.name++; + if (mapped.phoneNumber) counts.phone++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.verified }, + { label: "have a password hash", count: counts.password }, + { label: "have a display name", count: counts.name }, + { label: "have a phone number", count: counts.phone }, + ], + }; +} + +/** The exact `migrate run` invocation, with the project's own parameters. */ +export function formatHashConfigGuidance( + config: HashConfig | null, + outputPath: string, + passwordCount: number, +): string[] { + if (passwordCount === 0) { + return [dim("No password hashes in this export, so no hash parameters are needed.")]; + } + + if (!config) { + return [ + bold("Password hash parameters"), + "This export carries password hashes, which Clerk can only verify with the project's", + "scrypt parameters. Find them in the Firebase console under", + "Authentication → Users → (⋮) → Password hash parameters, then pass:", + dim( + " --firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost", + ), + ]; + } + + return [ + bold("Password hash parameters"), + "Read from the project. Import with:", + dim( + ` clerk migrate run -y --transformer firebase --file ${outputPath} \\\n` + + ` --firebase-signer-key "${config.signerKey}" \\\n` + + ` --firebase-salt-separator "${config.saltSeparator}" \\\n` + + ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, + ), + ]; +} + +export async function exportFirebase(options: ExportFirebaseOptions): Promise { + if (!options.serviceAccount) { + throwUsageError( + "`clerk migrate export firebase` needs a service account key file. Pass --service-account .", + DOCS_URL, + undefined, + [ + { + command: "clerk migrate export firebase --service-account ./service-account.json", + description: "Export using a downloaded service account key", + }, + ], + ); + } + + // Read and validate before anything reaches the network, so a wrong file + // fails in a second rather than after an auth round-trip. + const account = readServiceAccount(options.serviceAccount); + + await withGutter("Exporting users from Firebase", async () => { + const dateTime = getDateTimeStamp(); + log.info(`Exporting from the ${account.project_id} project.`); + + const token = await withSpinner("Authenticating with Google", () => fetchAccessToken(account)); + + const users = await withSpinner( + "Fetching users from Firebase", + (spinner) => fetchAllFirebaseUsers({ account, token, spinner }), + "Users fetched", + ); + + const { users: exported, coverage } = buildFirebaseExport(users, dateTime); + const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("firebase")); + + reportExport({ + platform: "firebase", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "firebase", + }); + + const passwordCount = coverage.find((entry) => entry.label.includes("password"))?.count ?? 0; + const hashConfig = passwordCount > 0 ? await fetchHashConfig(account, token) : null; + + log.blank(); + for (const line of formatHashConfigGuidance(hashConfig, outputPath, passwordCount)) { + log.info(line); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts new file mode 100644 index 000000000..2ac69b5b3 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -0,0 +1,181 @@ +import type { Command } from "@commander-js/extra-typings"; +import { throwUsageError } from "../../../lib/errors.ts"; +import { select } from "../../../lib/listage.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { exportAuth0 } from "./auth0.ts"; +import { exportAuthJs } from "./authjs.ts"; +import { exportBetterAuth } from "./betterauth.ts"; +import { exportClerk } from "./clerk.ts"; +import { exportFirebase } from "./firebase.ts"; +import { exportSupabase } from "./supabase.ts"; +import type { DbExportOptions } from "./db-options.ts"; +import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; + +/** + * Bare `clerk migrate export` — pick a platform, then run its export. + * + * The picker is built from the registry, so a new platform appears without a + * second place to update. Whatever the chosen platform needs beyond the + * platform name, it prompts for itself. + */ +export async function exportPicker(options: Record = {}): Promise { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export\` needs a platform and cannot prompt here. Name one: ${exportPlatformKeys().join(", ")}.`, + undefined, + undefined, + exportPlatforms.map((entry) => ({ + command: `clerk migrate export ${entry.key}`, + description: entry.description, + })), + ); + } + + const platform = await select({ + message: "Which platform are you exporting from?", + choices: exportPlatforms.map((entry) => ({ + name: entry.label, + value: entry.key, + description: entry.description, + })), + }); + + const entry = getExportPlatform(platform); + // Unreachable via the picker; a guard so a registry edit cannot silently + // produce a choice with nothing behind it. + if (!entry) throwUsageError(`Unknown export platform "${platform}".`); + + await entry.run(options); +} + +const handlers = { + picker: exportPicker, + clerk: exportClerk, + auth0: exportAuth0, + supabase: exportSupabase, + authjs: exportAuthJs, + betterauth: exportBetterAuth, + firebase: exportFirebase, +}; + +/** The three platforms that read a database, which share `--db-url`. */ +const DB_PLATFORMS = [ + { + key: "supabase", + summary: "Export users from a Supabase Postgres database", + envVar: "SUPABASE_DB_URL", + example: "postgres://postgres:password@db.xxx.supabase.co:5432/postgres", + }, + { + key: "authjs", + summary: "Export users from an Auth.js database", + envVar: "AUTHJS_DB_URL", + example: "mysql://user:password@127.0.0.1:3306/authjs", + }, + { + key: "betterauth", + summary: "Export users from a Better Auth database", + envVar: "BETTERAUTH_DB_URL", + example: "./db.sqlite", + }, +] as const; + +/** Registers `export [platform]` under the `migrate` group. */ +export function registerMigrateExport(migrateCommand: Command<[], Record>): void { + const exportCommand = migrateCommand + .command("export") + .description("Export users from a source platform, ready for `migrate run`") + .setExamples([ + { command: "clerk migrate export", description: "Pick a platform interactively" }, + { + command: "clerk migrate export clerk --output users.json", + description: "Export from a Clerk instance", + }, + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export from an Auth0 tenant", + }, + ]) + .action((_opts, cmd) => handlers.picker(cmd.optsWithGlobals() as Record)); + + exportCommand + .command("clerk") + .description("Export users from a Clerk instance (default: ./exports/clerk-export.json)") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("--secret-key ", "Backend API secret key to use") + .option("--clerk-secret-key ", "Deprecated alias for --secret-key") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate export clerk", + description: "Export to ./exports/clerk-export.json", + }, + { + command: "clerk migrate export clerk --instance prod --output prod-users.json", + description: "Export a specific instance to a chosen path", + }, + ]) + .action((_opts, cmd) => + handlers.clerk(cmd.optsWithGlobals() as Parameters[0]), + ); + + exportCommand + .command("auth0") + .description("Export users from an Auth0 tenant (default: ./exports/auth0-export.json)") + .option("--domain ", "Auth0 tenant domain, e.g. my-tenant.us.auth0.com") + .option("--client-id ", "Machine-to-machine application client ID") + .option("--client-secret ", "Machine-to-machine application client secret") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .setExamples([ + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export with explicit credentials", + }, + { + command: "clerk migrate export auth0", + description: "Read AUTH0_DOMAIN, AUTH0_CLIENT_ID and AUTH0_CLIENT_SECRET, or prompt", + }, + ]) + .action((_opts, cmd) => + handlers.auth0(cmd.optsWithGlobals() as Parameters[0]), + ); + + exportCommand + .command("firebase") + .description("Export users from a Firebase project (default: ./exports/firebase-export.json)") + .option("--service-account ", "Path to a service account key JSON file") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .setExamples([ + { + command: "clerk migrate export firebase --service-account ./service-account.json", + description: "Export using a downloaded service account key", + }, + ]) + .action((_opts, cmd) => + handlers.firebase(cmd.optsWithGlobals() as Parameters[0]), + ); + + // All three take exactly one connection string, so they are registered from + // a table rather than three near-identical blocks. + for (const platform of DB_PLATFORMS) { + exportCommand + .command(platform.key) + .description(`${platform.summary} (default: ./exports/${platform.key}-export.json)`) + .option("--db-url ", "Postgres, MySQL or SQLite connection string") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .setExamples([ + { + command: `clerk migrate export ${platform.key} --db-url "${platform.example}"`, + description: "Export from an explicit database", + }, + { + command: `clerk migrate export ${platform.key}`, + description: `Read ${platform.envVar}, or prompt`, + }, + ]) + .action((_opts, cmd) => handlers[platform.key](cmd.optsWithGlobals() as DbExportOptions)); + } +} diff --git a/packages/cli-core/src/commands/migrate/export/registry.test.ts b/packages/cli-core/src/commands/migrate/export/registry.test.ts new file mode 100644 index 000000000..1acded24f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/registry.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, test } from "bun:test"; +import { transformerKeys } from "../transformers/registry.ts"; +import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; + +describe("export registry", () => { + test("registers every source platform", () => { + expect(exportPlatformKeys()).toEqual([ + "clerk", + "auth0", + "supabase", + "authjs", + "firebase", + "betterauth", + ]); + }); + + test.each([...exportPlatforms])("$key carries a label and description", (entry) => { + expect(entry.label.length).toBeGreaterThan(0); + expect(entry.description.length).toBeGreaterThan(0); + }); + + // The picker, the docs and the "what next" line all read this, so a typo + // would send someone to a transformer that does not exist. + test.each([...exportPlatforms])("$key names a real transformer", (entry) => { + expect(transformerKeys()).toContain(entry.transformerKey); + }); + + test.each([...exportPlatforms])("$key has something to run", (entry) => { + expect(typeof entry.run).toBe("function"); + }); + + test("looks a platform up by key", () => { + expect(getExportPlatform("auth0")?.label).toBe("Auth0"); + }); + + test("returns nothing for a platform that is not registered", () => { + expect(getExportPlatform("okta")).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/registry.ts b/packages/cli-core/src/commands/migrate/export/registry.ts new file mode 100644 index 000000000..93c4f2605 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/registry.ts @@ -0,0 +1,80 @@ +/** + * Export registry. + * + * The picker behind a bare `clerk migrate export` is built from this array, so + * adding a platform is one file plus one entry — the same shape the transformer + * registry uses. + * + * `run` takes no arguments on purpose: each platform resolves its own flags, + * environment variables and prompts, because what Auth0 needs (a tenant domain + * and M2M credentials) has nothing in common with what a database export needs. + */ + +import { exportAuth0 } from "./auth0.ts"; +import { exportAuthJs } from "./authjs.ts"; +import { exportBetterAuth } from "./betterauth.ts"; +import { exportClerk } from "./clerk.ts"; +import { exportFirebase } from "./firebase.ts"; +import { exportSupabase } from "./supabase.ts"; + +export type ExportRegistryEntry = { + key: string; + label: string; + description: string; + /** Which `--transformer` reads the file this export writes. */ + transformerKey: string; + run: (options: Record) => Promise; +}; + +export const exportPlatforms: ExportRegistryEntry[] = [ + { + key: "clerk", + label: "Clerk", + description: "Another Clerk instance, e.g. development → production", + transformerKey: "clerk", + run: (options) => exportClerk(options), + }, + { + key: "auth0", + label: "Auth0", + description: "An Auth0 tenant, via the Management API", + transformerKey: "auth0", + run: (options) => exportAuth0(options), + }, + { + key: "supabase", + label: "Supabase", + description: "A Supabase Postgres database — includes password hashes", + transformerKey: "supabase", + run: (options) => exportSupabase(options), + }, + { + key: "authjs", + label: "Auth.js (NextAuth)", + description: "An Auth.js database — Postgres, MySQL or SQLite", + transformerKey: "authjs", + run: (options) => exportAuthJs(options), + }, + { + key: "firebase", + label: "Firebase", + description: "A Firebase project, via Identity Toolkit", + transformerKey: "firebase", + run: (options) => exportFirebase(options), + }, + { + key: "betterauth", + label: "Better Auth", + description: "A Better Auth database — plugin columns detected automatically", + transformerKey: "betterauth", + run: (options) => exportBetterAuth(options), + }, +]; + +export function exportPlatformKeys(): string[] { + return exportPlatforms.map((entry) => entry.key); +} + +export function getExportPlatform(key: string): ExportRegistryEntry | undefined { + return exportPlatforms.find((entry) => entry.key === key); +} diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts new file mode 100644 index 000000000..aa25ea50f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -0,0 +1,85 @@ +/** + * Shared plumbing for the export modules: where the file lands, and what the + * user is told about it. + * + * Ported from the standalone migration-tool's `src/lib/export.ts`, with one + * behavioural change: `--output` resolves against the **current working + * directory**, the way every other path flag in this CLI does. The original + * resolved a relative `--output` inside `exports/`, so `--output ./here.json` + * silently wrote to `exports/here.json`. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { dim, green, yellow } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; + +/** Where an export lands when `--output` is not given. */ +export function defaultOutputPath(platform: string): string { + return path.join("exports", `${platform}-export.json`); +} + +/** + * Writes the export, creating any missing parent directories. + * + * @returns The absolute path written, for reporting. + */ +export function writeExportOutput(users: unknown[], outputFile: string): string { + const resolved = path.resolve(process.cwd(), outputFile); + fs.mkdirSync(path.dirname(resolved), { recursive: true }); + fs.writeFileSync(resolved, JSON.stringify(users, null, 2)); + return resolved; +} + +export type CoverageField = { label: string; count: number }; + +/** + * How complete an export is, per field. + * + * ● every user, ○ some, dim ○ none. The point is to see *before* importing + * that, say, only 3 of 400 users have a password — which changes what the + * migration means. + */ +export function formatFieldCoverage(fields: CoverageField[], total: number): string[] { + return fields.map(({ label, count }) => { + const icon = count === total ? green("●") : count > 0 ? yellow("○") : dim("○"); + return ` ${icon} ${dim(`${count}/${total} ${label}`)}`; + }); +} + +export type ExportSummary = { + platform: string; + userCount: number; + outputPath: string; + coverage: CoverageField[]; + /** The transformer that reads this file, for the "what next" line. */ + transformerKey: string; +}; + +/** Reports the coverage table and the exact command that consumes the file. */ +export function reportExport(summary: ExportSummary): void { + log.blank(); + if (summary.userCount === 0) { + log.warn(`No users found to export. Wrote an empty file to ${summary.outputPath}.`); + return; + } + + log.info("Field coverage"); + for (const line of formatFieldCoverage(summary.coverage, summary.userCount)) { + log.info(line); + } + + log.blank(); + log.success(`Exported ${summary.userCount} user(s) to ${summary.outputPath}`); + log.info( + dim( + `Next: clerk migrate run --transformer ${summary.transformerKey} --file ${relativeIfInside(summary.outputPath)}`, + ), + ); +} + +/** Shortens a path for display when it sits under the working directory. */ +function relativeIfInside(absolute: string): string { + const relative = path.relative(process.cwd(), absolute); + return relative.startsWith("..") ? absolute : relative; +} diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts new file mode 100644 index 000000000..585ebf9d9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -0,0 +1,138 @@ +/** + * `clerk migrate export supabase` — read users straight out of `auth.users`. + * + * Ported from the standalone migration-tool's `src/export/supabase.ts`, on + * `Bun.sql` instead of `pg`. + * + * The database rather than the Admin API because **`encrypted_password` only + * exists here**. Supabase's API does not return password hashes, so an + * API-based export forces every user to reset their password; this one carries + * the bcrypt digests across. + */ + +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; + +/** + * `display_name` is coalesced into `first_name` here rather than in the + * transformer so a user who writes their own SQL sees the shape the + * transformer expects. + */ +const EXPORT_QUERY = ` + SELECT + id, + email, + email_confirmed_at, + encrypted_password, + phone, + phone_confirmed_at, + COALESCE( + raw_user_meta_data->>'display_name', + raw_user_meta_data->>'first_name', + raw_user_meta_data->>'name' + ) AS first_name, + raw_user_meta_data->>'last_name' AS last_name, + raw_user_meta_data, + raw_app_meta_data, + created_at + FROM auth.users + ORDER BY created_at +`; + +type SupabaseRow = Record & { + id?: unknown; + email?: string | null; + encrypted_password?: string | null; + raw_app_meta_data?: unknown; +}; + +/** Serializes values the JSON export cannot carry as-is. */ +function normalizeRow(row: SupabaseRow): Record { + const normalized: Record = {}; + + for (const [key, value] of Object.entries(row)) { + if (value === null || value === undefined) continue; + // Postgres returns timestamps as Date objects; the transformer parses + // strings, and JSON.stringify would otherwise bury the format difference. + normalized[key] = value instanceof Date ? value.toISOString() : value; + } + + return normalized; +} + +export async function fetchSupabaseUsers(client: DbClient): Promise { + return client.query(EXPORT_QUERY); +} + +export function buildSupabaseExport(rows: SupabaseRow[], dateTime: string) { + const users: Record[] = []; + const counts = { email: 0, emailConfirmed: 0, password: 0, phone: 0, firstName: 0, lastName: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + try { + users.push(normalizeRow(row)); + + if (row.email) counts.email++; + if (row.email_confirmed_at) counts.emailConfirmed++; + if (row.encrypted_password) counts.password++; + if (row.phone) counts.phone++; + if (row.first_name) counts.firstName++; + if (row.last_name) counts.lastName++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a confirmed email", count: counts.emailConfirmed }, + { label: "have a password hash", count: counts.password }, + { label: "have a phone number", count: counts.phone }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + ], + }; +} + +export async function exportSupabase(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, { + platform: "supabase", + envVar: "SUPABASE_DB_URL", + prompt: "Supabase Postgres connection string", + hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", + }); + + await withGutter("Exporting users from Supabase", async () => { + const dateTime = getDateTimeStamp(); + + const rows = await withSpinner("Reading auth.users", () => + withDbClient(dbUrl, "supabase", fetchSupabaseUsers), + ); + + const { users, coverage } = buildSupabaseExport(rows, dateTime); + const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("supabase")); + + reportExport({ + platform: "supabase", + userCount: users.length, + outputPath, + coverage, + transformerKey: "supabase", + }); + + if (users.length > 0) { + log.info( + "Password hashes are included — this is why the export reads the database rather than the Admin API.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts new file mode 100644 index 000000000..739833322 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -0,0 +1,335 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { BapiError } from "../../lib/errors.ts"; +import { + buildCreateUserBody, + importUsers, + normalizeErrorMessage, + readRetryAfter, + splitIdentifiers, +} from "./import-users.ts"; +import { getLogFilePath } from "./lib/logger.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import type { User } from "./types.ts"; + +const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurrencyLimit: 8 }; +const DATE_TIME = "2026-01-01T00:00:00"; + +const user = (overrides: Partial = {}): User => + ({ userId: "u1", email: "a@x.dev", ...overrides }) as User; + +describe("splitIdentifiers", () => { + test("promotes the first verified email and phone to primary", () => { + const result = splitIdentifiers( + user({ email: ["a@x.dev", "b@x.dev"], phone: ["+15555550100", "+15555550101"] }), + ); + expect(result.primaryEmail).toBe("a@x.dev"); + expect(result.additionalEmails).toEqual(["b@x.dev"]); + expect(result.primaryPhone).toBe("+15555550100"); + expect(result.additionalPhones).toEqual(["+15555550101"]); + }); + + test("merges the email and emailAddresses fields, deduping", () => { + const result = splitIdentifiers( + user({ email: "a@x.dev", emailAddresses: ["a@x.dev", "b@x.dev"] }), + ); + expect(result.primaryEmail).toBe("a@x.dev"); + expect(result.additionalEmails).toEqual(["b@x.dev"]); + }); + + test("drops an unverified identifier that is already verified", () => { + const result = splitIdentifiers( + user({ email: ["a@x.dev"], unverifiedEmailAddresses: ["a@x.dev", "c@x.dev"] }), + ); + expect(result.unverifiedEmails).toEqual(["c@x.dev"]); + }); + + test("copes with a user identified only by username", () => { + const result = splitIdentifiers({ userId: "u1", username: "alice" } as User); + expect(result.primaryEmail).toBeUndefined(); + expect(result.additionalEmails).toEqual([]); + }); +}); + +describe("buildCreateUserBody", () => { + test("maps the schema onto BAPI's snake_case body", () => { + const target = user({ + firstName: "Alice", + lastName: "Smith", + username: "alice", + createdAt: "2024-01-01T00:00:00.000Z", + publicMetadata: { plan: "pro" }, + createOrganizationsLimit: 3, + banned: true, + }); + const body = buildCreateUserBody(target, splitIdentifiers(target), true); + + expect(body).toMatchObject({ + external_id: "u1", + email_address: ["a@x.dev"], + first_name: "Alice", + last_name: "Smith", + username: "alice", + created_at: "2024-01-01T00:00:00.000Z", + public_metadata: { plan: "pro" }, + create_organizations_limit: 3, + banned: true, + }); + }); + + test("sends only the primary identifier; the rest are attached separately", () => { + const target = user({ email: ["a@x.dev", "b@x.dev"] }); + expect(buildCreateUserBody(target, splitIdentifiers(target), true).email_address).toEqual([ + "a@x.dev", + ]); + }); + + test("omits fields the source platform never recorded", () => { + const body = buildCreateUserBody(user(), splitIdentifiers(user()), true); + expect("first_name" in body).toBe(false); + expect("banned" in body).toBe(false); + expect("created_at" in body).toBe(false); + }); + + test("sends the password digest and hasher together", () => { + const target = user({ password: "digest", passwordHasher: "bcrypt" }); + const body = buildCreateUserBody(target, splitIdentifiers(target), true); + expect(body).toMatchObject({ password_digest: "digest", password_hasher: "bcrypt" }); + expect("skip_password_requirement" in body).toBe(false); + }); + + test.each([ + [true, true], + [false, false], + ])("skipPasswordRequirement=%p on a passwordless user -> flag present: %p", (skip, present) => { + const body = buildCreateUserBody(user(), splitIdentifiers(user()), skip); + expect("skip_password_requirement" in body).toBe(present); + }); +}); + +describe("readRetryAfter", () => { + const withHeader = (value: string) => + new BapiError(429, "{}", new Headers({ "retry-after": value })); + + test.each([ + ["12", 12], + ["0", undefined], + ["soon", undefined], + ])("Retry-After: %s -> %p", (header, expected) => { + expect(readRetryAfter(withHeader(header))).toBe(expected as number | undefined); + }); + + test("falls back to the error body's retryAfter meta", () => { + const error = new BapiError( + 429, + JSON.stringify({ + errors: [{ code: "rate_limit", message: "slow down", meta: { retryAfter: 7 } }], + }), + new Headers(), + ); + expect(readRetryAfter(error)).toBe(7); + }); + + test("returns undefined when neither source carries a value", () => { + expect(readRetryAfter(new BapiError(429, "{}", new Headers()))).toBeUndefined(); + }); +}); + +describe("normalizeErrorMessage", () => { + test("sorts field arrays so equivalent errors group together", () => { + const a = normalizeErrorMessage('["last_name" "first_name"] data does not match'); + const b = normalizeErrorMessage('["first_name" "last_name"] data does not match'); + expect(a).toBe(b); + expect(a).toBe('["first_name" "last_name"] data does not match'); + }); + + test("leaves messages without field arrays untouched", () => { + expect(normalizeErrorMessage("that email is taken")).toBe("that email is taken"); + }); +}); + +describe("importUsers", () => { + let workDir: string; + let originalCwd: string; + let originalFetch: typeof globalThis.fetch; + let requests: { method: string; url: string; body: unknown }[]; + + beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-import-")); + process.chdir(workDir); + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + }); + + beforeEach(() => { + requests = []; + fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + /** Installs a fetch that records every request and replies per `respond`. */ + function stub(respond: (url: string, attempt: number) => Response): void { + const attempts = new Map(); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + const attempt = (attempts.get(url) ?? 0) + 1; + attempts.set(url, attempt); + return respond(url, attempt); + }) as typeof fetch; + } + + const ok = (id: string) => new Response(JSON.stringify({ id }), { status: 200 }); + + const clerkError = (status: number, message: string, headers?: Record) => + new Response(JSON.stringify({ errors: [{ code: "err", message, long_message: message }] }), { + status, + headers, + }); + + const logEntries = () => + fs + .readFileSync(getLogFilePath("migration", DATE_TIME), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + + test("creates each user and reports them as successful", async () => { + stub(() => ok("user_created")); + + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ totalProcessed: 2, successful: 2, failed: 0 }); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + expect(logEntries().filter((e) => e.status === "success")).toHaveLength(2); + }); + + test("attaches additional and unverified identifiers after the user exists", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [ + user({ + email: ["a@x.dev", "b@x.dev"], + unverifiedEmailAddresses: ["c@x.dev"], + phone: ["+15555550100", "+15555550101"], + }), + ], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + const emails = requests.filter((r) => r.url.endsWith("/v1/email_addresses")); + expect(emails.map((r) => r.body)).toEqual([ + { user_id: "user_created", email_address: "b@x.dev", primary: false, verified: true }, + { user_id: "user_created", email_address: "c@x.dev", primary: false, verified: false }, + ]); + expect(requests.filter((r) => r.url.endsWith("/v1/phone_numbers"))).toHaveLength(1); + }); + + test("logs a failed additional identifier without failing the user", async () => { + stub((url) => + url.endsWith("/v1/email_addresses") + ? clerkError(422, "that email is taken") + : ok("user_created"), + ); + + const summary = await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + expect(logEntries().some((e) => e.status === "additional_email_error")).toBe(true); + }); + + test("records a failed user and keeps going", async () => { + stub((_url, attempt) => + attempt === 1 ? clerkError(422, "that email is taken") : ok("user_ok"), + ); + + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary.successful + summary.failed).toBe(2); + expect(summary.failed).toBe(1); + expect([...summary.errorBreakdown.values()]).toEqual([1]); + expect(logEntries().some((e) => e.status === "error" && e.code === "422")).toBe(true); + }); + + test("retries a 429 after the interval the server asked for", async () => { + stub((_url, attempt) => + attempt === 1 ? clerkError(429, "slow down", { "retry-after": "1" }) : ok("user_ok"), + ); + + const started = performance.now(); + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + expect(performance.now() - started).toBeGreaterThanOrEqual(900); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + expect(logEntries().some((e) => e.status === "429_retry")).toBe(true); + }); + + test("gives up after the retry ceiling and records the user as failed", async () => { + stub(() => clerkError(429, "slow down", { "retry-after": "1" })); + + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ successful: 0, failed: 1 }); + // One initial attempt plus MAX_RETRIES retries. + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(6); + expect(logEntries().some((e) => e.code === "429")).toBe(true); + }, 20_000); + + test("carries the validation failure count into the summary", async () => { + stub(() => ok("user_ok")); + + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + validationFailed: 4, + }); + + expect(summary.validationFailed).toBe(4); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts new file mode 100644 index 000000000..749142cf4 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -0,0 +1,347 @@ +/** + * Creates users in Clerk from a validated batch. + * + * Ported from the standalone migration-tool's `src/migrate/import-users.ts`, + * rewritten onto `bapiRequest` instead of `@clerk/backend`. Two things fall out + * of that move: + * + * - The request body is BAPI's snake_case shape directly, so `created_at` and + * `legal_accepted_at` stay RFC3339 strings rather than round-tripping + * through `Date`. + * - `banned`, `delete_self_enabled` and the organization limits are all + * accepted by `POST /v1/users`, so the follow-up `updateUser`/`banUser` + * calls the SDK version needed are gone. + * + * Run state is local to {@link importUsers} rather than module-level, so two + * runs in one process (or one test file) cannot see each other's counters. + */ + +import { bapiRequest } from "../../lib/bapi.ts"; +import { BapiError } from "../../lib/errors.ts"; +import type { SpinnerControls } from "../../lib/spinner.ts"; +import { errorLogger, importLogger } from "./lib/logger.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; +import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; +import type { ImportSummary, User } from "./types.ts"; + +// Re-exported for the tests and callers that grew up against this module. +export { readRetryAfter } from "./lib/retry.ts"; + +/** + * Groups error messages that differ only in field ordering, so the summary + * reports "12 users: [\"first_name\" \"last_name\"] ..." once instead of twice. + */ +export function normalizeErrorMessage(errorMessage: string): string { + let normalized = ""; + let lastCopiedIndex = 0; + let arrayStartIndex = -1; + + for (let i = 0; i < errorMessage.length; i++) { + const char = errorMessage[i]; + + if (arrayStartIndex === -1) { + if (char === "[") arrayStartIndex = i; + continue; + } + if (char !== "]") continue; + + normalized += errorMessage.slice(lastCopiedIndex, arrayStartIndex); + normalized += normalizeFieldArray(errorMessage.slice(arrayStartIndex + 1, i)); + lastCopiedIndex = i + 1; + arrayStartIndex = -1; + } + + return normalized + errorMessage.slice(lastCopiedIndex); +} + +function normalizeFieldArray(fields: string): string { + const fieldNames: string[] = []; + let current = ""; + + for (const char of fields) { + if (char === '"' || char === "'" || char.trim() === "") { + if (current.length > 0) { + fieldNames.push(current); + current = ""; + } + continue; + } + current += char; + } + if (current.length > 0) fieldNames.push(current); + + fieldNames.sort(); + return `[${fieldNames.map((name) => `"${name}"`).join(" ")}]`; +} + +function toArray(value: string | string[] | undefined): string[] { + if (!value) return []; + return Array.isArray(value) ? value : [value]; +} + +function dedupe(values: string[]): string[] { + const seen: string[] = []; + for (const value of values) { + if (value && !seen.includes(value)) seen.push(value); + } + return seen; +} + +type Identifiers = { + primaryEmail: string | undefined; + additionalEmails: string[]; + unverifiedEmails: string[]; + primaryPhone: string | undefined; + additionalPhones: string[]; + unverifiedPhones: string[]; +}; + +/** + * Splits a user's identifiers into the one that goes on `POST /v1/users` and + * the rest, which are attached afterwards. + */ +export function splitIdentifiers(user: User): Identifiers { + const verifiedEmails = dedupe([...toArray(user.email), ...toArray(user.emailAddresses)]); + const verifiedPhones = dedupe([...toArray(user.phone), ...toArray(user.phoneNumbers)]); + + return { + primaryEmail: verifiedEmails[0], + additionalEmails: verifiedEmails.slice(1), + unverifiedEmails: dedupe( + toArray(user.unverifiedEmailAddresses).filter((email) => !verifiedEmails.includes(email)), + ), + primaryPhone: verifiedPhones[0], + additionalPhones: verifiedPhones.slice(1), + unverifiedPhones: dedupe( + toArray(user.unverifiedPhoneNumbers).filter((phone) => !verifiedPhones.includes(phone)), + ), + }; +} + +/** + * Builds the `POST /v1/users` request body. + * + * Optional fields are omitted rather than sent as null so Clerk applies its own + * defaults for anything the source platform did not record. + */ +export function buildCreateUserBody( + user: User, + identifiers: Identifiers, + skipPasswordRequirement: boolean, +): Record { + const body: Record = { external_id: user.userId }; + + if (identifiers.primaryEmail) body.email_address = [identifiers.primaryEmail]; + if (identifiers.primaryPhone) body.phone_number = [identifiers.primaryPhone]; + if (user.firstName) body.first_name = user.firstName; + if (user.lastName) body.last_name = user.lastName; + if (user.username) body.username = user.username; + if (user.totpSecret) body.totp_secret = user.totpSecret; + if (user.backupCodes) body.backup_codes = user.backupCodes; + if (user.unsafeMetadata) body.unsafe_metadata = user.unsafeMetadata; + if (user.privateMetadata) body.private_metadata = user.privateMetadata; + if (user.publicMetadata) body.public_metadata = user.publicMetadata; + if (user.createdAt) body.created_at = user.createdAt; + if (user.legalAcceptedAt) body.legal_accepted_at = user.legalAcceptedAt; + if (user.skipLegalChecks !== undefined) body.skip_legal_checks = user.skipLegalChecks; + if (user.skipPasswordChecks !== undefined) body.skip_password_checks = user.skipPasswordChecks; + if (user.banned !== undefined) body.banned = user.banned; + if (user.bypassClientTrust !== undefined) body.bypass_client_trust = user.bypassClientTrust; + if (user.deleteSelfEnabled !== undefined) body.delete_self_enabled = user.deleteSelfEnabled; + if (user.createOrganizationEnabled !== undefined) { + body.create_organization_enabled = user.createOrganizationEnabled; + } + if (user.createOrganizationsLimit !== undefined) { + body.create_organizations_limit = user.createOrganizationsLimit; + } + + if (user.password && user.passwordHasher) { + body.password_digest = user.password; + body.password_hasher = user.passwordHasher; + } else if (skipPasswordRequirement) { + body.skip_password_requirement = true; + } + // Without a password and without skipPasswordRequirement, Clerk rejects the + // user — which is exactly what --require-password is asking for. + + return body; +} + +type CreateContext = { + secretKey: string; + schedule: ApiScheduler; + dateTime: string; +}; + +/** Attaches one extra identifier, logging (but not rethrowing) any failure. */ +async function attachIdentifier( + ctx: CreateContext, + userId: string, + clerkUserId: string, + kind: "email" | "phone", + value: string, + verified: boolean, +): Promise { + const path = kind === "email" ? "/v1/email_addresses" : "/v1/phone_numbers"; + const body = + kind === "email" + ? { user_id: clerkUserId, email_address: value, primary: false, verified } + : { user_id: clerkUserId, phone_number: value, primary: false, verified }; + + try { + await ctx.schedule(() => + bapiRequest({ + method: "POST", + path, + secretKey: ctx.secretKey, + body: JSON.stringify(body), + }), + ); + } catch (error) { + const label = `${verified ? "additional" : "unverified"} ${kind} ${value}`; + errorLogger( + { + userId, + status: `additional_${kind}_error`, + errors: [ + { + code: `additional_${kind}_failed`, + message: `Failed to add ${label}`, + longMessage: `Failed to add ${label}: ${(error as Error).message}`, + }, + ], + }, + ctx.dateTime, + ); + } +} + +/** Creates one user, then attaches any additional identifiers it carries. */ +async function createUser( + ctx: CreateContext, + user: User, + skipPasswordRequirement: boolean, +): Promise { + const identifiers = splitIdentifiers(user); + + const response = await ctx.schedule(() => + bapiRequest({ + method: "POST", + path: "/v1/users", + secretKey: ctx.secretKey, + body: JSON.stringify(buildCreateUserBody(user, identifiers, skipPasswordRequirement)), + }), + ); + + const clerkUserId = (response.body as { id?: string })?.id ?? ""; + + // Extra identifiers are best-effort: a duplicate secondary email should not + // undo a user who was otherwise imported successfully. + await Promise.all([ + ...identifiers.additionalEmails.map((email) => + attachIdentifier(ctx, user.userId, clerkUserId, "email", email, true), + ), + ...identifiers.unverifiedEmails.map((email) => + attachIdentifier(ctx, user.userId, clerkUserId, "email", email, false), + ), + ...identifiers.additionalPhones.map((phone) => + attachIdentifier(ctx, user.userId, clerkUserId, "phone", phone, true), + ), + ...identifiers.unverifiedPhones.map((phone) => + attachIdentifier(ctx, user.userId, clerkUserId, "phone", phone, false), + ), + ]); + + return clerkUserId; +} + +export type ImportUsersOptions = { + users: User[]; + secretKey: string; + limits: ResolvedLimits; + dateTime: string; + /** Allow users that carry no password. */ + skipPasswordRequirement?: boolean; + /** Carried into the summary so the report covers the whole file. */ + validationFailed?: number; + spinner?: SpinnerControls; +}; + +/** + * Imports every user, concurrently and within the instance's rate limit. + * + * A failed user is recorded and the run continues; a 429 backs off (honouring + * `Retry-After`) and retries up to {@link MAX_RETRIES} times. + */ +export async function importUsers(options: ImportUsersOptions): Promise { + const { + users, + secretKey, + limits, + dateTime, + skipPasswordRequirement = true, + validationFailed = 0, + spinner, + } = options; + + const total = users.length; + const errorBreakdown = new Map(); + let processed = 0; + let successful = 0; + let failed = 0; + + const ctx: CreateContext = { + secretKey, + dateTime, + schedule: createApiScheduler(limits.concurrencyLimit, limits.rateLimit), + }; + + const progress = () => + spinner?.update( + `Importing users: [${processed}/${total}] (${successful} succeeded, ${failed} failed)`, + ); + + const recordFailure = (userId: string, message: string, code: string) => { + failed++; + processed++; + const normalized = normalizeErrorMessage(message); + errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); + importLogger({ userId, status: "error", error: message, code }, dateTime); + progress(); + }; + + const processUser = async (user: User): Promise => { + try { + const clerkUserId = await retryOn429(() => createUser(ctx, user, skipPasswordRequirement), { + onRetry: ({ message }) => + errorLogger( + { + userId: user.userId, + status: "429_retry", + errors: [{ code: "rate_limit_retry", message, longMessage: message }], + }, + dateTime, + ), + }); + successful++; + processed++; + importLogger({ userId: user.userId, status: "success", clerkUserId }, dateTime); + progress(); + } catch (error) { + if (error instanceof RateLimitExceededError) { + recordFailure(user.userId, error.message, "429"); + return; + } + + const apiError = error as BapiError; + const message = apiError.longMessage ?? apiError.message ?? "Unknown error"; + recordFailure(user.userId, message, String(apiError.status ?? "unknown")); + } + }; + + progress(); + await Promise.all(users.map((user) => processUser(user))); + + return { totalProcessed: total, successful, failed, validationFailed, errorBreakdown }; +} diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts new file mode 100644 index 000000000..07454ad98 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -0,0 +1,208 @@ +import { describe, expect, test } from "bun:test"; +import { createProgram } from "../../cli-program.ts"; +import { exportPlatformKeys } from "./export/registry.ts"; +import { transformerKeys } from "./transformers/registry.ts"; + +function findCommand(names: string[]) { + let current = createProgram().commands.find((cmd) => cmd.name() === names[0]); + for (const name of names.slice(1)) { + current = current?.commands.find((cmd) => cmd.name() === name); + } + return current; +} + +describe("registerMigrate", () => { + test("registers migrate as a top-level command group", () => { + const migrate = findCommand(["migrate"]); + expect(migrate).toBeDefined(); + expect(migrate?.description()).toContain("Migrate users"); + }); + + test("registers the run subcommand", () => { + expect(findCommand(["migrate", "run"])).toBeDefined(); + }); + + // Bare `clerk migrate` dispatches to `migrate run`, mirroring how bare + // `clerk deploy` dispatches to `deploy run`. + test("makes run the default subcommand, so bare `clerk migrate` starts the wizard", () => { + const run = findCommand(["migrate", "run"]); + expect(run as unknown as { _defaultCommandName?: unknown }).toBeDefined(); + const migrate = findCommand(["migrate"]) as unknown as { _defaultCommandName?: string }; + expect(migrate._defaultCommandName).toBe("run"); + }); + + test("keeps run visible in help, unlike deploy's hidden default", () => { + expect(findCommand(["migrate", "run"])?.parent?.commands.map((c) => c.name())).toContain("run"); + expect( + (findCommand(["migrate", "run"]) as unknown as { _hidden?: boolean })._hidden, + ).toBeFalsy(); + }); + + test.each([ + "--transformer", + "--file", + "--resume-after", + "--require-password", + "--skip-unsupported-providers", + "--firebase-signer-key", + "--firebase-salt-separator", + "--firebase-rounds", + "--firebase-mem-cost", + "--yes", + "--secret-key", + "--clerk-secret-key", + "--app", + "--instance", + ])("migrate run accepts %s", (flag) => { + const flags = findCommand(["migrate", "run"])?.options.map((option) => option.long); + expect(flags).toContain(flag); + }); + + test.each([[["transformers"]], [["transformers", "list"]]])("registers migrate %p", (names) => { + expect(findCommand(["migrate", ...names])).toBeDefined(); + }); + + test.each([ + [["export"]], + [["export", "clerk"]], + [["export", "auth0"]], + [["export", "supabase"]], + [["export", "authjs"]], + [["export", "betterauth"]], + [["export", "firebase"]], + ])("registers migrate %p", (names) => { + expect(findCommand(["migrate", ...names])).toBeDefined(); + }); + + // Bare `migrate export` runs the picker rather than defaulting to a + // platform, so nobody exports from the wrong place by pressing enter. + test("leaves export with no default subcommand", () => { + const group = findCommand(["migrate", "export"]) as unknown as { + _defaultCommandName?: string; + }; + expect(group._defaultCommandName).toBeFalsy(); + }); + + test("registers an export subcommand per registered platform", () => { + const registered = findCommand(["migrate", "export"])?.commands.map((c) => c.name()); + for (const key of exportPlatformKeys()) expect(registered).toContain(key); + }); + + test.each(["--output", "--secret-key", "--app", "--instance"])( + "export clerk accepts %s", + (flag) => { + expect(findCommand(["migrate", "export", "clerk"])?.options.map((o) => o.long)).toContain( + flag, + ); + }, + ); + + test.each(["--domain", "--client-id", "--client-secret", "--output"])( + "export auth0 accepts %s", + (flag) => { + expect(findCommand(["migrate", "export", "auth0"])?.options.map((o) => o.long)).toContain( + flag, + ); + }, + ); + + test.each(["supabase", "authjs", "betterauth"])("export %s accepts --db-url", (platform) => { + expect(findCommand(["migrate", "export", platform])?.options.map((o) => o.long)).toContain( + "--db-url", + ); + }); + + test("export firebase accepts --service-account", () => { + expect(findCommand(["migrate", "export", "firebase"])?.options.map((o) => o.long)).toContain( + "--service-account", + ); + }); + + test("documents the default output location in help", () => { + expect(findCommand(["migrate", "export", "clerk"])?.description()).toContain( + "./exports/clerk-export.json", + ); + expect(findCommand(["migrate", "export", "auth0"])?.description()).toContain( + "./exports/auth0-export.json", + ); + }); + + test("makes list the default transformers subcommand", () => { + const group = findCommand(["migrate", "transformers"]) as unknown as { + _defaultCommandName?: string; + }; + expect(group._defaultCommandName).toBe("list"); + }); + + test.each(["--json", "--transformer-file"])("transformers list accepts %s", (flag) => { + expect(findCommand(["migrate", "transformers", "list"])?.options.map((o) => o.long)).toContain( + flag, + ); + }); + + test("migrate run accepts --transformer-file", () => { + expect(findCommand(["migrate", "run"])?.options.map((o) => o.long)).toContain( + "--transformer-file", + ); + }); + + // Flat rather than under a noun group: it is the one command in this tree + // that destroys data in Clerk. + test("registers delete as a direct subcommand of migrate", () => { + expect(findCommand(["migrate", "delete"])).toBeDefined(); + expect(findCommand(["migrate", "delete"])?.description()).toContain("last migration"); + }); + + test.each(["--yes", "--secret-key", "--clerk-secret-key", "--app", "--instance"])( + "migrate delete accepts %s", + (flag) => { + expect(findCommand(["migrate", "delete"])?.options.map((o) => o.long)).toContain(flag); + }, + ); + + test.each([[["logs"]], [["logs", "list"]], [["logs", "clean"]], [["logs", "convert"]]])( + "registers migrate %p", + (names) => { + expect(findCommand(["migrate", ...names])).toBeDefined(); + }, + ); + + // Listing is read-only, so it is safe as the default for a bare + // `clerk migrate logs`. + test("makes list the default logs subcommand", () => { + const logs = findCommand(["migrate", "logs"]) as unknown as { _defaultCommandName?: string }; + expect(logs._defaultCommandName).toBe("list"); + }); + + test.each([ + [["logs", "list"], "--json"], + [["logs", "clean"], "--yes"], + [["logs", "convert"], "--all"], + ])("%s accepts %s", (names, flag) => { + expect(findCommand(["migrate", ...names])?.options.map((option) => option.long)).toContain( + flag, + ); + }); + + test("logs convert takes variadic file positionals", () => { + const args = findCommand(["migrate", "logs", "convert"])?.registeredArguments; + expect(args?.[0]?.variadic).toBe(true); + expect(args?.[0]?.required).toBe(false); + }); + + test("constrains --transformer to the registered transformers, for validation and completion", () => { + const option = findCommand(["migrate", "run"])?.options.find((o) => o.long === "--transformer"); + // Tracks the registry so adding a platform needs no edit here. + expect(option?.argChoices).toEqual(transformerKeys()); + }); + + test.each([ + ["-t", "--transformer"], + ["-f", "--file"], + ["-r", "--resume-after"], + ["-y", "--yes"], + ])("exposes %s as the short form of %s", (short, long) => { + const option = findCommand(["migrate", "run"])?.options.find((o) => o.long === long); + expect(option?.short).toBe(short); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts new file mode 100644 index 000000000..3d34ca55c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -0,0 +1,134 @@ +import { createOption } from "@commander-js/extra-typings"; +import type { Program } from "../../cli-program.ts"; +import { parseIntegerOption } from "../../lib/option-parsers.ts"; +import { deleteMigration } from "./delete.ts"; +import { registerMigrateExport } from "./export/index.ts"; +import { registerMigrateLogs } from "./logs/index.ts"; +import { run } from "./run.ts"; +import { list as transformersList } from "./transformers/list.ts"; +import { transformerKeys } from "./transformers/registry.ts"; + +const migrate = { run, delete: deleteMigration, transformersList }; + +export function registerMigrate(program: Program): void { + const migrateCommand = program + .command("migrate") + .description("Migrate users into Clerk from another auth provider") + .setExamples([ + { + command: "clerk migrate", + description: "Walk through a migration interactively", + }, + { + command: "clerk migrate run -y --transformer clerk --file users.json", + description: "Import users from a Clerk export", + }, + ]); + + // `isDefault` so bare `clerk migrate` runs the wizard, mirroring how bare + // `clerk deploy` dispatches to `deploy run`. Not hidden: unlike deploy's, + // this subcommand is documented and carries every flag. + migrateCommand + .command("run", { isDefault: true }) + .description("Import users from an exported JSON or CSV file") + .addOption( + createOption( + "-t, --transformer ", + "Source platform the file was exported from", + ).choices(transformerKeys()), + ) + .option( + "--transformer-file ", + "Path to a transformer you wrote, for a platform with no built-in", + ) + .option("-f, --file ", "Path to the exported user data (JSON or CSV)") + .option("-r, --resume-after ", "Skip every user up to and including this source ID") + .option("--require-password", "Import only users that have a password") + .option( + "--skip-unsupported-providers", + "Supabase: skip users whose only social provider is not enabled in Clerk", + ) + .option("--firebase-signer-key ", "Firebase base64 signer key") + .option("--firebase-salt-separator ", "Firebase base64 salt separator") + .option("--firebase-rounds ", "Firebase scrypt rounds", (value) => + parseIntegerOption(value, "--firebase-rounds", { min: 1 }), + ) + .option("--firebase-mem-cost ", "Firebase scrypt memory cost", (value) => + parseIntegerOption(value, "--firebase-mem-cost", { min: 1 }), + ) + .option("-y, --yes", "Skip the confirmation prompt") + .option("--secret-key ", "Backend API secret key to use") + .option("--clerk-secret-key ", "Deprecated alias for --secret-key") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate run -y --transformer clerk --file users.json", + description: "Import a Clerk Dashboard export", + }, + { + command: "clerk migrate run -y -t clerk -f users.csv --require-password", + description: "Import only the users that carry a password digest", + }, + { + command: "clerk migrate run -y -t clerk -f users.json -r user_2x9k", + description: "Resume a partial migration after the last imported user", + }, + { + command: "clerk migrate run -y -t supabase -f users.json --skip-unsupported-providers", + description: "Skip Supabase users whose only provider is not enabled in Clerk", + }, + ]) + .action((_opts, cmd) => + migrate.run(cmd.optsWithGlobals() as Parameters[0]), + ); + + // Flat, not under a noun group: this is the one command in the tree that + // destroys data in Clerk, and it is worth keeping short and prominent. + migrateCommand + .command("delete") + .description("Delete the users created by the last migration in this directory") + .option("-y, --yes", "Skip the confirmation prompt") + .option("--secret-key ", "Backend API secret key to use") + .option("--clerk-secret-key ", "Deprecated alias for --secret-key") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate delete", + description: "Undo the last migration after confirming", + }, + { command: "clerk migrate delete -y", description: "Undo without prompting" }, + ]) + .action((_opts, cmd) => + migrate.delete(cmd.optsWithGlobals() as Parameters[0]), + ); + + registerMigrateExport(migrateCommand); + + // A compiled binary has no source tree to grep, so the available mappings + // need a command rather than only appearing in the interactive picker. + const transformersCommand = migrateCommand + .command("transformers") + .description("Inspect the available source-platform transformers"); + + transformersCommand + .command("list", { isDefault: true }) + .description("List the built-in transformers, and any loaded from a file") + .option("--json", "Output as JSON") + .option("--transformer-file ", "Also list a transformer you wrote") + .setExamples([ + { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { + command: "clerk migrate transformers list --transformer-file ./my-transformer.ts", + description: "Include one you wrote", + }, + ]) + .action((_opts, cmd) => + migrate.transformersList( + cmd.optsWithGlobals() as Parameters[0], + ), + ); + + registerMigrateLogs(migrateCommand); +} diff --git a/packages/cli-core/src/commands/migrate/lib/analysis.test.ts b/packages/cli-core/src/commands/migrate/lib/analysis.test.ts new file mode 100644 index 000000000..322c2d753 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/analysis.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "bun:test"; +import { analyzeFields, hasValue } from "./analysis.ts"; + +describe("hasValue", () => { + test.each([ + ["a string", "x", true], + ["zero", 0, true], + ["false", false, true], + ["a populated array", ["a"], true], + ["an object", {}, true], + ["an empty string", "", false], + ["an empty array", [], false], + ["null", null, false], + ["undefined", undefined, false], + ])("treats %s as present: %p", (_label, value, expected) => { + expect(hasValue(value)).toBe(expected); + }); +}); + +describe("analyzeFields", () => { + test("returns zeroed counts for an empty file", () => { + const result = analyzeFields([]); + expect(result.totalUsers).toBe(0); + expect(result.identifiers.hasAnyIdentifier).toBe(0); + expect(result.fieldCounts).toEqual({}); + }); + + test("counts each identifier kind separately", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev" }, + { userId: "2", unverifiedEmailAddresses: ["b@x.dev"] }, + { userId: "3", phone: "+15555550100" }, + { userId: "4", unverifiedPhoneNumbers: ["+15555550101"] }, + { userId: "5", username: "carol" }, + ]); + + expect(result.identifiers).toMatchObject({ + verifiedEmails: 1, + unverifiedEmails: 1, + verifiedPhones: 1, + unverifiedPhones: 1, + username: 1, + hasAnyIdentifier: 5, + }); + }); + + test("counts emailAddresses towards verified emails", () => { + const result = analyzeFields([{ userId: "1", emailAddresses: ["a@x.dev"] }]); + expect(result.identifiers.verifiedEmails).toBe(1); + }); + + test("counts a user with several identifiers once", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev", phone: "+15555550100", username: "ada" }, + ]); + expect(result.identifiers.hasAnyIdentifier).toBe(1); + expect(result.identifiers.verifiedEmails).toBe(1); + expect(result.identifiers.verifiedPhones).toBe(1); + }); + + // These users cannot be imported under any instance configuration, which is + // what makes the count worth surfacing separately. + test("counts users carrying no identifier at all", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev" }, + { userId: "2", firstName: "Nobody" }, + { userId: "3" }, + ]); + expect(result.totalUsers).toBe(3); + expect(result.identifiers.hasAnyIdentifier).toBe(1); + }); + + test("counts the analyzed non-identifier fields", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev", firstName: "Ada", password: "d", totpSecret: "s" }, + { userId: "2", email: "b@x.dev", firstName: "Grace" }, + { userId: "3", email: "c@x.dev", lastName: "Hopper" }, + ]); + expect(result.fieldCounts).toEqual({ + firstName: 2, + lastName: 1, + password: 1, + totpSecret: 1, + }); + }); + + test("omits fields no user carries, rather than reporting them as zero", () => { + const result = analyzeFields([{ userId: "1", email: "a@x.dev" }]); + expect("password" in result.fieldCounts).toBe(false); + }); + + test("does not count an empty value as present", () => { + const result = analyzeFields([{ userId: "1", email: "a@x.dev", firstName: "", lastName: [] }]); + expect(result.fieldCounts.firstName).toBeUndefined(); + expect(result.fieldCounts.lastName).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/analysis.ts b/packages/cli-core/src/commands/migrate/lib/analysis.ts new file mode 100644 index 000000000..848ec1d76 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/analysis.ts @@ -0,0 +1,80 @@ +/** + * Counts what an import file actually contains, per field. + * + * Ported from the standalone migration-tool's `src/lib/analysis.ts`. Runs on + * transformed-but-unvalidated users so the report describes the whole file, + * including the rows that will be skipped. + */ + +import type { User } from "../types.ts"; + +/** Non-identifier fields the readiness report reports coverage for. */ +export const ANALYZED_FIELDS = [ + { key: "firstName", label: "First name" }, + { key: "lastName", label: "Last name" }, + { key: "password", label: "Password" }, + { key: "totpSecret", label: "TOTP secret" }, +] as const; + +export type IdentifierCounts = { + verifiedEmails: number; + unverifiedEmails: number; + verifiedPhones: number; + unverifiedPhones: number; + username: number; + /** Users with at least one identifier — the rest cannot be imported at all. */ + hasAnyIdentifier: number; +}; + +export type FieldAnalysis = { + identifiers: IdentifierCounts; + totalUsers: number; + fieldCounts: Record; +}; + +/** True for anything with real content — `0` and `false` count, `""` and `[]` do not. */ +export function hasValue(value: unknown): boolean { + if (value === undefined || value === null || value === "") return false; + if (Array.isArray(value)) return value.length > 0; + return true; +} + +export function analyzeFields(users: (User | Record)[]): FieldAnalysis { + const identifiers: IdentifierCounts = { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: 0, + }; + const fieldCounts: Record = {}; + + for (const entry of users) { + const user = entry as Record; + + for (const field of ANALYZED_FIELDS) { + if (hasValue(user[field.key])) { + fieldCounts[field.key] = (fieldCounts[field.key] ?? 0) + 1; + } + } + + const verifiedEmail = hasValue(user.email) || hasValue(user.emailAddresses); + const unverifiedEmail = hasValue(user.unverifiedEmailAddresses); + const verifiedPhone = hasValue(user.phone) || hasValue(user.phoneNumbers); + const unverifiedPhone = hasValue(user.unverifiedPhoneNumbers); + const username = hasValue(user.username); + + if (verifiedEmail) identifiers.verifiedEmails++; + if (unverifiedEmail) identifiers.unverifiedEmails++; + if (verifiedPhone) identifiers.verifiedPhones++; + if (unverifiedPhone) identifiers.unverifiedPhones++; + if (username) identifiers.username++; + + if (verifiedEmail || unverifiedEmail || verifiedPhone || unverifiedPhone || username) { + identifiers.hasAnyIdentifier++; + } + } + + return { identifiers, totalUsers: users.length, fieldCounts }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts new file mode 100644 index 000000000..a94479cd9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts @@ -0,0 +1,85 @@ +import { test, expect, describe, mock, beforeEach, afterAll } from "bun:test"; +import { stubFetch, useCaptureLog } from "../../../test/lib/stubs.ts"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { fetchInstanceSettings } from "./clerk-config.ts"; + +const USER_SETTINGS = { + attributes: { email_address: { enabled: true, required: true } }, +} as unknown as UserSettingsJSON; + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +describe("fetchInstanceSettings", () => { + const originalFetch = globalThis.fetch; + useCaptureLog(); + const mockFetch = mock(); + + beforeEach(() => { + mockFetch.mockReset(); + stubFetch(mockFetch); + }); + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + /** Routes the three hops: BAPI domains → FAPI dev browser → FAPI environment. */ + function route(domains: unknown): void { + mockFetch.mockImplementation((input: string | URL) => { + const url = String(input); + if (url.includes("/v1/domains")) return Promise.resolve(json({ data: domains })); + if (url.includes("/v1/dev_browser")) return Promise.resolve(json({ token: "jwt" })); + if (url.includes("/v1/environment")) { + return Promise.resolve(json({ user_settings: USER_SETTINGS })); + } + throw new Error(`unexpected request: ${url}`); + }); + } + + test("reads settings off the primary domain's Frontend API", async () => { + route([{ is_satellite: false, frontend_api_url: "https://clerk.example.com" }]); + + expect(await fetchInstanceSettings("sk_test_abc")).toEqual(USER_SETTINGS); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.some((url) => url.includes("clerk.example.com/v1/dev_browser"))).toBe(true); + expect(urls.some((url) => url.includes("clerk.example.com/v1/environment"))).toBe(true); + }); + + test("prefers the primary domain over a satellite", async () => { + route([ + { is_satellite: true, frontend_api_url: "https://satellite.example.com" }, + { is_satellite: false, frontend_api_url: "https://clerk.example.com" }, + ]); + + await fetchInstanceSettings("sk_test_abc"); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.every((url) => !url.includes("satellite.example.com"))).toBe(true); + }); + + test("skips the dev browser bootstrap for a production key", async () => { + route([{ is_satellite: false, frontend_api_url: "https://clerk.example.com" }]); + + expect(await fetchInstanceSettings("sk_live_abc")).toEqual(USER_SETTINGS); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.some((url) => url.includes("/v1/dev_browser"))).toBe(false); + }); + + // `null` means "unknown", so callers degrade rather than treating a failed + // lookup as "nothing is enabled". + test("returns null when no domain names a Frontend API URL", async () => { + route([{ is_satellite: false }]); + expect(await fetchInstanceSettings("sk_test_abc")).toBeNull(); + }); + + test("returns null when the domains lookup fails", async () => { + mockFetch.mockResolvedValue(new Response("nope", { status: 401 })); + expect(await fetchInstanceSettings("sk_test_abc")).toBeNull(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts new file mode 100644 index 000000000..7b6cd7dfa --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -0,0 +1,119 @@ +/** + * The destination instance's live user settings: which identifiers it accepts + * and requires, and which social providers it has enabled. + * + * Ported from the standalone migration-tool's `src/lib/clerk.ts`, rewritten + * onto the CLI's own primitives: the FAPI host comes from BAPI `/v1/domains` + * — a secret key is all `migrate run` is given — and the settings come from + * `lib/fapi.ts` rather than a bespoke fetch. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import { + bootstrapDevBrowser, + fetchUserSettings, + type UserSettingsJSON, +} from "../../../lib/fapi.ts"; +import { log } from "../../../lib/log.ts"; +import { detectInstanceType } from "./instance.ts"; + +/** + * Supabase provider keys whose Clerk strategy is not simply `oauth_`. + * + * Everything not listed here maps by prefix, which covers google, github, + * discord, spotify, twitch, notion, figma, gitlab, bitbucket and the rest. + */ +const CLERK_STRATEGY_ALIASES: Record = { + azure: "oauth_microsoft", + twitter: "oauth_x", + slack_oidc: "oauth_slack", + fly: "oauth_fly", +}; + +/** Supabase's provider key as Clerk's OAuth strategy name. */ +export function toClerkStrategy(provider: string): string { + return CLERK_STRATEGY_ALIASES[provider] ?? `oauth_${provider}`; +} + +/** Human label for a provider key, for report output. */ +export function providerLabel(provider: string): string { + const special: Record = { + github: "GitHub", + gitlab: "GitLab", + linkedin_oidc: "LinkedIn (OIDC)", + slack_oidc: "Slack (OIDC)", + twitter: "Twitter (X)", + azure: "Microsoft (Azure)", + workos: "WorkOS", + fly: "Fly.io", + }; + return special[provider] ?? provider.charAt(0).toUpperCase() + provider.slice(1); +} + +/** + * The Frontend API host of the instance a secret key addresses. + * + * `/v1/instance` carries no publishable key — for any instance, linked or not + * — so the primary domain's `frontend_api_url` is the only route from a secret + * key to the host its settings live behind. Every instance has at least one + * domain; satellites share the primary's Frontend API, so ordering only + * matters for tidiness. + */ +async function fetchFapiHost(secretKey: string): Promise { + const response = await bapiRequest({ method: "GET", path: "/v1/domains", secretKey }); + const domains = (response.body as { data?: unknown })?.data; + if (!Array.isArray(domains)) return null; + + const primary = + domains.find((domain) => !(domain as { is_satellite?: boolean }).is_satellite) ?? domains[0]; + const frontendApiUrl = (primary as { frontend_api_url?: unknown })?.frontend_api_url; + if (typeof frontendApiUrl !== "string" || !frontendApiUrl) return null; + + return new URL(frontendApiUrl).host; +} + +/** + * Fetches the user settings for the instance a secret key addresses. + * + * @returns The settings, or `null` when they could not be read. Callers must + * treat `null` as "unknown" rather than as "nothing is enabled" — the + * readiness report degrades to a note, and provider skipping stands down. + */ +export async function fetchInstanceSettings(secretKey: string): Promise { + try { + const fapiHost = await fetchFapiHost(secretKey); + if (!fapiHost) { + log.debug("migrate: no domain on this instance named a Frontend API URL"); + return null; + } + + // Development FAPI rejects an environment request without a dev browser JWT. + const jwt = + detectInstanceType(secretKey) === "dev" ? await bootstrapDevBrowser(fapiHost) : undefined; + return await fetchUserSettings(fapiHost, jwt ? { jwt } : {}); + } catch (error) { + log.debug( + `migrate: could not read instance settings: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return null; + } +} + +/** The enabled social strategies (`oauth_google`, …) in a settings payload. */ +export function enabledSocialProviders(settings: UserSettingsJSON): string[] { + return Object.entries(settings.social ?? {}) + .filter(([, value]) => value?.enabled) + .map(([strategy]) => strategy); +} + +/** + * Convenience wrapper for callers that only need the enabled strategies. + * + * @returns `null` when the instance settings could not be read. + */ +export async function fetchEnabledSocialProviders(secretKey: string): Promise { + const settings = await fetchInstanceSettings(secretKey); + return settings ? enabledSocialProviders(settings) : null; +} diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts new file mode 100644 index 000000000..3fc969ee5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -0,0 +1,237 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { + createDbClient, + describeDbError, + detectDbType, + redactConnectionString, + sqlitePath, + withDbClient, +} from "./db.ts"; + +let workDir: string; +let dbPath: string; + +beforeAll(() => { + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-db-"))); + dbPath = path.join(workDir, "test.sqlite"); + + const db = new Database(dbPath, { create: true }); + db.run(`CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER)`); + db.run(`INSERT INTO "user" VALUES (?, ?, ?)`, ["u1", "a@x.dev", 1]); + db.run(`INSERT INTO "user" VALUES (?, ?, ?)`, ["u2", "b@x.dev", 0]); + db.close(); +}); + +afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("detectDbType", () => { + test.each([ + ["postgres://u:p@h/db", "postgres"], + ["postgresql://u:p@h/db", "postgres"], + ["POSTGRES://u:p@h/db", "postgres"], + ["mysql://u:p@h/db", "mysql"], + ["mysql2://u:p@h/db", "mysql"], + ["./db.sqlite", "sqlite"], + ["file:./db.sqlite", "sqlite"], + ["/abs/path.db", "sqlite"], + [" postgres://u:p@h/db ", "postgres"], + ])("%s -> %s", (input, expected) => { + expect(detectDbType(input)).toBe(expected as never); + }); +}); + +describe("redactConnectionString", () => { + test.each([ + ["postgres://user:secret@host:5432/db", "postgres://***@host:5432/db"], + ["mysql://root:hunter2@127.0.0.1:3306/app", "mysql://***@127.0.0.1:3306/app"], + ["postgres://host/db", "postgres://host/db"], + ])("%s -> %s", (input, expected) => { + expect(redactConnectionString(input)).toBe(expected); + }); + + // An unencoded `@` in the password is the most common mistake, and it is + // exactly when the string ends up in an error message. Matching the first + // `@` would leave the rest of the password visible. + test("redacts a password containing an unencoded @", () => { + const redacted = redactConnectionString("postgres://user:pa@ss@host/db"); + expect(redacted).toBe("postgres://***@host/db"); + expect(redacted).not.toContain("ss"); + }); + + test("redacts a password containing a colon", () => { + expect(redactConnectionString("postgres://user:a:b:c@host/db")).toBe("postgres://***@host/db"); + }); + + test.each([["./db.sqlite"], ["/var/data/app.db"], ["file:./local.sqlite"]])( + "leaves the credential-free path %s alone", + (input) => { + expect(redactConnectionString(input)).toBe(input); + }, + ); +}); + +describe("sqlitePath", () => { + test.each([ + ["./db.sqlite", "./db.sqlite"], + ["file:./db.sqlite", "./db.sqlite"], + ["file:/abs/db.sqlite", "/abs/db.sqlite"], + ["./db.sqlite?mode=ro", "./db.sqlite"], + [" ./db.sqlite ", "./db.sqlite"], + ])("%s -> %s", (input, expected) => { + expect(sqlitePath(input)).toBe(expected); + }); +}); + +describe("a sqlite client", () => { + test("connects and queries", async () => { + const client = await createDbClient(dbPath); + try { + const rows = await client.query<{ id: string }>(`SELECT id FROM "user" ORDER BY id`); + expect(rows.map((row) => row.id)).toEqual(["u1", "u2"]); + } finally { + await client.close(); + } + }); + + test("binds parameters", async () => { + const client = await createDbClient(dbPath); + try { + const rows = await client.query<{ email: string }>(`SELECT email FROM "user" WHERE id = ?`, [ + "u2", + ]); + expect(rows[0]?.email).toBe("b@x.dev"); + } finally { + await client.close(); + } + }); + + test("reports its dialect's placeholder and quoting", async () => { + const client = await createDbClient(dbPath); + try { + expect(client.dbType).toBe("sqlite"); + expect(client.placeholder(1)).toBe("?"); + expect(client.quote("user")).toBe('"user"'); + } finally { + await client.close(); + } + }); + + test("accepts a file: URL", async () => { + const client = await createDbClient(`file:${dbPath}`); + try { + expect(await client.query(`SELECT 1 AS n`)).toHaveLength(1); + } finally { + await client.close(); + } + }); + + // bun:sqlite opens lazily, so without an explicit probe a missing file would + // surface at the first real query, long after "connecting" finished. + test("fails at connect time when the file is missing, not mid-export", async () => { + await expect(createDbClient(path.join(workDir, "nope.sqlite"))).rejects.toThrow(CliError); + }); + + test("names the file in the failure", async () => { + await expect(createDbClient(path.join(workDir, "nope.sqlite"))).rejects.toThrow( + /Could not open the SQLite file/, + ); + }); +}); + +describe("withDbClient", () => { + test("returns the work's value", async () => { + expect(await withDbClient(dbPath, undefined, async () => "done")).toBe("done"); + }); + + test("closes the client even when the work throws", async () => { + // A leaked handle keeps the process alive after the export has written its + // file, which reads as a hang. + await expect( + withDbClient(dbPath, undefined, async () => { + throw new Error("boom"); + }), + ).rejects.toThrow(/boom/); + + // The file is still usable, so nothing is holding it open. + expect(await withDbClient(dbPath, undefined, async () => "reopened")).toBe("reopened"); + }); + + test("attaches a hint to a query failure, not just a connection failure", async () => { + await expect( + withDbClient(dbPath, undefined, (client) => client.query(`SELECT * FROM missing_table`)), + ).rejects.toThrow(/expected table was not found/); + }); + + test("passes a CliError through unchanged", async () => { + await expect( + withDbClient(dbPath, undefined, async () => { + throw new CliError("already explained"); + }), + ).rejects.toThrow(/already explained/); + }); +}); + +describe("describeDbError", () => { + const withCode = (code: string, message = "") => Object.assign(new Error(message), { code }); + + // Bun reports an unreachable host and a closed port identically, as + // "Connection closed" — precisely where a bare driver error helps least. + test.each([ + ["ERR_POSTGRES_CONNECTION_CLOSED", "Connection closed"], + ["ERR_MYSQL_CONNECTION_CLOSED", "Connection closed"], + ])("turns %s into host/port guidance", (code, message) => { + expect(describeDbError(withCode(code, message))).toMatch(/Check the host and port/); + }); + + test("gives Supabase the IPv4 add-on hint, which is the usual cause there", () => { + const hint = describeDbError( + withCode("ERR_POSTGRES_CONNECTION_CLOSED", "Connection closed"), + "supabase", + ); + expect(hint).toMatch(/pooler connection string/); + expect(hint).toMatch(/IPv4/); + }); + + test.each([ + ['password authentication failed for user "postgres"'], + ["Access denied for user 'root'@'localhost' (using password: YES)"], + ])("recognizes the rejected credentials in %p", (message) => { + expect(describeDbError(new Error(message))).toMatch(/rejected those credentials/); + }); + + test.each([ + ['relation "auth.users" does not exist'], + ["no such table: user"], + ["permission denied for table users"], + ])("recognizes the missing table in %p", (message) => { + expect(describeDbError(new Error(message))).toMatch(/table was not found|cannot read it/); + }); + + test("points Supabase at Auth being enabled and the postgres role", () => { + const hint = describeDbError(new Error('relation "auth.users" does not exist'), "supabase"); + expect(hint).toMatch(/Supabase Auth is enabled/); + expect(hint).toMatch(/postgres` role/); + }); + + test("recognizes an unopenable SQLite file", () => { + expect(describeDbError(new Error("unable to open database file"))).toMatch( + /Could not open the SQLite file/, + ); + }); + + test("still says something useful for an error it does not recognize", () => { + expect(describeDbError(new Error("something odd"))).toMatch(/Check the connection string/); + }); + + test("never echoes the error's own text, which could carry a connection string", () => { + const hint = describeDbError(new Error("failed for postgres://user:secret@host/db")); + expect(hint).not.toContain("secret"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts new file mode 100644 index 000000000..d048a3348 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -0,0 +1,227 @@ +/** + * One query interface over Postgres, MySQL and SQLite. + * + * Rewritten from the standalone migration-tool's `src/lib/db.ts`, which used + * `pg`, `mysql2` and `better-sqlite3`. None of those belong in a statically + * compiled binary — `better-sqlite3` is a native addon outright — so this runs + * on `Bun.sql` (Postgres and MySQL) and `bun:sqlite`, both built into the + * runtime. That swap is the entire reason the `engines.bun` floor exists. + * + * **Placeholders are not unified.** `Bun.sql` passes the query through to each + * server as written, so Postgres wants `$1` and MySQL wants `?` — verified + * against both. Rather than rewrite SQL strings (Postgres uses `?` as a JSONB + * operator, so a naive rewriter would corrupt real queries), callers ask the + * client for the placeholder and the identifier quoting they need. They already + * build per-dialect SQL for table casing, so this adds no new branching. + */ + +import { Database } from "bun:sqlite"; +import { SQL } from "bun"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; + +export type DbType = "postgres" | "mysql" | "sqlite"; + +export interface DbClient { + dbType: DbType; + query>(sql: string, params?: unknown[]): Promise; + /** The bind placeholder for the 1-indexed `position`. */ + placeholder(position: number): string; + /** Quotes an identifier for this dialect. */ + quote(identifier: string): string; + close(): Promise; +} + +/** + * Reads the database type from a connection string. + * + * Anything that is not a recognized URL scheme is treated as a SQLite path, + * matching how the standalone tool behaved and how users actually pass + * `./db.sqlite`. + */ +export function detectDbType(connectionString: string): DbType { + const lower = connectionString.trim().toLowerCase(); + if (lower.startsWith("postgresql://") || lower.startsWith("postgres://")) return "postgres"; + if (lower.startsWith("mysql://") || lower.startsWith("mysql2://")) return "mysql"; + return "sqlite"; +} + +/** + * Replaces any credentials in a connection string with `***`. + * + * Connection strings reach the CLI on the command line and end up in error + * messages and `--verbose` output. Bun's own errors do not echo them, and + * nothing here should either. + */ +export function redactConnectionString(connectionString: string): string { + // Greedy up to the LAST `@`: an unencoded `@` in the password is the most + // common connection-string mistake, and matching the first one would leave + // the rest of the password in the message. Everything before the final `@` + // is userinfo, so redacting all of it is always safe. + // Non-URL forms (SQLite paths) have no `://` and are left alone. + return connectionString.replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@"); +} + +/** Strips a `file:` prefix and any URL query, leaving a filesystem path. */ +export function sqlitePath(connectionString: string): string { + const trimmed = connectionString.trim(); + const withoutScheme = trimmed.startsWith("file:") ? trimmed.slice("file:".length) : trimmed; + return withoutScheme.split("?")[0] ?? withoutScheme; +} + +const QUOTING: Record string> = { + // Doubling the delimiter is the escape in every dialect here, so an + // identifier containing one cannot break out of the quotes. + postgres: (identifier) => `"${identifier.replace(/"/g, '""')}"`, + sqlite: (identifier) => `"${identifier.replace(/"/g, '""')}"`, + mysql: (identifier) => `\`${identifier.replace(/`/g, "``")}\``, +}; + +function bunSqlClient(connectionString: string, dbType: "postgres" | "mysql"): DbClient { + const sql = new SQL(connectionString); + + return { + dbType, + async query>(query: string, params: unknown[] = []) { + const rows = await sql.unsafe(query, params); + return (Array.isArray(rows) ? rows : []) as T[]; + }, + placeholder: dbType === "postgres" ? (position) => `$${position}` : () => "?", + quote: QUOTING[dbType], + async close() { + await sql.close(); + }, + }; +} + +function sqliteClient(connectionString: string): DbClient { + const database = new Database(sqlitePath(connectionString), { readonly: true }); + + return { + dbType: "sqlite", + query>(query: string, params: unknown[] = []) { + // bun:sqlite is synchronous; the Promise keeps one interface for callers. + return Promise.resolve(database.query(query).all(...(params as never[])) as T[]); + }, + placeholder: () => "?", + quote: QUOTING.sqlite, + close() { + database.close(); + return Promise.resolve(); + }, + }; +} + +/** + * Connects to the database a connection string names. + * + * @param platform - Tailors the failure hint; the same "Connection closed" + * means something different on Supabase than on a local SQLite file. + */ +export async function createDbClient( + connectionString: string, + platform?: DbPlatform, +): Promise { + const dbType = detectDbType(connectionString); + + try { + if (dbType === "sqlite") { + const client = sqliteClient(connectionString); + // bun:sqlite opens lazily, so a missing file would not surface until the + // first real query — long after the "connecting" spinner has stopped. + await client.query("SELECT 1"); + return client; + } + + const client = bunSqlClient(connectionString, dbType); + await client.query("SELECT 1"); + return client; + } catch (error) { + throw connectionError(error, connectionString, platform); + } +} + +export type DbPlatform = "supabase" | "betterauth" | "authjs"; + +/** + * Turns a driver error into something a user can act on. + * + * Rewritten rather than ported: the standalone tool matched on `pg`'s + * `ENOTFOUND`/`ETIMEDOUT`, which `Bun.sql` never emits. Bun reports both an + * unreachable host and a closed port as `ERR_*_CONNECTION_CLOSED` with the + * message "Connection closed" — exactly the case where a bare driver error + * helps least. + */ +export function describeDbError(error: unknown, platform?: DbPlatform): string { + const message = error instanceof Error ? error.message : String(error); + const code = (error as { code?: string })?.code ?? ""; + + if (code.includes("CONNECTION_CLOSED") || /connection closed|econnrefused/i.test(message)) { + if (platform === "supabase") { + return ( + "Could not reach the database. Check the host and port in the connection string.\n" + + "Supabase direct connections need the IPv4 add-on — use the pooler connection string\n" + + "(Dashboard → Connect → Session pooler), or enable IPv4 under Settings → Add-Ons." + ); + } + return "Could not reach the database. Check the host and port, and that the server accepts connections from here."; + } + + if (/password authentication failed|access denied/i.test(message)) { + return "The database rejected those credentials. Check the user and password in the connection string."; + } + + if (/does not exist|unknown database|no such table|permission denied/i.test(message)) { + if (platform === "supabase") { + return ( + "The auth.users table was not readable. It is created automatically when Supabase Auth is enabled.\n" + + "Check Authentication is enabled, and connect as the `postgres` role rather than an application role." + ); + } + return "The expected table was not found, or the user cannot read it. Check the database name and the user's SELECT permission."; + } + + if (/unable to open database|sqlitecantopen|no such file/i.test(message)) { + return "Could not open the SQLite file. Check the path, and that the file exists and is readable."; + } + + return "Check the connection string, that the server is running, and that it is reachable from here."; +} + +function connectionError( + error: unknown, + connectionString: string, + platform?: DbPlatform, +): CliError { + const message = error instanceof Error ? error.message : String(error); + return new CliError( + `Could not connect to ${redactConnectionString(connectionString)}: ${message}\n\n${describeDbError(error, platform)}`, + { code: ERROR_CODE.USAGE_ERROR }, + ); +} + +/** + * Runs `work` against a fresh client and always closes it. + * + * A leaked connection keeps the process alive after the export has written its + * file, which looks like a hang. + */ +export async function withDbClient( + connectionString: string, + platform: DbPlatform | undefined, + work: (client: DbClient) => Promise, +): Promise { + const client = await createDbClient(connectionString, platform); + try { + return await work(client); + } catch (error) { + // A query failure carries the same actionable hints as a connection one: + // a missing table is the most common thing that goes wrong here. + if (error instanceof CliError) throw error; + throw new CliError( + `${error instanceof Error ? error.message : String(error)}\n\n${describeDbError(error, platform)}`, + { code: ERROR_CODE.USAGE_ERROR }, + ); + } finally { + await client.close().catch(() => {}); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/instance.test.ts b/packages/cli-core/src/commands/migrate/lib/instance.test.ts new file mode 100644 index 000000000..6f6330f56 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/instance.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, test } from "bun:test"; +import { + DEV_USER_LIMIT, + detectInstanceType, + getDefaultConcurrencyLimit, + getDefaultRateLimit, + getRetryDelay, + resolveLimits, +} from "./instance.ts"; + +describe("detectInstanceType", () => { + test.each([ + ["sk_live_abc123", "prod"], + ["sk_test_abc123", "dev"], + ["sk_something_else", "dev"], + ["nonsense", "dev"], + ])("%s -> %s", (key, expected) => { + expect(detectInstanceType(key)).toBe(expected as "dev" | "prod"); + }); +}); + +describe("default limits", () => { + test.each([ + ["prod", 100], + ["dev", 10], + ])("%s instances get %i req/s", (instanceType, expected) => { + expect(getDefaultRateLimit(instanceType as "dev" | "prod")).toBe(expected); + }); + + test.each([ + [100, 9], + [10, 1], + [1, 1], + ])("a %i req/s limit yields %i concurrent calls", (rateLimit, expected) => { + expect(getDefaultConcurrencyLimit(rateLimit)).toBe(expected); + }); + + test("development instances are capped at 500 users", () => { + expect(DEV_USER_LIMIT).toBe(500); + }); +}); + +describe("resolveLimits", () => { + test("derives both limits from the key when nothing is overridden", () => { + expect(resolveLimits("sk_live_x", {})).toEqual({ + instanceType: "prod", + rateLimit: 100, + concurrencyLimit: 9, + }); + }); + + test("honours environment overrides", () => { + expect( + resolveLimits("sk_test_x", { + CLERK_MIGRATE_RATE_LIMIT: "50", + CLERK_MIGRATE_CONCURRENCY_LIMIT: "4", + }), + ).toEqual({ instanceType: "dev", rateLimit: 50, concurrencyLimit: 4 }); + }); + + test("derives concurrency from an overridden rate limit", () => { + expect(resolveLimits("sk_test_x", { CLERK_MIGRATE_RATE_LIMIT: "200" }).concurrencyLimit).toBe( + 19, + ); + }); + + test.each([["0"], ["-5"], ["fast"], [""]])( + "ignores the unusable override %p in favour of the default", + (value) => { + expect(resolveLimits("sk_test_x", { CLERK_MIGRATE_RATE_LIMIT: value }).rateLimit).toBe(10); + }, + ); +}); + +describe("getRetryDelay", () => { + test.each([ + [undefined, 10_000, 10_000, 10], + [15, 10_000, 15_000, 15], + [1, 10_000, 1000, 1], + ])("Retry-After %p -> %i ms", (retryAfter, fallback, delayMs, delaySeconds) => { + expect(getRetryDelay(retryAfter, fallback)).toEqual({ delayMs, delaySeconds }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/instance.ts b/packages/cli-core/src/commands/migrate/lib/instance.ts new file mode 100644 index 000000000..d094c11cf --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/instance.ts @@ -0,0 +1,93 @@ +/** + * Instance-type detection and the throughput limits that follow from it. + * + * Ported from the standalone migration-tool's `src/envs-constants.ts`, minus + * its dotenv/Zod env bootstrap: the secret key arrives from + * `resolveBapiSecretKey`, and only the two override knobs read the environment. + */ + +/** Development instances are capped at this many users by Clerk. */ +export const DEV_USER_LIMIT = 500; + +/** How many times a 429 is retried before the user is recorded as failed. */ +export const MAX_RETRIES = 5; + +/** Fallback backoff when a 429 response carries no `Retry-After`. */ +export const RETRY_DELAY_MS = 10_000; + +export type InstanceType = "dev" | "prod"; + +/** + * Derives the instance type from the secret key's prefix. + * + * @example detectInstanceType("sk_live_xxx") // "prod" + * @example detectInstanceType("sk_test_xxx") // "dev" + */ +export function detectInstanceType(secretKey: string): InstanceType { + return secretKey.split("_")[1] === "live" ? "prod" : "dev"; +} + +/** + * Clerk's documented `POST /v1/users` rate limits, as requests per second: + * 1000 per 10s for production, 100 per 10s for development. + */ +export function getDefaultRateLimit(instanceType: InstanceType): number { + return instanceType === "prod" ? 100 : 10; +} + +/** + * Concurrency that saturates ~95% of the rate limit, assuming ~100ms of API + * latency per call: N concurrent requests at 100ms each yield N * 10 req/s. + * + * Override with `CLERK_MIGRATE_CONCURRENCY_LIMIT` when actual latency differs. + */ +export function getDefaultConcurrencyLimit(rateLimit: number): number { + return Math.max(1, Math.floor(rateLimit * 0.095)); +} + +export type ResolvedLimits = { + instanceType: InstanceType; + rateLimit: number; + concurrencyLimit: number; +}; + +/** + * Resolves throughput limits for a run: defaults from the detected instance + * type, each overridable by an environment variable. + * + * Non-numeric or non-positive overrides are ignored in favour of the default + * rather than failing the run — an unusable limit would stall the import. + */ +export function resolveLimits( + secretKey: string, + env: Record = process.env, +): ResolvedLimits { + const instanceType = detectInstanceType(secretKey); + + const positive = (value: string | undefined): number | undefined => { + if (!value) return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; + }; + + const rateLimit = positive(env.CLERK_MIGRATE_RATE_LIMIT) ?? getDefaultRateLimit(instanceType); + const concurrencyLimit = + positive(env.CLERK_MIGRATE_CONCURRENCY_LIMIT) ?? getDefaultConcurrencyLimit(rateLimit); + + return { instanceType, rateLimit, concurrencyLimit }; +} + +/** + * Backoff for a 429, preferring the server's `Retry-After` over the default. + * + * @param retryAfterSeconds - `Retry-After` value from the response, if present. + * @param defaultDelayMs - Fallback delay in milliseconds. + */ +export function getRetryDelay( + retryAfterSeconds: number | undefined, + defaultDelayMs: number, +): { delayMs: number; delaySeconds: number } { + const delayMs = retryAfterSeconds ? retryAfterSeconds * 1000 : defaultDelayMs; + const delaySeconds = retryAfterSeconds || defaultDelayMs / 1000; + return { delayMs, delaySeconds }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.test.ts b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts new file mode 100644 index 000000000..1af83c1f0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts @@ -0,0 +1,200 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { classifyLogFile, findLogFile, formatSize, listLogFiles, readNdjson } from "./log-files.ts"; +import { getLogDir } from "./logger.ts"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logfiles-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); +}); + +/** Writes a log file with one NDJSON line per entry. */ +function writeLog(name: string, entries: unknown[]): string { + fs.mkdirSync(getLogDir(), { recursive: true }); + const filePath = path.join(getLogDir(), name); + fs.writeFileSync(filePath, entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n"); + return filePath; +} + +describe("classifyLogFile", () => { + test.each([ + ["migration-2026-01-01T12-00-00.log", "migration", "2026-01-01T12-00-00"], + ["user-deletion-2026-01-01T12-00-00.log", "deletion", "2026-01-01T12-00-00"], + ["export-2026-01-01T12-00-00.log", "export", "2026-01-01T12-00-00"], + ])("%s is a %s log from %s", (name, kind, timestamp) => { + expect(classifyLogFile(name)).toEqual({ kind: kind as never, timestamp }); + }); + + test.each([["random.log"], ["migration.log"], ["notes.txt"]])( + "%s is unrecognized rather than a parse failure", + (name) => { + expect(classifyLogFile(name)).toEqual({ kind: "unknown", timestamp: "" }); + }, + ); +}); + +describe("listLogFiles", () => { + test("returns nothing when the directory does not exist", () => { + expect(fs.existsSync(getLogDir())).toBe(false); + expect(listLogFiles()).toEqual([]); + }); + + test("returns nothing when the directory is empty", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + expect(listLogFiles()).toEqual([]); + }); + + test("reports kind, timestamp, size and entry count per file", () => { + writeLog("migration-2026-01-01T12-00-00.log", [{ userId: "u1" }, { userId: "u2" }]); + + const [file] = listLogFiles(); + expect(file).toMatchObject({ + name: "migration-2026-01-01T12-00-00.log", + kind: "migration", + timestamp: "2026-01-01T12-00-00", + entryCount: 2, + }); + expect(file?.sizeBytes).toBeGreaterThan(0); + }); + + test("ignores files that are not logs", () => { + writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + fs.writeFileSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"), "[]"); + fs.writeFileSync(path.join(getLogDir(), "notes.txt"), "hi"); + + expect(listLogFiles().map((file) => file.name)).toEqual(["migration-2026-01-01T12-00-00.log"]); + }); + + test("ignores subdirectories", () => { + fs.mkdirSync(path.join(getLogDir(), "nested.log"), { recursive: true }); + expect(listLogFiles()).toEqual([]); + }); + + test("returns the newest run first", () => { + writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + writeLog("migration-2026-03-01T12-00-00.log", [{ a: 1 }]); + writeLog("migration-2026-02-01T12-00-00.log", [{ a: 1 }]); + + expect(listLogFiles().map((file) => file.timestamp)).toEqual([ + "2026-03-01T12-00-00", + "2026-02-01T12-00-00", + "2026-01-01T12-00-00", + ]); + }); + + // Sorting on the filename would put every "user-deletion-" ahead of every + // "migration-", regardless of when the runs actually happened. + test("orders by timestamp across log kinds, not by the name's prefix", () => { + writeLog("user-deletion-2026-01-30T17-02-51.log", [{ a: 1 }]); + writeLog("migration-2026-02-01T09-14-22.log", [{ a: 1 }]); + + expect(listLogFiles().map((file) => file.kind)).toEqual(["migration", "deletion"]); + }); + + test("sorts unrecognized names last", () => { + writeLog("something-else.log", [{ a: 1 }]); + writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + + expect(listLogFiles().map((file) => file.name)).toEqual([ + "migration-2026-01-01T12-00-00.log", + "something-else.log", + ]); + }); + + test("does not count blank lines as entries", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync(path.join(getLogDir(), "migration-x.log"), '{"a":1}\n\n\n{"b":2}\n'); + expect(listLogFiles()[0]?.entryCount).toBe(2); + }); + + test("lists a log whose name does not match the convention", () => { + writeLog("something-else.log", [{ a: 1 }]); + expect(listLogFiles()[0]).toMatchObject({ kind: "unknown", timestamp: "", entryCount: 1 }); + }); +}); + +describe("findLogFile", () => { + beforeEach(() => { + writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + }); + + test("finds a log by name", () => { + expect(findLogFile("migration-2026-01-01T12-00-00.log")?.entryCount).toBe(1); + }); + + test("accepts a path and matches on the basename", () => { + expect(findLogFile("./logs/migration-2026-01-01T12-00-00.log")?.entryCount).toBe(1); + }); + + test("returns nothing for a name that is not there", () => { + expect(findLogFile("migration-nope.log")).toBeUndefined(); + }); +}); + +describe("readNdjson", () => { + test("parses one entry per line", () => { + const file = writeLog("migration-a.log", [{ userId: "u1" }, { userId: "u2" }]); + const { entries, errors } = readNdjson(file); + + expect(entries).toEqual([{ userId: "u1" }, { userId: "u2" }]); + expect(errors).toEqual([]); + }); + + test("skips blank lines without reporting them", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + const file = path.join(getLogDir(), "migration-b.log"); + fs.writeFileSync(file, '\n{"a":1}\n \n{"b":2}\n\n'); + + const { entries, errors } = readNdjson(file); + expect(entries).toHaveLength(2); + expect(errors).toEqual([]); + }); + + // A run killed mid-write leaves one truncated line; the complete entries + // before it are still worth having, so the read reports rather than aborts. + test("reports a malformed line by number and keeps the rest", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + const file = path.join(getLogDir(), "migration-c.log"); + fs.writeFileSync(file, '{"a":1}\n{"b":\n{"c":3}\n'); + + const { entries, errors } = readNdjson(file); + expect(entries).toEqual([{ a: 1 }, { c: 3 }]); + expect(errors).toHaveLength(1); + expect(errors[0]?.line).toBe(2); + }); + + test("numbers lines from one, counting blanks", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + const file = path.join(getLogDir(), "migration-d.log"); + fs.writeFileSync(file, '\n\n{"a":1}\nnot json\n'); + + expect(readNdjson(file).errors[0]?.line).toBe(4); + }); +}); + +describe("formatSize", () => { + test.each([ + [0, "0 B"], + [512, "512 B"], + [1024, "1.0 KB"], + [1536, "1.5 KB"], + [1024 * 1024, "1.0 MB"], + ])("%i bytes reads as %s", (bytes, expected) => { + expect(formatSize(bytes)).toBe(expected); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.ts b/packages/cli-core/src/commands/migrate/lib/log-files.ts new file mode 100644 index 000000000..1498ddd36 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/log-files.ts @@ -0,0 +1,141 @@ +/** + * Enumerating and reading the cwd-relative `./logs/` directory. + * + * The standalone migration-tool re-read the directory inside both of its log + * commands to build their pickers. `list`, `clean` and `convert` all share + * this instead, which is also what makes `logs list` nearly free. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { getLogDir } from "./logger.ts"; + +/** The run that produced a log file, read from its filename prefix. */ +export type LogKind = "migration" | "deletion" | "export" | "unknown"; + +const FILENAME_PATTERN = /^(migration|user-deletion|export)-(.+)\.log$/; + +const KIND_BY_PREFIX: Record = { + migration: "migration", + "user-deletion": "deletion", + export: "export", +}; + +export type LogFile = { + name: string; + path: string; + kind: LogKind; + /** Timestamp as recorded in the filename, or `""` for an unrecognized name. */ + timestamp: string; + sizeBytes: number; + /** Non-empty NDJSON lines, malformed ones included. */ + entryCount: number; +}; + +export function classifyLogFile(name: string): { kind: LogKind; timestamp: string } { + const match = FILENAME_PATTERN.exec(name); + if (!match) return { kind: "unknown", timestamp: "" }; + return { kind: KIND_BY_PREFIX[match[1] as string] ?? "unknown", timestamp: match[2] as string }; +} + +function countEntries(filePath: string): number { + try { + return fs + .readFileSync(filePath, "utf-8") + .split("\n") + .filter((line) => line.trim().length > 0).length; + } catch { + // An unreadable file still belongs in the listing; its count is unknown. + return 0; + } +} + +/** + * Every `.log` file in `./logs/`, newest first. + * + * @returns An empty array when the directory is absent — "no logs yet" and "no + * logs directory" are the same thing to every caller. + */ +export function listLogFiles(): LogFile[] { + const dir = getLogDir(); + if (!fs.existsSync(dir)) return []; + + const files: LogFile[] = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith(".log")) continue; + + const filePath = path.join(dir, name); + let stats: fs.Stats; + try { + stats = fs.statSync(filePath); + } catch { + continue; + } + if (!stats.isFile()) continue; + + files.push({ + name, + path: filePath, + ...classifyLogFile(name), + sizeBytes: stats.size, + entryCount: countEntries(filePath), + }); + } + + // Sort on the timestamp, not the filename: the kind prefix sorts first in a + // filename comparison, which would interleave a run from January ahead of one + // from March purely because "user-deletion" > "migration". Timestamps are + // ISO-ish and zero-padded, so lexical order is chronological. Names without + // one sort last, then alphabetically. + return files.sort( + (a, b) => b.timestamp.localeCompare(a.timestamp) || a.name.localeCompare(b.name), + ); +} + +/** Resolves a user-supplied name or path to a log file in `./logs/`. */ +export function findLogFile(nameOrPath: string): LogFile | undefined { + const wanted = path.basename(nameOrPath); + return listLogFiles().find((file) => file.name === wanted); +} + +export type NdjsonLineError = { + /** 1-indexed line number in the source file. */ + line: number; + message: string; +}; + +export type NdjsonReadResult = { + entries: unknown[]; + errors: NdjsonLineError[]; +}; + +/** + * Parses an NDJSON file line by line. + * + * Malformed lines are collected with their line numbers rather than aborting + * the read: a run killed mid-write leaves one truncated final line, and the + * hundreds of complete entries before it are still worth having. + */ +export function readNdjson(filePath: string): NdjsonReadResult { + const entries: unknown[] = []; + const errors: NdjsonLineError[] = []; + + const lines = fs.readFileSync(filePath, "utf-8").split("\n"); + for (const [index, line] of lines.entries()) { + if (line.trim().length === 0) continue; + try { + entries.push(JSON.parse(line)); + } catch (error) { + errors.push({ line: index + 1, message: (error as Error).message }); + } + } + + return { entries, errors }; +} + +/** Human-readable file size. */ +export function formatSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} diff --git a/packages/cli-core/src/commands/migrate/lib/logger.test.ts b/packages/cli-core/src/commands/migrate/lib/logger.test.ts new file mode 100644 index 000000000..db973f00a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/logger.test.ts @@ -0,0 +1,110 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + errorLogger, + getDateTimeStamp, + getLogDir, + getLogFilePath, + importLogger, + validationLogger, +} from "./logger.ts"; + +const DATE_TIME = "2026-01-01T12:00:00"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + // realpath so the comparison against process.cwd() survives macOS's + // /var -> /private/var symlink. + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logger-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); +}); + +function readEntries(): Record[] { + return fs + .readFileSync(getLogFilePath("migration", DATE_TIME), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); +} + +describe("log file paths", () => { + test("writes under the current working directory, not next to the binary", () => { + expect(getLogDir()).toBe(path.join(workDir, "logs")); + }); + + test("replaces the timestamp's colons so the name is valid on Windows", () => { + expect(path.basename(getLogFilePath("migration", DATE_TIME))).toBe( + "migration-2026-01-01T12-00-00.log", + ); + }); + + test("getDateTimeStamp drops milliseconds", () => { + expect(getDateTimeStamp()).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$/); + }); +}); + +describe("log writers", () => { + test("creates the logs directory on first write", () => { + expect(fs.existsSync(getLogDir())).toBe(false); + importLogger({ userId: "u1", status: "success", clerkUserId: "user_x" }, DATE_TIME); + expect(fs.existsSync(getLogDir())).toBe(true); + }); + + test("appends one NDJSON line per entry", () => { + importLogger({ userId: "u1", status: "success", clerkUserId: "user_x" }, DATE_TIME); + importLogger({ userId: "u2", status: "error", error: "boom", code: "422" }, DATE_TIME); + + const entries = readEntries(); + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ userId: "u1", status: "success", clerkUserId: "user_x" }); + expect(entries[1]).toEqual({ userId: "u2", status: "error", error: "boom", code: "422" }); + }); + + test("writes one line per error in a failed payload", () => { + errorLogger( + { + userId: "u1", + status: "422", + errors: [ + { code: "a", message: "short a", longMessage: "long a" }, + { code: "b", message: "short b" }, + ], + }, + DATE_TIME, + ); + + const entries = readEntries(); + expect(entries).toHaveLength(2); + expect(entries[0]).toMatchObject({ type: "User Creation Error", error: "long a" }); + // Falls back to `message` when the API omitted a long form. + expect(entries[1]).toMatchObject({ error: "short b" }); + }); + + test("records validation failures in the same run log", () => { + validationLogger( + { error: "missing identifier", path: ["email"], userId: "u3", row: 4 }, + DATE_TIME, + ); + expect(readEntries()[0]).toEqual({ + userId: "u3", + status: "fail", + error: "missing identifier", + path: ["email"], + row: 4, + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.ts b/packages/cli-core/src/commands/migrate/lib/logger.ts new file mode 100644 index 000000000..72cd98372 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/logger.ts @@ -0,0 +1,114 @@ +/** + * NDJSON migration logs. + * + * Ported from the standalone migration-tool's `src/logger.ts`, with one + * behavioural fix: logs are written relative to the current working directory + * rather than to `__dirname/../logs`. In a `bun build --compile` binary there + * is no source tree next to the executable, so the original path would land + * logs inside wherever the binary happens to live. + * + * Writes are synchronous appends so a run interrupted with Ctrl-C still leaves + * a complete record of everything already processed. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { log } from "../../../lib/log.ts"; +import type { + DeleteLogEntry, + ErrorLog, + ErrorPayload, + ExportLogEntry, + ImportLogEntry, + ValidationErrorPayload, +} from "../types.ts"; + +/** Absolute path of the cwd-relative `logs/` directory. */ +export function getLogDir(): string { + return path.join(process.cwd(), "logs"); +} + +/** Absolute path of the log file a run with this timestamp writes to. */ +export function getLogFilePath(logFile: string, dateTime: string): string { + // Colons are illegal in Windows filenames, and the timestamp is an ISO string. + return path.join(getLogDir(), `${logFile}-${dateTime}.log`.replace(/:/g, "-")); +} + +/** ISO timestamp without milliseconds — the log-file name discriminator. */ +export function getDateTimeStamp(): string { + return new Date().toISOString().split(".")[0] ?? ""; +} + +function appendToLogFile(fullPath: string, entry: unknown): void { + try { + fs.mkdirSync(path.dirname(fullPath), { recursive: true }); + fs.appendFileSync(fullPath, `${JSON.stringify(entry)}\n`); + } catch (error) { + // A broken log destination must not abort an in-flight migration; the run + // is still making real progress against the API. + log.warn(`Could not write migration log: ${(error as Error).message}`); + } +} + +/** Writes each error in a failed API call as its own NDJSON line. */ +export function errorLogger(payload: ErrorPayload, dateTime: string): void { + for (const err of payload.errors) { + const entry: ErrorLog = { + type: "User Creation Error", + userId: payload.userId, + status: payload.status, + error: err.longMessage ?? err.message, + }; + appendToLogFile(getLogFilePath("migration", dateTime), entry); + } +} + +/** Writes a user that failed schema validation before any API call. */ +export function validationLogger(payload: ValidationErrorPayload, dateTime: string): void { + appendToLogFile(getLogFilePath("migration", dateTime), { + userId: payload.userId, + status: "fail" as const, + error: payload.error, + path: payload.path, + row: payload.row, + }); +} + +/** Writes the outcome of one import attempt. */ +export function importLogger(entry: ImportLogEntry, dateTime: string): void { + appendToLogFile(getLogFilePath("migration", dateTime), entry); +} + +/** + * Writes the outcome of one deletion attempt. + * + * A separate `user-deletion-` file rather than another line in the migration + * log: undoing a migration is its own run, and mixing the two would make + * "what did this import do" unanswerable after an undo. + */ +export function deleteLogger(entry: DeleteLogEntry, dateTime: string): void { + appendToLogFile(getLogFilePath("user-deletion", dateTime), entry); +} + +/** + * Writes the outcome of exporting one user. + * + * Its own `export-` file for the same reason deletions get theirs: an export + * is a distinct run, and `migrate logs list` reports each kind separately. + */ +export function exportLogger(entry: ExportLogEntry, dateTime: string): void { + appendToLogFile(getLogFilePath("export", dateTime), entry); +} + +/** Writes each error in a failed deletion as its own NDJSON line. */ +export function deleteErrorLogger(payload: ErrorPayload, dateTime: string): void { + for (const err of payload.errors) { + const entry: ErrorLog = { + type: "User Deletion Error", + userId: payload.userId, + status: payload.status, + error: err.longMessage ?? err.message, + }; + appendToLogFile(getLogFilePath("user-deletion", dateTime), entry); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts new file mode 100644 index 000000000..cdac7d866 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts @@ -0,0 +1,322 @@ +import { describe, expect, test } from "bun:test"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import type { FieldAnalysis } from "./analysis.ts"; +import { buildReadinessReport, formatReadinessReport, type ReadinessItem } from "./readiness.ts"; + +/** Instance settings carrying only the attributes and providers a test names. */ +function settings(config: { + attributes?: Record; + social?: Record; +}): UserSettingsJSON { + return { + attributes: Object.fromEntries( + Object.entries(config.attributes ?? {}).map(([name, value]) => [ + name, + { enabled: value.enabled, required: value.required ?? false }, + ]), + ), + social: config.social ?? {}, + } as unknown as UserSettingsJSON; +} + +/** Field analysis with everything absent unless the test says otherwise. */ +function analysis(overrides: Partial & { totalUsers: number }): FieldAnalysis { + return { + identifiers: { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: overrides.totalUsers, + ...overrides.identifiers, + }, + fieldCounts: overrides.fieldCounts ?? {}, + totalUsers: overrides.totalUsers, + }; +} + +const item = (report: { items: ReadinessItem[] }, label: string) => + report.items.find((entry) => entry.label === label); + +describe("which rows appear", () => { + test("reports only the fields the file actually carries", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 3, identifiers: { verifiedEmails: 3 } as never }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }); + expect(report.items.map((entry) => entry.label)).toEqual(["Email"]); + }); + + test("counts verified and unverified identifiers together", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, unverifiedEmails: 2, hasAnyIdentifier: 5 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }); + expect(item(report, "Email")?.userCount).toBe(5); + }); + + test("groups rows into identifiers, auth and user model", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, username: 2, hasAnyIdentifier: 2 } as never, + fieldCounts: { password: 2, firstName: 2, lastName: 1 }, + }), + settings: settings({}), + }); + expect(report.items.map((entry) => [entry.label, entry.section])).toEqual([ + ["Email", "identifiers"], + ["Username", "identifiers"], + ["Password", "auth"], + ["First name", "model"], + ["Last name", "model"], + ]); + }); +}); + +describe("required in Clerk but missing from the file", () => { + // The expensive case: those users fail one at a time, mid-import, after + // earlier users have already been created. + test("flags an attribute Clerk requires that not every user has", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 10, username: 10 } as never, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }), + }); + + const email = item(report, "Email"); + expect(email?.blocking).toBe(true); + expect(email?.detail).toContain("3 users lack it"); + expect(report.blocking).toHaveLength(1); + }); + + test("does not flag a required attribute every user has", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + expect(report.blocking).toHaveLength(0); + }); + + test("does not flag an enabled-but-optional attribute that some users lack", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 10, hasAnyIdentifier: 10 } as never, + fieldCounts: { firstName: 2 }, + }), + settings: settings({ + attributes: { email_address: { enabled: true }, first_name: { enabled: true } }, + }), + }); + expect(report.blocking).toHaveLength(0); + }); + + test("uses the singular form for a single missing user", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 1, hasAnyIdentifier: 2, username: 2 } as never, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }), + }); + expect(item(report, "Email")?.detail).toContain("1 user lacks it"); + }); +}); + +describe("present in the file but disabled in Clerk", () => { + test("flags an attribute the instance has switched off", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, username: 3, hasAnyIdentifier: 3 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true }, username: { enabled: false } }, + }), + }); + + const username = item(report, "Username"); + expect(username?.blocking).toBe(true); + expect(username?.detail).toBe("not enabled in Clerk"); + }); + + test("flags a social provider users signed up with that Clerk lacks", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }), + providerCounts: { google: 3, discord: 1 }, + }); + + expect(item(report, "Google")?.blocking).toBe(false); + expect(item(report, "Discord")?.blocking).toBe(true); + expect(report.blocking.map((entry) => entry.label)).toEqual(["Discord"]); + }); + + test("maps a provider whose Clerk strategy name differs", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 1, identifiers: { verifiedEmails: 1 } as never }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_microsoft: { enabled: true } }, + }), + providerCounts: { azure: 1 }, + }); + expect(item(report, "Microsoft (Azure)")?.blocking).toBe(false); + }); + + test("ignores a provider no user actually signed up with", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 1, identifiers: { verifiedEmails: 1 } as never }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + providerCounts: { discord: 0 }, + }); + expect(item(report, "Discord")).toBeUndefined(); + }); +}); + +describe("when the instance settings cannot be read", () => { + const unreadable = () => + buildReadinessReport({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 3 } as never, + }), + settings: null, + }); + + test("marks the report as degraded rather than failing", () => { + expect(unreadable().settingsUnavailable).toBe(true); + }); + + // `null` means "not read", which must not be confused with `false` + // ("read, and it is off") — the latter blocks, the former cannot. + test("claims nothing about Clerk, so nothing blocks", () => { + const report = unreadable(); + expect(item(report, "Email")?.clerkEnabled).toBeNull(); + expect(report.blocking).toHaveLength(0); + }); + + test("still reports what the file contains", () => { + expect(unreadable().items.map((entry) => entry.label)).toEqual(["Email"]); + }); + + test("renders a note explaining the checks are coverage only", () => { + const output = formatReadinessReport(unreadable()).join("\n"); + expect(output).toContain("Could not read this instance's settings"); + expect(output).toContain("dashboard.clerk.com"); + }); +}); + +describe("file-level totals", () => { + test("counts users with no identifier at all", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 7 } as never, + }), + settings: settings({}), + }); + expect(report.withoutIdentifier).toBe(3); + }); + + test("carries the validation failure count through", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 2 }), + settings: settings({}), + validationFailed: 5, + }); + expect(report.validationFailed).toBe(5); + }); +}); + +describe("rendering", () => { + const blocked = () => + buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 8, username: 10 } as never, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }), + validationFailed: 2, + }); + + test("leads with the counts an operator needs before confirming", () => { + const output = formatReadinessReport(blocked()).join("\n"); + expect(output).toContain("10 users ready to import"); + expect(output).toContain("2 failed validation"); + expect(output).toContain("2 without any identifier"); + }); + + test("names the blocking rows and points at the dashboard", () => { + const output = formatReadinessReport(blocked()).join("\n"); + expect(output).toContain("1 setting needs attention"); + expect(output).toContain("3 users lack it"); + expect(output).toContain("dashboard.clerk.com"); + }); + + test("confirms a clean report when nothing blocks", () => { + const output = formatReadinessReport( + buildReadinessReport({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }), + ).join("\n"); + expect(output).toContain("Every field in this file is configured in Clerk"); + }); + + test("does not claim everything is configured when settings were unreadable", () => { + const output = formatReadinessReport( + buildReadinessReport({ analysis: analysis({ totalUsers: 1 }), settings: null }), + ).join("\n"); + expect(output).not.toContain("Every field in this file is configured"); + }); + + test("renders section headings only for sections that have rows", () => { + const output = formatReadinessReport( + buildReadinessReport({ + analysis: analysis({ + totalUsers: 1, + identifiers: { verifiedEmails: 1, hasAnyIdentifier: 1 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }), + ).join("\n"); + expect(output).toContain("Identifiers"); + expect(output).not.toContain("Social connections"); + expect(output).not.toContain("User model"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts new file mode 100644 index 000000000..353823415 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -0,0 +1,238 @@ +/** + * The Migration Readiness report: what the import file contains, cross- + * referenced against what the destination instance actually accepts. + * + * Ported from the standalone migration-tool's `displayCrossReference`. Split + * into a pure {@link buildReadinessReport} and a separate renderer so the + * cross-reference decisions are testable without parsing coloured output. + * + * The point of the report is to surface, *before* anything is written to + * Clerk, the two failure modes a migration only discovers halfway through: + * a field Clerk requires that some users lack, and a social provider users + * signed up with that Clerk has not enabled. + */ + +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { bold, dim, green, red, yellow } from "../../../lib/color.ts"; +// Pure attribute lookups, shared with the `users` create wizard. +import { isEnabled, isRequired, type AttributeName } from "../../users/interactive/attributes.ts"; +import type { FieldAnalysis } from "./analysis.ts"; +import { providerLabel, toClerkStrategy } from "./clerk-config.ts"; + +const DASHBOARD_URL = "https://dashboard.clerk.com/~/user-authentication"; + +export type ReadinessSection = "identifiers" | "auth" | "social" | "model"; + +/** + * One row of the report. + * + * @property clerkEnabled - `null` when the instance settings could not be read, + * which is different from `false` ("read, and it is off"). + * @property blocking - This row will cost users unless the operator acts. + */ +export type ReadinessItem = { + label: string; + section: ReadinessSection; + /** Users in the file that carry this field or provider. */ + userCount: number; + clerkEnabled: boolean | null; + clerkRequired: boolean | null; + blocking: boolean; + /** Why it blocks — omitted when it does not. */ + detail?: string; +}; + +export type ReadinessReport = { + totalUsers: number; + /** Users with no identifier at all; they cannot be imported under any settings. */ + withoutIdentifier: number; + validationFailed: number; + items: ReadinessItem[]; + /** Every item flagged `blocking`, in report order. */ + blocking: ReadinessItem[]; + /** True when the instance settings could not be read. */ + settingsUnavailable: boolean; +}; + +type BuildInput = { + analysis: FieldAnalysis; + /** `null` when no publishable key was available, or FAPI could not be read. */ + settings: UserSettingsJSON | null; + validationFailed?: number; + /** Source-platform provider key → user count. Supabase exports only. */ + providerCounts?: Record; +}; + +/** An identifier or user-model row, with its blocking verdict. */ +function buildAttributeItem( + label: string, + section: ReadinessSection, + attribute: AttributeName, + userCount: number, + settings: UserSettingsJSON | null, + totalUsers: number, +): ReadinessItem { + const enabled = settings ? isEnabled(settings, attribute) : null; + const required = settings ? isRequired(settings, attribute) : null; + const missing = totalUsers - userCount; + + // Required but not universal is the expensive case: those users fail one by + // one, mid-import, after earlier users have already been created. + if (required === true && missing > 0) { + return { + label, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: true, + detail: + missing === 1 + ? "required in Clerk, but 1 user lacks it" + : `required in Clerk, but ${missing} users lack it`, + }; + } + + // Present in the file but switched off in Clerk: the data is silently dropped. + if (enabled === false && userCount > 0) { + return { + label, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: true, + detail: "not enabled in Clerk", + }; + } + + return { + label, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: false, + }; +} + +/** + * Cross-references the file against the instance. + * + * A field absent from the file contributes no row — the report describes what + * is actually being imported, not every setting Clerk supports. + */ +export function buildReadinessReport(input: BuildInput): ReadinessReport { + const { analysis, settings, validationFailed = 0, providerCounts = {} } = input; + const total = analysis.totalUsers; + const items: ReadinessItem[] = []; + + const emailCount = analysis.identifiers.verifiedEmails + analysis.identifiers.unverifiedEmails; + const phoneCount = analysis.identifiers.verifiedPhones + analysis.identifiers.unverifiedPhones; + + const attributeRows: [string, ReadinessSection, AttributeName, number][] = [ + ["Email", "identifiers", "email_address", emailCount], + ["Phone", "identifiers", "phone_number", phoneCount], + ["Username", "identifiers", "username", analysis.identifiers.username], + ["Password", "auth", "password", analysis.fieldCounts.password ?? 0], + ["First name", "model", "first_name", analysis.fieldCounts.firstName ?? 0], + ["Last name", "model", "last_name", analysis.fieldCounts.lastName ?? 0], + ]; + + for (const [label, section, attribute, count] of attributeRows) { + if (count > 0) { + items.push(buildAttributeItem(label, section, attribute, count, settings, total)); + } + } + + for (const [provider, count] of Object.entries(providerCounts)) { + if (count === 0) continue; + const enabled = settings + ? (settings.social?.[toClerkStrategy(provider) as keyof typeof settings.social]?.enabled ?? + false) + : null; + items.push({ + label: providerLabel(provider), + section: "social", + userCount: count, + clerkEnabled: enabled, + clerkRequired: null, + blocking: enabled === false, + ...(enabled === false ? { detail: "not enabled in Clerk" } : {}), + }); + } + + return { + totalUsers: total, + withoutIdentifier: total - analysis.identifiers.hasAnyIdentifier, + validationFailed, + items, + blocking: items.filter((item) => item.blocking), + settingsUnavailable: settings === null, + }; +} + +const SECTION_ORDER: ReadinessSection[] = ["identifiers", "auth", "social", "model"]; +const SECTION_LABELS: Record = { + identifiers: "Identifiers", + auth: "Authentication", + social: "Social connections", + model: "User model", +}; + +function renderItem(item: ReadinessItem, total: number): string { + const coverage = item.userCount === total ? "all users" : `${item.userCount}/${total} users`; + + if (item.blocking) { + return ` ${yellow("⚠")} ${item.label} — ${yellow(item.detail ?? "needs attention")} — ${dim(coverage)}`; + } + if (item.clerkEnabled === true) { + return ` ${green("✓")} ${item.label} — ${dim(`enabled in Clerk — ${coverage}`)}`; + } + // Settings unavailable: state coverage without claiming anything about Clerk. + return ` ${yellow("○")} ${item.label} — ${dim(`${coverage} — check it is enabled in Clerk`)}`; +} + +/** Renders the report for a human, as lines. */ +export function formatReadinessReport(report: ReadinessReport): string[] { + const lines: string[] = [bold("Migration readiness")]; + + lines.push(` ${report.totalUsers} user${report.totalUsers === 1 ? "" : "s"} ready to import`); + if (report.validationFailed > 0) { + lines.push(` ${yellow(`${report.validationFailed} failed validation and will be skipped`)}`); + } + if (report.withoutIdentifier > 0) { + lines.push( + ` ${red(`${report.withoutIdentifier} without any identifier — cannot be imported`)}`, + ); + } + + if (report.settingsUnavailable) { + lines.push( + "", + ` ${yellow("○")} ${dim("Could not read this instance's settings, so the checks below are coverage only.")}`, + ` ${dim(` Verify your settings at ${DASHBOARD_URL}`)}`, + ); + } + + for (const section of SECTION_ORDER) { + const sectionItems = report.items.filter((item) => item.section === section); + if (sectionItems.length === 0) continue; + + lines.push("", bold(SECTION_LABELS[section])); + for (const item of sectionItems) lines.push(renderItem(item, report.totalUsers)); + } + + lines.push(""); + if (report.blocking.length > 0) { + const count = report.blocking.length; + lines.push( + yellow(`⚠ ${count} setting${count === 1 ? "" : "s"} need${count === 1 ? "s" : ""} attention`), + dim(` ${DASHBOARD_URL}`), + ); + } else if (!report.settingsUnavailable) { + lines.push(green("✓ Every field in this file is configured in Clerk")); + } + + return lines; +} diff --git a/packages/cli-core/src/commands/migrate/lib/retry.test.ts b/packages/cli-core/src/commands/migrate/lib/retry.test.ts new file mode 100644 index 000000000..ed7f10091 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/retry.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, test } from "bun:test"; +import { BapiError, CliError } from "../../../lib/errors.ts"; +import { RateLimitExceededError, readRetryAfter, retryOn429 } from "./retry.ts"; + +const rateLimited = (headers: Record = {}) => + new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow" }] }), + new Headers(headers), + ); + +const failed = (status: number) => + new BapiError( + status, + JSON.stringify({ errors: [{ code: "e", message: "nope" }] }), + new Headers(), + ); + +describe("readRetryAfter", () => { + test.each([ + ["12", 12], + ["0", undefined], + ["-1", undefined], + ["soon", undefined], + ])("Retry-After: %s -> %p", (header, expected) => { + expect(readRetryAfter(rateLimited({ "retry-after": header }))).toBe( + expected as number | undefined, + ); + }); + + test("falls back to the error body's retryAfter meta", () => { + const error = new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow", meta: { retryAfter: 7 } }] }), + new Headers(), + ); + expect(readRetryAfter(error)).toBe(7); + }); + + test("prefers the header over the body", () => { + const error = new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow", meta: { retryAfter: 7 } }] }), + new Headers({ "retry-after": "3" }), + ); + expect(readRetryAfter(error)).toBe(3); + }); + + test("returns undefined when neither carries a value", () => { + expect(readRetryAfter(rateLimited())).toBeUndefined(); + }); +}); + +describe("retryOn429", () => { + test("returns the value when the call succeeds first time", async () => { + expect(await retryOn429(async () => "ok")).toBe("ok"); + }); + + test("retries after a 429 and returns the eventual value", async () => { + let attempts = 0; + const result = await retryOn429( + async () => { + attempts++; + if (attempts === 1) throw rateLimited({ "retry-after": "1" }); + return "ok"; + }, + { defaultDelayMs: 5 }, + ); + + expect(result).toBe("ok"); + expect(attempts).toBe(2); + }); + + test("waits the interval the server asked for", async () => { + let attempts = 0; + const started = performance.now(); + + await retryOn429(async () => { + attempts++; + if (attempts === 1) throw rateLimited({ "retry-after": "1" }); + return "ok"; + }); + + expect(performance.now() - started).toBeGreaterThanOrEqual(900); + }); + + test("falls back to the default delay when no Retry-After is given", async () => { + let attempts = 0; + await retryOn429( + async () => { + attempts++; + if (attempts === 1) throw rateLimited(); + return "ok"; + }, + { defaultDelayMs: 5 }, + ); + expect(attempts).toBe(2); + }); + + test("reports each backoff to the caller so it can log against its own run", async () => { + const seen: { attempt: number; delaySeconds: number }[] = []; + let attempts = 0; + + await retryOn429( + async () => { + attempts++; + if (attempts <= 2) throw rateLimited(); + return "ok"; + }, + { + defaultDelayMs: 5, + onRetry: ({ attempt, delaySeconds }) => seen.push({ attempt, delaySeconds }), + }, + ); + + expect(seen.map((entry) => entry.attempt)).toEqual([1, 2]); + expect(seen[0]?.delaySeconds).toBe(0.005); + }); + + test("gives up after the ceiling, distinctly from an ordinary failure", async () => { + let attempts = 0; + + await expect( + retryOn429( + async () => { + attempts++; + throw rateLimited(); + }, + { maxRetries: 2, defaultDelayMs: 5 }, + ), + ).rejects.toThrow(RateLimitExceededError); + + // One initial attempt plus maxRetries retries. + expect(attempts).toBe(3); + }); + + // Only rate limiting is transient; retrying a 422 would just repeat it. + test.each([[400], [401], [404], [422], [500]])("lets a %i through untouched", async (status) => { + let attempts = 0; + + await expect( + retryOn429(async () => { + attempts++; + throw failed(status); + }), + ).rejects.toThrow(BapiError); + + expect(attempts).toBe(1); + }); + + test("lets a non-API error through untouched", async () => { + await expect(retryOn429(async () => Promise.reject(new CliError("boom")))).rejects.toThrow( + CliError, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/retry.ts b/packages/cli-core/src/commands/migrate/lib/retry.ts new file mode 100644 index 000000000..f9d1fc158 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/retry.ts @@ -0,0 +1,66 @@ +/** + * Rate-limit backoff, shared by `migrate run` and `migrate delete`. + * + * Both walk the whole user set through BAPI and hit the same limits, so they + * back off identically rather than approximately: extracting this is what + * makes "deletion retries the same as import" true by construction. + */ + +import { BapiError } from "../../../lib/errors.ts"; +import { MAX_RETRIES, RETRY_DELAY_MS, getRetryDelay } from "./instance.ts"; + +/** Seconds to wait per a 429's `Retry-After` header or error meta, if given. */ +export function readRetryAfter(error: BapiError): number | undefined { + const header = error.headers?.get("retry-after"); + if (header) { + const parsed = Number(header); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + const meta = error.meta?.retryAfter; + return typeof meta === "number" && meta > 0 ? meta : undefined; +} + +/** Raised once a 429 has been retried {@link MAX_RETRIES} times. */ +export class RateLimitExceededError extends Error { + constructor(public readonly attempts: number) { + super(`Rate limit exceeded after ${attempts} retries`); + this.name = "RateLimitExceededError"; + } +} + +export type RetryOptions = { + /** Called before each backoff, so the caller can log it against its own run. */ + onRetry?: (info: { attempt: number; delaySeconds: number; message: string }) => void; + maxRetries?: number; + /** Backoff when the response carries no `Retry-After`. */ + defaultDelayMs?: number; +}; + +/** + * Runs `fn`, backing off and retrying whenever BAPI answers 429. + * + * Anything other than a 429 propagates untouched — only rate limiting is + * transient. Exhausting the retries raises {@link RateLimitExceededError} so + * the caller can record it distinctly from an ordinary API failure. + */ +export async function retryOn429(fn: () => Promise, options: RetryOptions = {}): Promise { + const maxRetries = options.maxRetries ?? MAX_RETRIES; + const defaultDelayMs = options.defaultDelayMs ?? RETRY_DELAY_MS; + + for (let attempt = 0; ; attempt++) { + try { + return await fn(); + } catch (error) { + if (!(error instanceof BapiError) || error.status !== 429) throw error; + if (attempt >= maxRetries) throw new RateLimitExceededError(maxRetries); + + const { delayMs, delaySeconds } = getRetryDelay(readRetryAfter(error), defaultDelayMs); + options.onRetry?.({ + attempt: attempt + 1, + delaySeconds, + message: `Rate limit hit (429), retrying in ${delaySeconds}s (attempt ${attempt + 1}/${maxRetries})`, + }); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts b/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts new file mode 100644 index 000000000..68c44e50b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts @@ -0,0 +1,69 @@ +import { expect, test } from "bun:test"; +import { createApiScheduler } from "./scheduler.ts"; + +/** Resolves after `ms`, so a task can be held open while others queue behind it. */ +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test("never runs more tasks at once than the concurrency limit", async () => { + const schedule = createApiScheduler(3, 10_000); + let active = 0; + let peak = 0; + + await Promise.all( + Array.from({ length: 20 }, () => + schedule(async () => { + active++; + peak = Math.max(peak, active); + await wait(5); + active--; + }), + ), + ); + + expect(peak).toBe(3); + expect(active).toBe(0); +}); + +test("frees a slot when a task throws, instead of deadlocking the queue", async () => { + const schedule = createApiScheduler(1, 10_000); + + await expect(schedule(() => Promise.reject(new Error("boom")))).rejects.toThrow("boom"); + + // If release() had been skipped on the failure path, this would hang. + expect(await schedule(async () => "ok")).toBe("ok"); +}); + +test("paces calls to the rate limit", async () => { + // 100 req/s -> 10ms between starts; 5 calls span at least 4 intervals. + const schedule = createApiScheduler(5, 100); + const started = performance.now(); + + await Promise.all(Array.from({ length: 5 }, () => schedule(async () => {}))); + + expect(performance.now() - started).toBeGreaterThanOrEqual(35); +}); + +test("returns each task's own resolved value", async () => { + const schedule = createApiScheduler(2, 10_000); + const results = await Promise.all([1, 2, 3].map((n) => schedule(async () => n * 2))); + expect(results).toEqual([2, 4, 6]); +}); + +test("treats zero or negative limits as one", async () => { + const schedule = createApiScheduler(0, 10_000); + let active = 0; + let peak = 0; + + await Promise.all( + Array.from({ length: 4 }, () => + schedule(async () => { + active++; + peak = Math.max(peak, active); + await wait(2); + active--; + }), + ), + ); + + expect(peak).toBe(1); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/scheduler.ts b/packages/cli-core/src/commands/migrate/lib/scheduler.ts new file mode 100644 index 000000000..3f6704be9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/scheduler.ts @@ -0,0 +1,51 @@ +/** + * Concurrency gate plus rate pacing for BAPI calls. + * + * Replaces the standalone migration-tool's `p-limit` dependency: a bounded + * queue is a few lines, and the compiled binary carries one fewer package. + * + * Both limits apply to individual API calls rather than whole users, so a user + * with ten extra email addresses cannot burst past the instance's rate limit. + */ + +/** Runs `fn` once a slot is free and the pacing interval has elapsed. */ +export type ApiScheduler = (fn: () => Promise) => Promise; + +export function createApiScheduler(concurrencyLimit: number, rateLimit: number): ApiScheduler { + const maxConcurrent = Math.max(1, Math.floor(concurrencyLimit)); + const intervalMs = Math.ceil(1000 / Math.max(1, rateLimit)); + const waiting: (() => void)[] = []; + let active = 0; + let nextRequestAt = 0; + + function acquire(): Promise { + if (active < maxConcurrent) { + active++; + return Promise.resolve(); + } + return new Promise((resolve) => waiting.push(resolve)); + } + + function release(): void { + const next = waiting.shift(); + // Hand the slot straight to the next waiter; `active` is unchanged because + // the slot never actually frees up. + if (next) next(); + else active--; + } + + return async (fn) => { + await acquire(); + try { + const now = Date.now(); + const waitMs = Math.max(0, nextRequestAt - now); + nextRequestAt = Math.max(now, nextRequestAt) + intervalMs; + if (waitMs > 0) { + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + return await fn(); + } finally { + release(); + } + }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/settings.test.ts b/packages/cli-core/src/commands/migrate/lib/settings.test.ts new file mode 100644 index 000000000..579d508df --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/settings.test.ts @@ -0,0 +1,42 @@ +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { loadSettings, saveSettings } from "./settings.ts"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-")); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(path.join(workDir, ".settings"), { force: true }); +}); + +test("returns empty settings when the file is absent", () => { + expect(loadSettings()).toEqual({}); +}); + +test("round-trips the transformer key and file path", () => { + saveSettings({ key: "clerk", file: "users.json" }); + expect(loadSettings()).toEqual({ key: "clerk", file: "users.json" }); +}); + +test("writes to the current working directory", () => { + saveSettings({ key: "clerk" }); + expect(fs.existsSync(path.join(workDir, ".settings"))).toBe(true); +}); + +test("treats a corrupt settings file as empty rather than failing the run", () => { + fs.writeFileSync(path.join(workDir, ".settings"), "{not json"); + expect(loadSettings()).toEqual({}); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/settings.ts b/packages/cli-core/src/commands/migrate/lib/settings.ts new file mode 100644 index 000000000..fc9dfcd7e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/settings.ts @@ -0,0 +1,43 @@ +/** + * The cwd-relative `.settings` file: what this directory last migrated, and + * with which transformer. + * + * Ported from the standalone migration-tool's `src/lib/settings.ts`. Kept out + * of `~/.config/clerk/config.json` on purpose — that file is keyed by linked + * project identity, not by "which export file am I working through". + * + * Both halves fail silently: a missing, unreadable or unwritable `.settings` + * only costs the user a remembered default. + */ + +import fs from "node:fs"; +import path from "node:path"; +import type { Settings } from "../types.ts"; + +const SETTINGS_FILE = ".settings"; + +function settingsPath(): string { + return path.join(process.cwd(), SETTINGS_FILE); +} + +/** Reads saved settings, or `{}` when absent or corrupt. */ +export function loadSettings(): Settings { + try { + const file = settingsPath(); + if (fs.existsSync(file)) { + return JSON.parse(fs.readFileSync(file, "utf-8")) as Settings; + } + } catch { + // Corrupt or unreadable settings are indistinguishable from none. + } + return {}; +} + +/** Persists settings for the next run in this directory. */ +export function saveSettings(settings: Settings): void { + try { + fs.writeFileSync(settingsPath(), JSON.stringify(settings, null, 2)); + } catch { + // Read-only cwd; the run itself is unaffected. + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts b/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts new file mode 100644 index 000000000..00cde92c5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, test } from "bun:test"; +import { toClerkStrategy } from "./clerk-config.ts"; +import { + countProviders, + findDisabledProviders, + findUsersWithOnlyDisabledProviders, + getUserProviders, +} from "./supabase-providers.ts"; + +/** A Supabase row carrying the given providers, in the JSON export's shape. */ +const user = (id: string, providers: string[] | string | undefined) => ({ + id, + raw_app_meta_data: + providers === undefined ? undefined : JSON.stringify({ provider: "email", providers }), +}); + +describe("getUserProviders", () => { + test("reads providers from a JSON-string column, as a CSV export writes it", () => { + expect(getUserProviders(user("u1", ["email", "discord"]))).toEqual(["email", "discord"]); + }); + + test("reads providers from an object column, as a JSON export writes it", () => { + expect(getUserProviders({ id: "u1", raw_app_meta_data: { providers: ["google"] } })).toEqual([ + "google", + ]); + }); + + test("splits a delimited providers string", () => { + expect( + getUserProviders({ id: "u1", raw_app_meta_data: { providers: "email, discord" } }), + ).toEqual(["email", "discord"]); + }); + + test.each([ + ["missing column", { id: "u1" }], + ["unparseable column", { id: "u1", raw_app_meta_data: "{not json" }], + ["array column", { id: "u1", raw_app_meta_data: "[]" }], + ["no providers key", { id: "u1", raw_app_meta_data: '{"provider":"email"}' }], + ])("returns nothing for a %s", (_label, row) => { + expect(getUserProviders(row)).toEqual([]); + }); +}); + +describe("toClerkStrategy", () => { + test.each([ + ["google", "oauth_google"], + ["discord", "oauth_discord"], + ["github", "oauth_github"], + ["azure", "oauth_microsoft"], + ["twitter", "oauth_x"], + ["slack_oidc", "oauth_slack"], + ])("%s -> %s", (provider, strategy) => { + expect(toClerkStrategy(provider)).toBe(strategy); + }); +}); + +describe("countProviders", () => { + test("counts each provider across the export", () => { + expect( + countProviders([ + user("u1", ["email"]), + user("u2", ["email", "discord"]), + user("u3", ["discord"]), + ]), + ).toEqual({ email: 2, discord: 2 }); + }); +}); + +describe("findDisabledProviders", () => { + test("names the social providers Clerk does not have enabled", () => { + const rows = [user("u1", ["email", "google"]), user("u2", ["discord"])]; + expect(findDisabledProviders(rows, ["oauth_google"], toClerkStrategy)).toEqual(["discord"]); + }); + + test("never treats email or phone as disabled", () => { + const rows = [user("u1", ["email"]), user("u2", ["phone"]), user("u3", ["anonymous_users"])]; + expect(findDisabledProviders(rows, [], toClerkStrategy)).toEqual([]); + }); + + test("returns nothing when every provider is enabled", () => { + const rows = [user("u1", ["google"]), user("u2", ["github"])]; + expect(findDisabledProviders(rows, ["oauth_google", "oauth_github"], toClerkStrategy)).toEqual( + [], + ); + }); +}); + +describe("findUsersWithOnlyDisabledProviders", () => { + test("excludes a user whose sole provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", ["discord"])], ["discord"]); + expect([...result.excludedIds]).toEqual(["u1"]); + expect(result.byProvider).toEqual({ discord: 1 }); + }); + + test("keeps a user who can still sign in with email", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["email", "discord"])], + ["discord"], + ); + expect(result.excludedIds.size).toBe(0); + }); + + test("keeps a user who has another enabled social provider", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["google", "discord"])], + ["discord"], + ); + expect(result.excludedIds.size).toBe(0); + }); + + test("excludes a user whose every provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["discord", "twitch"])], + ["discord", "twitch"], + ); + expect([...result.excludedIds]).toEqual(["u1"]); + expect(result.byProvider).toEqual({ discord: 1, twitch: 1 }); + }); + + test("keeps a user with no provider data at all", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", undefined)], ["discord"]); + expect(result.excludedIds.size).toBe(0); + }); + + test("excludes nobody when no provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", ["discord"])], []); + expect(result.excludedIds.size).toBe(0); + }); + + test("reports a per-provider breakdown across many users", () => { + const result = findUsersWithOnlyDisabledProviders( + [ + user("u1", ["discord"]), + user("u2", ["discord"]), + user("u3", ["twitch"]), + user("u4", ["email", "discord"]), + ], + ["discord", "twitch"], + ); + expect([...result.excludedIds]).toEqual(["u1", "u2", "u3"]); + expect(result.byProvider).toEqual({ discord: 2, twitch: 1 }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts b/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts new file mode 100644 index 000000000..d123dd5e0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts @@ -0,0 +1,145 @@ +/** + * Cross-references the social providers in a Supabase export against what the + * destination Clerk instance has enabled. + * + * Ported from the standalone migration-tool's `src/lib/supabase.ts`, minus its + * hand-rolled CSV parser — the export is read through the same + * `readRawUsers` path every other Supabase read uses. + */ + +import { readRawUsers } from "./transform.ts"; + +/** + * Supabase lists these alongside social providers in `providers`, but they are + * built into Clerk and can never be "not enabled". + */ +export const NON_SOCIAL_PROVIDERS = new Set(["email", "phone", "anonymous_users"]); + +function parseMaybeJson(value: unknown): unknown { + if (typeof value !== "string") return value; + try { + return JSON.parse(value); + } catch { + return value; + } +} + +/** + * Reads a user's auth providers from `raw_app_meta_data`. + * + * The column arrives as a JSON string from a CSV export and as an object from + * a JSON one, and its `providers` value is itself sometimes a string. + */ +export function getUserProviders(user: Record): string[] { + const appMeta = parseMaybeJson(user.raw_app_meta_data); + if (!appMeta || typeof appMeta !== "object" || Array.isArray(appMeta)) return []; + + const providers = parseMaybeJson((appMeta as Record).providers); + if (Array.isArray(providers)) { + return providers.map((provider) => String(provider).trim()).filter(Boolean); + } + if (typeof providers === "string") { + return providers + .split(/[,|]/) + .map((provider) => provider.trim()) + .filter(Boolean); + } + return []; +} + +export type ProviderExclusions = { + /** Source IDs of users to skip. */ + excludedIds: Set; + /** How many excluded users each disabled provider accounts for. */ + byProvider: Record; +}; + +/** + * Finds the users whose *only* way in is a provider Clerk does not have + * enabled. + * + * A user keeps their place if any one of their providers still works — + * including email and phone. Excluding on "has at least one disabled provider" + * instead would drop users who could sign in perfectly well another way. + * + * @param disabled - Supabase provider keys not enabled in Clerk. + */ +export function findUsersWithOnlyDisabledProviders( + users: Record[], + disabled: string[], +): ProviderExclusions { + const empty: ProviderExclusions = { excludedIds: new Set(), byProvider: {} }; + if (disabled.length === 0) return empty; + + const disabledSet = new Set(disabled); + const excludedIds = new Set(); + const byProvider: Record = {}; + + for (const user of users) { + const providers = getUserProviders(user); + // No provider data means no basis to exclude — err towards importing. + if (providers.length === 0) continue; + + const hasUsableProvider = providers.some( + (provider) => NON_SOCIAL_PROVIDERS.has(provider) || !disabledSet.has(provider), + ); + if (hasUsableProvider) continue; + + excludedIds.add(String(user.id)); + for (const provider of providers.filter((p) => disabledSet.has(p))) { + byProvider[provider] = (byProvider[provider] ?? 0) + 1; + } + } + + return { excludedIds, byProvider }; +} + +/** Counts users per provider across the export, for reporting. */ +export function countProviders(users: Record[]): Record { + const counts: Record = {}; + for (const user of users) { + for (const provider of getUserProviders(user)) { + counts[provider] = (counts[provider] ?? 0) + 1; + } + } + return counts; +} + +/** + * The same counts, minus Supabase's pseudo-providers. + * + * Supabase lists `email` and `phone` in `providers` next to real connections, + * but Clerk has no `oauth_email` to enable — so anything cross-referencing + * against the instance's social settings must drop them, or every + * password-based user reads as "not enabled in Clerk". + */ +export function countSocialProviders(users: Record[]): Record { + return Object.fromEntries( + Object.entries(countProviders(users)).filter( + ([provider]) => !NON_SOCIAL_PROVIDERS.has(provider), + ), + ); +} + +/** + * Every social provider present in the export that Clerk does not have + * enabled. + * + * @param enabledStrategies - Clerk strategy names (`oauth_google`, …). + * @param toStrategy - Maps a Supabase provider key to its Clerk strategy. + */ +export function findDisabledProviders( + users: Record[], + enabledStrategies: string[], + toStrategy: (provider: string) => string, +): string[] { + const enabled = new Set(enabledStrategies); + return Object.keys(countSocialProviders(users)).filter( + (provider) => !enabled.has(toStrategy(provider)), + ); +} + +/** Reads a Supabase export and returns its raw rows for provider analysis. */ +export async function readSupabaseRows(file: string): Promise[]> { + return readRawUsers(file, "supabase"); +} diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts new file mode 100644 index 000000000..efb9f1e52 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -0,0 +1,218 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import clerkTransformer from "../transformers/clerk.ts"; +import { + consolidateClerkIdentifiers, + flattenObjectSelectively, + getFileType, + loadUsersFromFile, + normalizeUserData, + transformKeys, + transformUsers, + validatePreparedUsers, +} from "./transform.ts"; + +const DATE_TIME = "2026-01-01T00-00-00"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-transform-")); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("getFileType", () => { + test.each([ + ["users.json", "application/json"], + ["users.CSV", "text/csv"], + ["users.txt", undefined], + ["users", undefined], + ])("%s -> %p", (file, expected) => { + expect(getFileType(file)).toBe(expected as never); + }); +}); + +describe("flattenObjectSelectively", () => { + test("flattens only paths the transformer references", () => { + const result = flattenObjectSelectively( + { _id: { $oid: "123" }, meta: { keep: "nested" }, email: "a@example.com" }, + { "_id.$oid": "userId", email: "email" }, + ); + expect(result).toEqual({ + "_id.$oid": "123", + meta: { keep: "nested" }, + email: "a@example.com", + }); + }); + + test("leaves arrays intact", () => { + expect(flattenObjectSelectively({ tags: [{ a: 1 }] }, { "tags.a": "x" })).toEqual({ + tags: [{ a: 1 }], + }); + }); +}); + +describe("transformKeys", () => { + test("renames mapped fields and passes unmapped ones through", () => { + expect( + transformKeys( + { id: "u1", primary_email_address: "a@example.com", extra: "kept" }, + clerkTransformer, + ), + ).toEqual({ userId: "u1", email: "a@example.com", extra: "kept" }); + }); + + test.each([ + ["empty string", ""], + ["stringified empty object", '"{}"'], + ["null", null], + ])("drops fields whose value is %s", (_label, value) => { + expect(transformKeys({ id: "u1", first_name: value }, clerkTransformer)).toEqual({ + userId: "u1", + }); + }); +}); + +describe("normalizeUserData", () => { + test.each([ + ["comma-delimited emails", { email: "a@x.dev,b@x.dev" }, { email: ["a@x.dev", "b@x.dev"] }], + ["pipe-delimited emails", { email: "a@x.dev|b@x.dev" }, { email: ["a@x.dev", "b@x.dev"] }], + ["JSON array string", { email: '["a@x.dev"]' }, { email: ["a@x.dev"] }], + ["string boolean", { banned: "true" }, { banned: true }], + ["numeric boolean", { banned: 1 }, { banned: true }], + ["numeric string limit", { createOrganizationsLimit: "5" }, { createOrganizationsLimit: 5 }], + ["JSON metadata", { publicMetadata: '{"plan":"pro"}' }, { publicMetadata: { plan: "pro" } }], + ["date string", { createdAt: "2024-01-01" }, { createdAt: "2024-01-01T00:00:00.000Z" }], + ])("normalizes %s", (_label, input, expected) => { + expect(normalizeUserData(input)).toMatchObject(expected); + }); + + test("deletes fields that normalize to nothing", () => { + const result = normalizeUserData({ email: " ", publicMetadata: "", createdAt: "" }); + expect("email" in result).toBe(false); + expect("publicMetadata" in result).toBe(false); + expect("createdAt" in result).toBe(false); + }); + + test("leaves an unparseable date as-is for the schema to reject", () => { + expect(normalizeUserData({ createdAt: "yesterday" }).createdAt).toBe("yesterday"); + }); +}); + +describe("consolidateClerkIdentifiers", () => { + test("merges primary and verified emails, deduping", () => { + const user: Record = { + email: "a@x.dev", + emailAddresses: ["a@x.dev", "b@x.dev"], + unverifiedEmailAddresses: ["b@x.dev", "c@x.dev"], + }; + consolidateClerkIdentifiers(user); + expect(user.email).toEqual(["a@x.dev", "b@x.dev"]); + expect(user.emailAddresses).toBeUndefined(); + // b@x.dev is already verified, so it must not reappear as unverified. + expect(user.unverifiedEmailAddresses).toEqual(["c@x.dev"]); + }); + + test("drops the unverified list when every entry is already verified", () => { + const user: Record = { + phone: "+15555550100", + unverifiedPhoneNumbers: ["+15555550100"], + }; + consolidateClerkIdentifiers(user); + expect(user.phone).toEqual(["+15555550100"]); + expect("unverifiedPhoneNumbers" in user).toBe(false); + }); +}); + +describe("validatePreparedUsers", () => { + test("keeps valid users and counts the rest", () => { + const result = validatePreparedUsers( + [{ userId: "u1", email: "a@x.dev" }, { userId: "u2" }, { userId: "u3", username: "carol" }], + DATE_TIME, + ); + expect(result.users.map((user) => user.userId)).toEqual(["u1", "u3"]); + expect(result.validationFailed).toBe(1); + }); + + test("aborts the whole run on an unknown password hasher", () => { + expect(() => + validatePreparedUsers( + [{ userId: "u1", email: "a@x.dev", password: "d", passwordHasher: "rot13" }], + DATE_TIME, + ), + ).toThrow(CliError); + }); +}); + +describe("transformUsers", () => { + test("maps, consolidates and validates a Clerk export", () => { + const { transformedData, validationFailed } = transformUsers( + [ + { + id: "u1", + primary_email_address: "a@x.dev", + verified_email_addresses: ["a@x.dev", "b@x.dev"], + first_name: "Alice", + }, + ], + "clerk", + DATE_TIME, + ); + expect(validationFailed).toBe(0); + expect(transformedData[0]).toMatchObject({ + userId: "u1", + email: ["a@x.dev", "b@x.dev"], + firstName: "Alice", + }); + }); + + test("skips validation when asked, so analysis passes see every row", () => { + const { transformedData, validationFailed } = transformUsers( + [{ id: "u1" }], + "clerk", + DATE_TIME, + { + validate: false, + }, + ); + expect(transformedData).toHaveLength(1); + expect(validationFailed).toBe(0); + }); +}); + +describe("loadUsersFromFile", () => { + test("reads a JSON export", async () => { + fs.writeFileSync( + path.join(workDir, "users.json"), + JSON.stringify([{ id: "u1", primary_email_address: "a@x.dev" }]), + ); + const { users } = await loadUsersFromFile("users.json", "clerk", DATE_TIME); + expect(users).toHaveLength(1); + expect(users[0]?.userId).toBe("u1"); + }); + + test("reads a CSV export, including quoted commas", async () => { + fs.writeFileSync( + path.join(workDir, "users.csv"), + 'id,primary_email_address,verified_email_addresses\nu2,a@x.dev,"a@x.dev,b@x.dev"\n', + ); + const { users } = await loadUsersFromFile("users.csv", "clerk", DATE_TIME); + expect(users[0]?.userId).toBe("u2"); + expect(users[0]?.email).toEqual(["a@x.dev", "b@x.dev"]); + }); + + test("rejects a JSON file that is not an array of users", async () => { + fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); + await expect(loadUsersFromFile("wrapped.json", "clerk", DATE_TIME)).rejects.toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts new file mode 100644 index 000000000..9f1b4fe08 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -0,0 +1,469 @@ +/** + * The load → transform → validate pipeline. + * + * Ported from the standalone migration-tool's `src/migrate/functions.ts` and + * the transform helpers in its `src/lib/index.ts`. Two dependencies were + * dropped along the way: `mime-types` (an extension check covers the two + * formats we accept) and the repo-specific `/samples/` path special-case. + */ + +import fs from "node:fs"; +import path from "node:path"; +import csvParser from "csv-parser"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import { getTransformer } from "../transformers/registry.ts"; +import { + PASSWORD_HASHERS, + type TransformContext, + type TransformerRegistryEntry, + type User, +} from "../types.ts"; +import { userSchema } from "../validator.ts"; +import { validationLogger } from "./logger.ts"; + +export type FileType = "application/json" | "text/csv"; + +export type TransformOptions = { + /** Set `false` to keep invalid rows, for analysis passes that count fields. */ + validate?: boolean; + /** Per-run values `postTransform` may need. */ + context?: TransformContext; +}; + +/** Resolves an import path against the current working directory. */ +export function resolveImportFilePath(file: string): string { + return path.resolve(process.cwd(), file.trim()); +} + +export function fileExists(file: string): boolean { + return fs.existsSync(resolveImportFilePath(file)); +} + +/** + * Classifies an import file by extension. + * + * @returns The MIME type, or `undefined` for anything that is not JSON or CSV. + */ +export function getFileType(file: string): FileType | undefined { + const ext = path.extname(resolveImportFilePath(file)).toLowerCase(); + if (ext === ".json") return "application/json"; + if (ext === ".csv") return "text/csv"; + return undefined; +} + +// --- Field mapping --------------------------------------------------------- + +/** + * Flattens only the nested paths a transformer actually references. + * + * Lets a transformer map `"_id.$oid"` onto `userId` without flattening + * (and thereby mangling) metadata objects it does not mention. + */ +export function flattenObjectSelectively( + obj: Record, + transformer: Record, + prefix = "", +): Record { + const result: Record = {}; + + for (const [key, value] of Object.entries(obj)) { + const currentPath = prefix ? `${prefix}.${key}` : key; + const hasNestedMapping = Object.keys(transformer).some((mapped) => + mapped.startsWith(`${currentPath}.`), + ); + + if (hasNestedMapping && value && typeof value === "object" && !Array.isArray(value)) { + Object.assign( + result, + flattenObjectSelectively(value as Record, transformer, currentPath), + ); + } else { + result[currentPath] = value; + } + } + + return result; +} + +/** Renames source fields onto Clerk's import schema, dropping empty values. */ +export function transformKeys( + data: Record, + transformerConfig: { transformer: Record }, +): Record { + const transformed: Record = {}; + const { transformer } = transformerConfig; + const flat = flattenObjectSelectively(data, transformer); + + for (const [key, value] of Object.entries(flat)) { + if (value !== "" && value !== '"{}"' && value !== null) { + transformed[transformer[key] ?? key] = value; + } + } + + return transformed; +} + +// --- Value normalization --------------------------------------------------- + +function parseJsonValue(value: string): unknown { + const trimmed = value.trim(); + if (!trimmed) return value; + if (!["[", "{", '"'].includes(trimmed[0] ?? "")) return value; + try { + return JSON.parse(trimmed); + } catch { + return value; + } +} + +function parseDelimitedStrings(field: unknown): string[] { + if (Array.isArray(field)) return field as string[]; + if (typeof field === "string" && field) { + const parsed = parseJsonValue(field); + if (Array.isArray(parsed)) { + return parsed.map((value) => String(value).trim()).filter(Boolean); + } + return field + .split(/[,|]/) + .map((value) => value.trim()) + .filter(Boolean); + } + return []; +} + +function normalizeStringArrayField(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => String(item).trim()).filter(Boolean); + } + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const parsed = parseJsonValue(trimmed); + if (Array.isArray(parsed)) { + return parsed.map((item) => String(item).trim()).filter(Boolean); + } + if (typeof parsed === "string") { + const parsedString = parsed.trim(); + if (parsedString.includes(",") || parsedString.includes("|")) { + return parsedString + .split(/[,|]/) + .map((item) => item.trim()) + .filter(Boolean); + } + return parsedString; + } + return parsed; +} + +function normalizeBooleanField(value: unknown): unknown { + if (typeof value === "boolean") return value; + if (typeof value === "number") { + if (value === 1) return true; + if (value === 0) return false; + return value; + } + if (typeof value !== "string") return value; + + const normalized = value.trim().toLowerCase(); + if (["true", "1", "yes", "y"].includes(normalized)) return true; + if (["false", "0", "no", "n"].includes(normalized)) return false; + return value; +} + +function normalizeNumberField(value: unknown): unknown { + if (typeof value === "number") return value; + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? parsed : value; +} + +function normalizeMetadataField(value: unknown): unknown { + if (value === undefined || value === null || value === "") return undefined; + if (typeof value !== "string") return value; + + const parsed = parseJsonValue(value); + return typeof parsed === "string" ? value : parsed; +} + +function normalizeDateField(value: unknown): unknown { + if (value instanceof Date) return value.toISOString(); + if (typeof value === "number") { + const date = new Date(value); + return Number.isNaN(date.getTime()) ? value : date.toISOString(); + } + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const date = new Date(trimmed); + return Number.isNaN(date.getTime()) ? value : date.toISOString(); +} + +const ARRAY_FIELDS = [ + "email", + "emailAddresses", + "unverifiedEmailAddresses", + "phone", + "phoneNumbers", + "unverifiedPhoneNumbers", + "backupCodes", +] as const; + +const BOOLEAN_FIELDS = [ + "backupCodesEnabled", + "banned", + "bypassClientTrust", + "createOrganizationEnabled", + "deleteSelfEnabled", + "skipLegalChecks", + "skipPasswordChecks", +] as const; + +const METADATA_FIELDS = ["unsafeMetadata", "publicMetadata", "privateMetadata"] as const; + +const DATE_FIELDS = ["createdAt", "legalAcceptedAt"] as const; + +/** + * Coerces CSV's all-strings-everything into the shapes the schema expects. + * + * A field that normalizes to `undefined` is deleted rather than set, so an + * empty CSV column does not look like an explicitly-null value to Clerk. + */ +export function normalizeUserData(user: Record): Record { + const normalized = { ...user }; + + const setOrDelete = (field: string, value: unknown) => { + if (value === undefined) delete normalized[field]; + else normalized[field] = value; + }; + + for (const field of ARRAY_FIELDS) { + setOrDelete(field, normalizeStringArrayField(normalized[field])); + } + for (const field of BOOLEAN_FIELDS) { + normalized[field] = normalizeBooleanField(normalized[field]); + } + for (const field of METADATA_FIELDS) { + setOrDelete(field, normalizeMetadataField(normalized[field])); + } + for (const field of DATE_FIELDS) { + setOrDelete(field, normalizeDateField(normalized[field])); + } + setOrDelete( + "createOrganizationsLimit", + normalizeNumberField(normalized.createOrganizationsLimit), + ); + + return normalized; +} + +/** + * Merges a Clerk export's three email fields (and three phone fields) into the + * verified/unverified pair the schema models, deduping across all of them. + */ +export function consolidateClerkIdentifiers(user: Record): void { + const merge = (primaryKey: string, verifiedKey: string, unverifiedKey: string) => { + const primary = user[primaryKey] as string | undefined; + const verified = parseDelimitedStrings(user[verifiedKey]); + const unverified = parseDelimitedStrings(user[unverifiedKey]); + + const all: string[] = []; + if (primary) all.push(primary); + for (const value of verified) { + if (!all.includes(value)) all.push(value); + } + if (all.length > 0) user[primaryKey] = all; + delete user[verifiedKey]; + + const extraUnverified = unverified.filter((value) => !all.includes(value)); + if (extraUnverified.length > 0) user[unverifiedKey] = extraUnverified; + else delete user[unverifiedKey]; + }; + + merge("email", "emailAddresses", "unverifiedEmailAddresses"); + merge("phone", "phoneNumbers", "unverifiedPhoneNumbers"); +} + +// --- Validation ------------------------------------------------------------ + +/** + * Validates prepared users, logging each failure and dropping it from the run. + * + * An unrecognized `passwordHasher` is the one failure that aborts instead: + * importing those users would store credentials nobody can ever sign in with, + * and the fix is a one-word edit to the transformer. + */ +export function validatePreparedUsers( + users: Record[], + dateTime: string, +): { users: User[]; validationFailed: number } { + const validated: User[] = []; + let validationFailed = 0; + + for (let i = 0; i < users.length; i++) { + const user = users[i] as Record; + const result = userSchema.safeParse(user); + + if (result.success) { + validated.push(result.data); + continue; + } + + validationFailed++; + const firstIssue = result.error.issues[0]; + if (!firstIssue) continue; + + if (firstIssue.path.includes("passwordHasher") && user.passwordHasher) { + const invalidHasher = + typeof user.passwordHasher === "string" + ? user.passwordHasher + : JSON.stringify(user.passwordHasher); + throw new CliError( + `Invalid password hasher "${invalidHasher}" on user ${String(user.userId)} (row ${i + 1}).\n` + + `Expected one of: ${PASSWORD_HASHERS.join(", ")}`, + { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: "https://clerk.com/docs/guides/development/migrating/overview", + }, + ); + } + + validationLogger( + { + error: firstIssue.message, + path: firstIssue.path as (string | number)[], + userId: (user.userId as string) || `row-${i}`, + row: i, + }, + dateTime, + ); + } + + return { users: validated, validationFailed }; +} + +function addDefaultFields( + users: Record[], + transformer: TransformerRegistryEntry, +): Record[] { + if (!transformer.defaults) return users; + return users.map((user) => ({ ...user, ...transformer.defaults })); +} + +/** + * Maps, normalizes and (unless disabled) validates a batch of raw users. + * + * @param options.validate - Set `false` to get the mapped shape without + * dropping invalid rows, for analysis passes that count fields. + * @param options.context - Per-run values `postTransform` may need, e.g. + * Firebase's hash parameters. + */ +export function transformUsers( + users: Record[], + key: string, + dateTime: string, + options: TransformOptions = {}, +): { transformedData: User[]; validationFailed: number } { + const transformer = getTransformer(key); + const context = options.context ?? {}; + const transformed: Record[] = []; + + for (const user of users) { + const mapped = transformKeys(user, transformer); + + if (key === "clerk") { + consolidateClerkIdentifiers(mapped); + } + transformer.postTransform?.(mapped, context); + + transformed.push(normalizeUserData(mapped)); + } + + if (options.validate === false) { + return { transformedData: transformed as User[], validationFailed: 0 }; + } + + const result = validatePreparedUsers(transformed, dateTime); + return { transformedData: result.users, validationFailed: result.validationFailed }; +} + +// --- File loading ---------------------------------------------------------- + +async function readCsv(filePath: string): Promise[]> { + return new Promise((resolve, reject) => { + const users: Record[] = []; + fs.createReadStream(filePath) + .pipe(csvParser({ skipComments: true })) + .on("data", (row: Record) => users.push(row)) + .on("error", reject) + .on("end", () => resolve(users)); + }); +} + +async function readUsersFromFile( + file: string, + transformer: TransformerRegistryEntry, +): Promise[]> { + let filePath = resolveImportFilePath(file); + const type = getFileType(file); + let preExtracted: Record[] | undefined; + + if (transformer.preTransform) { + const result = await transformer.preTransform(filePath, type ?? ""); + filePath = result.filePath; + preExtracted = result.data; + } + + if (type === "text/csv") return readCsv(filePath); + if (preExtracted) return preExtracted; + + const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + if (!Array.isArray(parsed)) { + throw new CliError(`Expected ${file} to contain a JSON array of users, got ${typeof parsed}.`, { + code: ERROR_CODE.INVALID_JSON, + }); + } + return parsed as Record[]; +} + +/** + * Reads the export exactly as the transformer sees it, before any field + * mapping. + * + * Used by the Supabase provider cross-reference, which reads + * `raw_app_meta_data` — a column no transformer maps, so it is gone by the time + * users are transformed. + */ +export async function readRawUsers(file: string, key: string): Promise[]> { + return readUsersFromFile(file, getTransformer(key)); +} + +/** + * Reads a JSON or CSV export and returns the users ready to import. + * + * @param options - Passed through to {@link transformUsers}. + */ +export async function loadUsersFromFile( + file: string, + key: string, + dateTime: string, + options: TransformOptions = {}, +): Promise<{ users: User[]; validationFailed: number }> { + const transformer = getTransformer(key); + const raw = await readUsersFromFile(file, transformer); + const withDefaults = addDefaultFields(raw, transformer); + const { transformedData, validationFailed } = transformUsers( + withDefaults, + key, + dateTime, + options, + ); + return { users: transformedData, validationFailed }; +} diff --git a/packages/cli-core/src/commands/migrate/logs/clean.ts b/packages/cli-core/src/commands/migrate/logs/clean.ts new file mode 100644 index 000000000..32d60355d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/clean.ts @@ -0,0 +1,69 @@ +/** + * `clerk migrate logs clean` — delete the local log files. + * + * Ported from the standalone migration-tool's `src/clean-logs/index.ts`. + * + * Destructive, and it sits one word away from `clerk migrate delete`, which + * destroys something entirely different (users in a Clerk instance). So the + * confirmation is not optional: interactive runs prompt, and non-interactive + * ones must say `-y` rather than being allowed to assume. + */ + +import fs from "node:fs"; +import { throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { confirm } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { listLogFiles } from "../lib/log-files.ts"; +import { getLogDir } from "../lib/logger.ts"; + +export type LogsCleanOptions = { + yes?: boolean; +}; + +export async function clean(options: LogsCleanOptions = {}): Promise { + const files = listLogFiles(); + + if (files.length === 0) { + log.info(`No migration logs to clean in ${getLogDir()}.`); + return; + } + + const label = `${files.length} log file${files.length === 1 ? "" : "s"}`; + + if (!options.yes) { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate logs clean\` deletes ${label} from ${getLogDir()} and cannot prompt here. Pass -y to confirm.`, + undefined, + undefined, + [ + { + command: "clerk migrate logs clean -y", + description: "Delete every migration log without prompting", + }, + ], + ); + } + + const proceed = await confirm({ message: `Delete ${label}?`, default: false }); + if (!proceed) throwUserAbort(); + } + + let deleted = 0; + const failures: string[] = []; + + for (const file of files) { + try { + fs.unlinkSync(file.path); + deleted++; + } catch (error) { + failures.push(`${file.name}: ${(error as Error).message}`); + } + } + + for (const failure of failures) log.warn(`Could not delete ${failure}`); + + log.success(`Deleted ${deleted} log file${deleted === 1 ? "" : "s"}.`); + if (failures.length > 0) process.exitCode = 1; +} diff --git a/packages/cli-core/src/commands/migrate/logs/convert.ts b/packages/cli-core/src/commands/migrate/logs/convert.ts new file mode 100644 index 000000000..104a236fe --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/convert.ts @@ -0,0 +1,121 @@ +/** + * `clerk migrate logs convert` — NDJSON to a JSON array. + * + * Ported from the standalone migration-tool's `src/convert-logs/index.ts`, + * with two changes: files can be named as positionals or `--all` instead of + * only through a picker, and a malformed line is reported with its line number + * rather than aborting the whole file. + */ + +import fs from "node:fs"; +import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { multiselect } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { findLogFile, listLogFiles, readNdjson, type LogFile } from "../lib/log-files.ts"; +import { getLogDir } from "../lib/logger.ts"; + +export type LogsConvertOptions = { + all?: boolean; + files?: string[]; +}; + +/** The `.json` sibling a log converts into. */ +export function outputPathFor(file: LogFile): string { + return file.path.replace(/\.log$/, ".json"); +} + +/** + * Resolves which files to convert: explicit positionals, `--all`, or a + * multiselect when a human gave neither. + */ +async function resolveTargets(options: LogsConvertOptions): Promise { + const available = listLogFiles(); + + if (available.length === 0) { + log.info(`No migration logs to convert in ${getLogDir()}.`); + return []; + } + + if (options.files && options.files.length > 0) { + return options.files.map((name) => { + const found = findLogFile(name); + if (!found) { + throw new CliError(`No log file named ${name} in ${getLogDir()}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + }); + } + return found; + }); + } + + if (options.all) return available; + + if (isAgent() || !isHuman()) { + throwUsageError( + "`clerk migrate logs convert` needs a file to convert and cannot prompt here. Name one or more log files, or pass --all.", + undefined, + undefined, + [ + { command: "clerk migrate logs convert --all", description: "Convert every log file" }, + { + command: `clerk migrate logs convert ${available[0]?.name ?? "migration-....log"}`, + description: "Convert one log file", + }, + ], + ); + } + + const chosen = await multiselect({ + message: "Which log files should be converted to JSON?", + options: available.map((file) => ({ + value: file.name, + label: file.name, + hint: `${file.entryCount} entries`, + })), + }); + if (chosen.length === 0) throwUserAbort(); + + return available.filter((file) => chosen.includes(file.name)); +} + +export async function convert(options: LogsConvertOptions = {}): Promise { + const targets = await resolveTargets(options); + if (targets.length === 0) return; + + let converted = 0; + let malformed = 0; + + for (const file of targets) { + const output = outputPathFor(file); + + try { + const { entries, errors } = readNdjson(file.path); + + // Reported per line, so a truncated final line from an interrupted run + // is visible rather than silently missing from the output. + for (const error of errors) { + malformed++; + log.warn(`${file.name}:${error.line} is not valid JSON and was skipped — ${error.message}`); + } + + fs.writeFileSync(output, JSON.stringify(entries, null, 2)); + converted++; + const count = `${entries.length} ${entries.length === 1 ? "entry" : "entries"}`; + log.info(`${file.name} → ${output.split("/").pop()} ${dim(`(${count})`)}`); + } catch (error) { + log.warn(`Could not convert ${file.name}: ${(error as Error).message}`); + process.exitCode = 1; + } + } + + if (converted > 0) { + log.success( + `Converted ${converted} log file${converted === 1 ? "" : "s"}. Originals left in place.`, + ); + } + if (malformed > 0) { + log.warn(`${malformed} malformed line${malformed === 1 ? "" : "s"} skipped.`); + } +} diff --git a/packages/cli-core/src/commands/migrate/logs/index.ts b/packages/cli-core/src/commands/migrate/logs/index.ts new file mode 100644 index 000000000..50279c8ff --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/index.ts @@ -0,0 +1,72 @@ +import { createArgument } from "@commander-js/extra-typings"; +import type { Command } from "@commander-js/extra-typings"; +import { clean } from "./clean.ts"; +import { convert } from "./convert.ts"; +import { list } from "./list.ts"; + +const logs = { clean, convert, list }; + +/** + * Registers `logs list|clean|convert` under the `migrate` group. + * + * Noun-verb, matching every other group in the CLI (`config pull`, `users + * list`) rather than the standalone tool's `clean-logs`/`convert-logs`, which + * were npm script names. Grouping also disambiguates the two deletes in this + * tree: `migrate logs clean` removes local files, `migrate delete` removes + * users from a Clerk instance. + */ +export function registerMigrateLogs(migrateCommand: Command<[], Record>): void { + const logsCommand = migrateCommand + .command("logs") + .description("Inspect, convert and clean up local migration logs") + .setExamples([ + { command: "clerk migrate logs", description: "List the local migration logs" }, + { command: "clerk migrate logs clean -y", description: "Delete every migration log" }, + { + command: "clerk migrate logs convert --all", + description: "Convert every log to a JSON array", + }, + ]); + + // Listing is read-only, so it is safe as the default for a bare + // `clerk migrate logs`. + logsCommand + .command("list", { isDefault: true }) + .description("List the log files in ./logs/") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate logs list", description: "Show type, timestamp, size and entries" }, + { command: "clerk migrate logs list --json", description: "Machine-readable listing" }, + ]) + .action((_opts, cmd) => logs.list(cmd.optsWithGlobals() as Parameters[0])); + + logsCommand + .command("clean") + .description("Delete the log files in ./logs/") + .option("-y, --yes", "Skip the confirmation prompt") + .setExamples([ + { command: "clerk migrate logs clean", description: "Delete after confirming" }, + { command: "clerk migrate logs clean -y", description: "Delete without prompting" }, + ]) + .action((_opts, cmd) => logs.clean(cmd.optsWithGlobals() as Parameters[0])); + + logsCommand + .command("convert") + .description("Convert NDJSON logs to JSON arrays for analysis") + .addArgument(createArgument("[file...]", "Log files to convert. Omit to pick interactively.")) + .option("--all", "Convert every log file") + .setExamples([ + { command: "clerk migrate logs convert --all", description: "Convert every log file" }, + { + command: "clerk migrate logs convert migration-2026-01-01T12-00-00.log", + description: "Convert one log file", + }, + { command: "clerk migrate logs convert", description: "Pick files interactively" }, + ]) + .action((files, _opts, cmd) => + logs.convert({ + ...(cmd.optsWithGlobals() as Parameters[0]), + files, + }), + ); +} diff --git a/packages/cli-core/src/commands/migrate/logs/list.ts b/packages/cli-core/src/commands/migrate/logs/list.ts new file mode 100644 index 000000000..cf30e24e0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/list.ts @@ -0,0 +1,64 @@ +/** + * `clerk migrate logs list` — what is in `./logs/`. + * + * New in the CLI: the standalone tool enumerated the directory only to build + * its own pickers. Exposing it gives a human a "what did I just do" view and + * an agent a read-only way to inspect a migration without parsing NDJSON. + */ + +import { cyan, dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { formatSize, listLogFiles, type LogFile } from "../lib/log-files.ts"; +import { getLogDir } from "../lib/logger.ts"; + +export type LogsListOptions = { + json?: boolean; +}; + +function toJson(files: LogFile[]) { + return files.map((file) => ({ + name: file.name, + kind: file.kind, + timestamp: file.timestamp, + size_bytes: file.sizeBytes, + entry_count: file.entryCount, + path: file.path, + })); +} + +export function list(options: LogsListOptions = {}): void { + const files = listLogFiles(); + + if (options.json) { + log.data(JSON.stringify(toJson(files), null, 2)); + return; + } + + if (files.length === 0) { + log.info(`No migration logs in ${getLogDir()}.`); + return; + } + + const kindWidth = Math.max(...files.map((file) => file.kind.length), "TYPE".length) + 2; + const timeWidth = Math.max(...files.map((file) => file.timestamp.length), "TIMESTAMP".length) + 2; + const sizeWidth = Math.max(...files.map((file) => formatSize(file.sizeBytes).length), 4) + 2; + + log.info( + dim("TYPE".padEnd(kindWidth)) + + dim("TIMESTAMP".padEnd(timeWidth)) + + dim("SIZE".padEnd(sizeWidth)) + + dim("ENTRIES"), + ); + + for (const file of files) { + log.info( + cyan(file.kind.padEnd(kindWidth)) + + (file.timestamp || dim("—")).padEnd(timeWidth) + + dim(formatSize(file.sizeBytes).padEnd(sizeWidth)) + + String(file.entryCount), + ); + } + + log.info(""); + log.info(dim(`${files.length} log file${files.length === 1 ? "" : "s"} in ${getLogDir()}`)); +} diff --git a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts new file mode 100644 index 000000000..36bd10747 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts @@ -0,0 +1,177 @@ +/** + * The prompting half of `logs clean` and `logs convert`. + * + * Kept separate because `mock.module` registrations are process-lifetime, and + * `bun test --parallel` puts several files in each worker — so a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. Human mode itself needs no mock: `setMode` is the + * supported override. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +type ConfirmPrompt = { message: string; default?: boolean }; +type MultiselectPrompt = { + message: string; + options: { value: string; label: string; hint?: string }[]; +}; + +const mockConfirm = mock(async (_config: ConfirmPrompt) => true); +const mockMultiselect = mock(async (_config: MultiselectPrompt) => [] as string[]); + +mock.module("../../../lib/prompts.ts", () => ({ + confirm: (config: ConfirmPrompt) => mockConfirm(config), + multiselect: (config: MultiselectPrompt) => mockMultiselect(config), + text: async () => "", + password: async () => "", + editor: async () => "{}", +})); + +const { clean } = await import("./clean.ts"); +const { convert } = await import("./convert.ts"); +const { UserAbortError } = await import("../../../lib/errors.ts"); +const { getLogDir } = await import("../lib/logger.ts"); + +let originalMode: Mode; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; + +const MIGRATION = "migration-2026-01-01T12-00-00.log"; +const DELETION = "user-deletion-2026-02-01T12-00-00.log"; + +beforeAll(() => { + originalMode = getMode(); + setMode("human"); + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logs-int-"))); + process.chdir(workDir); +}); + +afterAll(() => { + setMode(originalMode); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + mockConfirm.mockReset(); + mockMultiselect.mockReset(); + mockConfirm.mockResolvedValue(true); + mockMultiselect.mockResolvedValue([]); + fs.rmSync(getLogDir(), { recursive: true, force: true }); + process.exitCode = 0; +}); + +function writeLog(name: string, entries: unknown[]): void { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync( + path.join(getLogDir(), name), + entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n", + ); +} + +describe("logs clean", () => { + test("prompts before deleting anything", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ a: 1 }]); + + await clean(); + + expect(mockConfirm).toHaveBeenCalledTimes(1); + expect(mockConfirm.mock.calls[0]?.[0]?.message).toContain("2 log files"); + expect(fs.readdirSync(getLogDir())).toEqual([]); + }); + + // Deleting on a stray enter would be the wrong default for a destructive + // command sitting next to `clerk migrate delete`. + test("defaults the prompt to no", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await clean(); + + expect(mockConfirm.mock.calls[0]?.[0]?.default).toBe(false); + }); + + test("declining leaves every file in place", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + mockConfirm.mockResolvedValue(false); + + await expect(clean()).rejects.toThrow(UserAbortError); + + expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + }); + + test("-y skips the prompt entirely", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await clean({ yes: true }); + + expect(mockConfirm).not.toHaveBeenCalled(); + expect(fs.readdirSync(getLogDir())).toEqual([]); + }); + + test("does not prompt when there is nothing to delete", async () => { + await clean(); + expect(mockConfirm).not.toHaveBeenCalled(); + }); +}); + +describe("logs convert", () => { + test("offers a multiselect when given neither files nor --all", async () => { + writeLog(MIGRATION, [{ a: 1 }, { b: 2 }]); + writeLog(DELETION, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([MIGRATION]); + + await convert(); + + const options = mockMultiselect.mock.calls[0]?.[0]?.options; + expect(options?.map((option) => option.value)).toEqual([DELETION, MIGRATION]); + expect(options?.[1]?.hint).toBe("2 entries"); + }); + + test("converts only what was selected", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([MIGRATION]); + + await convert(); + + expect(fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json"))).toEqual([ + "migration-2026-01-01T12-00-00.json", + ]); + }); + + test("selecting nothing aborts without writing", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([]); + + await expect(convert()).rejects.toThrow(UserAbortError); + + expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + }); + + test("does not prompt when --all was passed", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await convert({ all: true }); + + expect(mockMultiselect).not.toHaveBeenCalled(); + expect(captured.err).toContain("Converted 1 log file"); + }); + + test("does not prompt when files were named", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await convert({ files: [MIGRATION] }); + + expect(mockMultiselect).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/logs/logs.test.ts b/packages/cli-core/src/commands/migrate/logs/logs.test.ts new file mode 100644 index 000000000..5be2d232b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/logs.test.ts @@ -0,0 +1,219 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { clean } from "./clean.ts"; +import { convert } from "./convert.ts"; +import { list } from "./list.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logs-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); + process.exitCode = 0; +}); + +function writeLog(name: string, entries: unknown[]): void { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync( + path.join(getLogDir(), name), + entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n", + ); +} + +const MIGRATION = "migration-2026-01-01T12-00-00.log"; +const DELETION = "user-deletion-2026-02-01T12-00-00.log"; + +describe("logs list", () => { + test("says so plainly when there is no logs directory", () => { + list(); + expect(captured.err).toContain("No migration logs in"); + }); + + test("says so plainly when the directory is empty", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + list(); + expect(captured.err).toContain("No migration logs in"); + }); + + test("reports type, timestamp, size and entry count", () => { + writeLog(MIGRATION, [{ userId: "u1" }, { userId: "u2" }, { userId: "u3" }]); + + list(); + + expect(captured.err).toContain("TYPE"); + expect(captured.err).toContain("TIMESTAMP"); + expect(captured.err).toContain("SIZE"); + expect(captured.err).toContain("ENTRIES"); + expect(captured.err).toContain("migration"); + expect(captured.err).toContain("2026-01-01T12-00-00"); + expect(captured.err).toMatch(/\bB\b/); + expect(captured.err).toContain("3"); + }); + + test("lists every log kind", () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ a: 1 }]); + + list(); + + expect(captured.err).toContain("migration"); + expect(captured.err).toContain("deletion"); + expect(captured.err).toContain("2 log files"); + }); + + test("--json emits a machine-readable listing on stdout", () => { + writeLog(MIGRATION, [{ userId: "u1" }]); + + list({ json: true }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed).toHaveLength(1); + expect(parsed[0]).toMatchObject({ + name: MIGRATION, + kind: "migration", + timestamp: "2026-01-01T12-00-00", + entry_count: 1, + }); + }); + + test("--json emits an empty array rather than prose when there are no logs", () => { + list({ json: true }); + expect(JSON.parse(captured.out)).toEqual([]); + }); +}); + +describe("logs clean", () => { + test("says so plainly when there is nothing to clean", async () => { + await clean({ yes: true }); + expect(captured.err).toContain("No migration logs to clean"); + }); + + // Tests run non-TTY, which is the same signal an agent gives. + test("refuses without -y when it cannot prompt, and explains", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await expect(clean()).rejects.toThrow(/cannot prompt here.*Pass -y/s); + expect(fs.existsSync(path.join(getLogDir(), MIGRATION))).toBe(true); + }); + + test("names how many files are at stake when it refuses", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ a: 1 }]); + + await expect(clean()).rejects.toThrow(/2 log files/); + }); + + test("-y deletes the log files and reports the count", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ a: 1 }]); + + await clean({ yes: true }); + + expect(fs.readdirSync(getLogDir())).toEqual([]); + expect(captured.err).toContain("Deleted 2 log files"); + }); + + test("leaves converted JSON output alone", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + fs.writeFileSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"), "[]"); + + await clean({ yes: true }); + + expect(fs.readdirSync(getLogDir())).toEqual(["migration-2026-01-01T12-00-00.json"]); + }); +}); + +describe("logs convert", () => { + test("says so plainly when there is nothing to convert", async () => { + await convert({ all: true }); + expect(captured.err).toContain("No migration logs to convert"); + }); + + test("writes a JSON array alongside the original, leaving it intact", async () => { + writeLog(MIGRATION, [{ userId: "u1" }, { userId: "u2" }]); + + await convert({ files: [MIGRATION] }); + + const output = path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"); + expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([ + { userId: "u1" }, + { userId: "u2" }, + ]); + expect(fs.existsSync(path.join(getLogDir(), MIGRATION))).toBe(true); + expect(captured.err).toContain("Originals left in place"); + }); + + test("--all converts every log file", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + writeLog(DELETION, [{ b: 2 }]); + + await convert({ all: true }); + + const written = fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json")); + expect(written.sort()).toEqual([ + "migration-2026-01-01T12-00-00.json", + "user-deletion-2026-02-01T12-00-00.json", + ]); + }); + + test("accepts a path and resolves it against ./logs/", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await convert({ files: [`./logs/${MIGRATION}`] }); + + expect(fs.existsSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"))).toBe(true); + }); + + test("fails clearly on a file that is not there", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await expect(convert({ files: ["migration-nope.log"] })).rejects.toThrow(CliError); + }); + + // Silently dropping the line would leave a JSON array that looks complete. + test("reports a malformed line by number and converts the rest", async () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync(path.join(getLogDir(), MIGRATION), '{"a":1}\n{"b":\n{"c":3}\n'); + + await convert({ files: [MIGRATION] }); + + expect(captured.err).toContain(`${MIGRATION}:2`); + expect(captured.err).toContain("1 malformed line skipped"); + + const output = path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"); + expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([{ a: 1 }, { c: 3 }]); + }); + + test("refuses without a target when it cannot prompt, naming the alternatives", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await expect(convert()).rejects.toThrow(/cannot prompt here/); + expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + }); + + test("reports the entry count per converted file", async () => { + writeLog(MIGRATION, [{ a: 1 }, { b: 2 }, { c: 3 }]); + + await convert({ all: true }); + + expect(captured.err).toContain("3 entries"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts new file mode 100644 index 000000000..ef5a05f85 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -0,0 +1,124 @@ +/** + * Keeps README.md and the command tree honest about each other. + * + * This README documents six export platforms, six transformers and three log + * subcommands across ~600 lines. Checking it by eye at review time does not + * scale, and a doc that names a flag the binary rejects is worse than no doc: + * the reader trusts it and gets a usage error. + * + * Both directions are checked — every example must resolve, and every flag must + * be written down — so neither renaming a flag nor adding one passes silently. + */ + +import { describe, expect, test } from "bun:test"; +import type { Command } from "commander"; +import { createProgram } from "../../cli-program.ts"; + +const README = await Bun.file(new URL("./README.md", import.meta.url)).text(); + +/** Fenced blocks only, so prose that merely mentions a flag is not parsed. */ +function fencedBlocks(markdown: string): string[] { + return [...markdown.matchAll(/^```[a-z]*\n([\s\S]*?)^```/gm)].map((match) => match[1] ?? ""); +} + +/** + * Every `clerk migrate …` invocation the README puts in front of a reader, from + * fenced blocks and inline backticks alike — both get copied. + */ +function documentedCommands(markdown: string): string[] { + const found = new Set(); + + for (const block of fencedBlocks(markdown)) { + // Line continuations first: the Firebase example spans three lines. + for (const line of block.replace(/\\\n\s*/g, " ").split("\n")) { + const start = line.indexOf("clerk migrate"); + // A command never contains a backtick or a `#`; the sample error output + // that quotes `clerk migrate` mid-sentence does. + if (start !== -1) found.add(line.slice(start).split(/[`#]/)[0]!.trim()); + } + } + + for (const match of markdown.matchAll(/`(clerk migrate[^`]*)`/g)) { + found.add(match[1]!.trim()); + } + + return [...found]; +} + +/** Walks as deep as the tree allows; the first flag or positional stops it. */ +function resolve(tokens: string[]): { command: Command; rest: string[] } { + let command = createProgram() as Command; + let index = 0; + for (; index < tokens.length; index++) { + const child = command.commands.find( + (candidate) => + candidate.name() === tokens[index] || candidate.aliases().includes(tokens[index]!), + ); + if (!child) break; + command = child; + } + return { command, rest: tokens.slice(index) }; +} + +function flagsOf(command: Command): string[] { + return command.options.flatMap( + (option) => [option.short, option.long].filter(Boolean) as string[], + ); +} + +/** Every command under `migrate`, so no subcommand escapes the flag sweep. */ +function migrateTree(): { path: string; command: Command }[] { + const collected: { path: string; command: Command }[] = []; + const visit = (command: Command, path: string) => { + collected.push({ path, command }); + for (const child of command.commands) { + if (child.name() !== "help") visit(child, `${path} ${child.name()}`); + } + }; + visit(resolve(["migrate"]).command, "migrate"); + return collected; +} + +const EXAMPLES = documentedCommands(README); + +/** One case per (example, flag) pair, so a failure names the exact flag. */ +const FLAG_USES: [string, string][] = EXAMPLES.flatMap((example) => + example + .split(/\s+/) + .filter((token) => token.startsWith("-")) + .map((token) => [example, token.split("=")[0]!] as [string, string]), +); + +describe("migrate README", () => { + // Guards the extractor: a regex that silently matched nothing would make + // every check below pass vacuously. + test("finds the documented examples", () => { + expect(EXAMPLES.length).toBeGreaterThan(20); + expect(FLAG_USES.length).toBeGreaterThan(20); + }); + + test.each(EXAMPLES)("`%s` resolves to a real command", (example) => { + const tokens = example.split(/\s+/).slice(1); + const { command, rest } = resolve(tokens); + const firstFlag = rest.findIndex((token) => token.startsWith("-")); + const positionals = firstFlag === -1 ? rest : rest.slice(0, firstFlag); + // Leftover words before any flag are positionals — only some commands take + // them, and a subcommand that does not exist lands here too. + if (positionals.length > 0) expect(command.registeredArguments.length).toBeGreaterThan(0); + // A parent means at least `migrate` resolved. Checking the name instead + // would be wrong: `migrate export clerk` is itself named `clerk`. + expect(command.parent).not.toBeNull(); + }); + + test.each(FLAG_USES)("`%s` uses %s, which the command accepts", (example, flag) => { + const { command } = resolve(example.split(/\s+/).slice(1)); + expect(flagsOf(command)).toContain(flag); + }); + + test.each(migrateTree())("$path documents every flag it accepts", ({ command }) => { + const undocumented = flagsOf(command).filter( + (flag) => flag.startsWith("--") && flag !== "--help" && !README.includes(flag), + ); + expect(undocumented).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts new file mode 100644 index 000000000..ef518506c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -0,0 +1,301 @@ +/** + * The human-mode half of `migrate run`: the wizard fills in missing flags, the + * readiness report renders, and declining the confirmation writes nothing. + * + * Kept in its own file because `mock.module` registrations are process-lifetime, + * and `bun test --parallel` puts several files in each worker — so a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. Human mode itself needs no mock: `setMode` is the + * supported override. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { getMode, setMode, type Mode } from "../../mode.ts"; +import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +const mockSelect = mock(async () => "clerk" as unknown); +const mockText = mock(async () => "export.json" as unknown); +let confirmAnswer = true; +let originalMode: Mode; + +mock.module("../../lib/listage.ts", () => ({ + ...listageStubs, + select: (...args: unknown[]) => mockSelect(...(args as [])), +})); + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../lib/prompts.ts", () => ({ + confirm: async () => confirmAnswer, + multiselect: async () => [], + text: (...args: unknown[]) => mockText(...(args as [])), + password: async () => "", + editor: async () => "{}", +})); + +const { run } = await import("./run.ts"); +const { deleteMigration } = await import("./delete.ts"); +const { UserAbortError } = await import("../../lib/errors.ts"); +const { loadSettings, saveSettings } = await import("./lib/settings.ts"); + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { method: string; url: string; body: unknown }[]; + +const EXPORT = [ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", primary_email_address: "b@x.dev" }, +]; + +const baseOptions = { transformer: "clerk", file: "export.json", secretKey: "sk_test_x" }; + +beforeAll(() => { + originalMode = getMode(); + setMode("human"); + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); + process.chdir(workDir); +}); + +afterAll(() => { + setMode(originalMode); + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + confirmAnswer = true; + mockSelect.mockReset(); + mockText.mockReset(); + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("export.json"); + fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); + stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); +}); + +afterEach(() => { + process.exitCode = 0; +}); + +/** Stubs BAPI plus the FAPI environment lookup the readiness report needs. */ +function stubInstanceSettings(settings: { attributes?: object; social?: object } | null) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + if (url.endsWith("/v1/domains")) { + if (!settings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.includes("/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.includes("/v1/environment")) return Response.json({ user_settings: settings }); + return Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; +} + +const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); + +describe("the wizard fills in missing flags", () => { + test("bare `clerk migrate` prompts for the transformer and file, then imports", async () => { + await run({ secretKey: "sk_test_x" }); + + expect(mockSelect).toHaveBeenCalledTimes(1); + expect(mockText).toHaveBeenCalledTimes(1); + expect(created()).toHaveLength(2); + }); + + test("asks only for what the flags did not supply", async () => { + await run({ ...baseOptions, transformer: "clerk" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(mockText).not.toHaveBeenCalled(); + }); + + test("prompts for the file when only the transformer was passed", async () => { + await run({ transformer: "clerk", secretKey: "sk_test_x" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(mockText).toHaveBeenCalledTimes(1); + }); + + test("records the wizard's answers for the next run", async () => { + await run({ secretKey: "sk_test_x" }); + + expect(loadSettings()).toMatchObject({ key: "clerk", file: "export.json" }); + }); +}); + +describe("the readiness report", () => { + test("renders before the confirmation", async () => { + await run(baseOptions); + expect(captured.err).toContain("Migration readiness"); + expect(captured.err).toContain("2 users ready to import"); + }); + + // The whole point of the report: seeing what will go wrong, then backing out + // before a single user exists in the destination instance. + test("declining afterwards writes nothing to Clerk", async () => { + confirmAnswer = false; + + await expect(run(baseOptions)).rejects.toThrow(UserAbortError); + + expect(captured.err).toContain("Migration readiness"); + expect(created()).toHaveLength(0); + }); + + test("accepting proceeds with the import", async () => { + confirmAnswer = true; + + await run(baseOptions); + + expect(created()).toHaveLength(2); + }); + + test("flags a field Clerk requires that not every user has", async () => { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", username: "bob" }, + ]), + ); + + await run(baseOptions); + + expect(captured.err).toContain("1 user lacks it"); + expect(captured.err).toContain("1 setting needs attention"); + }); + + test("degrades to a note when the instance settings cannot be read", async () => { + stubInstanceSettings(null); + + await run(baseOptions); + + expect(captured.err).toContain("Could not read this instance's settings"); + expect(created()).toHaveLength(2); + }); + + test("is skipped for a -y run, which pays for no extra round-trips", async () => { + await run({ ...baseOptions, yes: true }); + + expect(requests.some((r) => r.url.endsWith("/v1/domains"))).toBe(false); + expect(captured.err).not.toContain("Migration readiness"); + expect(created()).toHaveLength(2); + }); +}); + +describe("guards that still apply interactively", () => { + test("the dev-instance 500-user cap", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify( + Array.from({ length: 501 }, (_, i) => ({ + id: `u${i}`, + primary_email_address: `u${i}@x.dev`, + })), + ), + ); + + await expect(run(baseOptions)).rejects.toThrow(/development instance/); + expect(created()).toHaveLength(0); + }); + + test("an unrecognized password hasher aborts before any request", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + await expect(run(baseOptions)).rejects.toThrow(/Invalid password hasher/); + expect(created()).toHaveLength(0); + }); +}); + +describe("migrate delete confirmation", () => { + /** Answers the external-id lookup, then the deletes. */ + function stubDeleteTargets(present: Record) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ method: init?.method ?? "GET", url, body: null }); + + if (url.includes("/v1/users?")) { + const asked = new URL(url).searchParams.getAll("external_id"); + return Response.json( + asked + .filter((externalId) => externalId in present) + .map((externalId) => ({ id: present[externalId], external_id: externalId })), + ); + } + return Response.json({ deleted: true }); + }) as unknown as typeof fetch; + } + + const deleted = () => requests.filter((r) => r.method === "DELETE"); + + beforeEach(() => { + saveSettings({ key: "clerk", file: "export.json" }); + stubDeleteTargets({ legacy_a: "user_1", legacy_b: "user_2" }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "legacy_a", primary_email_address: "a@x.dev" }, + { id: "legacy_b", primary_email_address: "b@x.dev" }, + ]), + ); + }); + + test("reports the count and confirms before deleting", async () => { + confirmAnswer = true; + + await deleteMigration({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("About to delete 2 users"); + expect(deleted()).toHaveLength(2); + }); + + // The undo for a bad undo does not exist, so declining must cost nothing. + test("declining deletes nobody", async () => { + confirmAnswer = false; + + await expect(deleteMigration({ secretKey: "sk_test_x" })).rejects.toThrow(UserAbortError); + + expect(deleted()).toHaveLength(0); + }); + + test("-y skips the prompt", async () => { + confirmAnswer = false; + + await deleteMigration({ yes: true, secretKey: "sk_test_x" }); + + expect(deleted()).toHaveLength(2); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts new file mode 100644 index 000000000..e5408e422 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -0,0 +1,749 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../lib/errors.ts"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { getLogDir } from "./lib/logger.ts"; +import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; +import { loadSettings } from "./lib/settings.ts"; +import { applyResumeAfter, resolveFirebaseHashConfig, run, validateRunOptions } from "./run.ts"; +import type { FirebaseHashConfig, User } from "./types.ts"; + +let workDir: string; +let originalCwd: string; + +const users = (...ids: string[]): User[] => ids.map((userId) => ({ userId }) as User); + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-"))); + process.chdir(workDir); + fs.writeFileSync(path.join(workDir, "users.json"), "[]"); + fs.writeFileSync(path.join(workDir, "users.txt"), ""); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("validateRunOptions", () => { + test("accepts a transformer and an existing JSON file", () => { + expect(validateRunOptions({ transformer: "clerk", file: "users.json" })).toEqual({ + transformer: "clerk", + file: "users.json", + }); + }); + + test.each([ + ["no transformer", { file: "users.json" }, /--transformer/], + ["an unknown transformer", { transformer: "okta", file: "users.json" }, /Unknown transformer/], + ["no file", { transformer: "clerk" }, /--file/], + ["a missing file", { transformer: "clerk", file: "nope.json" }, /File not found/], + [ + "an unsupported extension", + { transformer: "clerk", file: "users.txt" }, + /Unsupported file type/, + ], + ])("rejects %s", (_label, options, message) => { + expect(() => validateRunOptions(options)).toThrow(message); + }); + + test("names the valid transformers when one is missing", () => { + expect(() => validateRunOptions({ file: "users.json" })).toThrow(/clerk/); + }); +}); + +describe("resolveFirebaseHashConfig", () => { + const ALL = { + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }; + + test("builds the config when all four flags are present", () => { + expect(resolveFirebaseHashConfig(ALL)).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + // A digest built from a partial set is well-formed but verifies against + // nothing, so every migrated user would silently fail to sign in. + test.each([ + ["firebaseSignerKey", "--firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost"], + ] as const)("rejects a set missing %s, naming the flag", (omit, flag) => { + const partial = { ...ALL }; + delete (partial as Record)[omit]; + expect(() => resolveFirebaseHashConfig(partial)).toThrow(new RegExp(flag)); + }); + + test("names every missing flag at once", () => { + expect(() => resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" })).toThrow( + /--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/, + ); + }); + + test("falls back to saved settings when no flag is passed", () => { + const saved: FirebaseHashConfig = { + base64_signer_key: "S", + base64_salt_separator: "B", + rounds: 8, + mem_cost: 14, + }; + expect(resolveFirebaseHashConfig({}, saved)).toEqual(saved); + }); + + test("prefers flags over saved settings", () => { + const saved: FirebaseHashConfig = { + base64_signer_key: "OLD", + base64_salt_separator: "B", + rounds: 1, + mem_cost: 1, + }; + expect(resolveFirebaseHashConfig(ALL, saved)?.base64_signer_key).toBe("SIGNER"); + }); + + test("returns nothing when neither flags nor settings supply a config", () => { + expect(resolveFirebaseHashConfig({})).toBeUndefined(); + }); +}); + +describe("applyResumeAfter", () => { + test("returns everything when no ID is given", () => { + expect(applyResumeAfter(users("a", "b"), undefined)).toHaveLength(2); + }); + + test("skips up to and including the named user", () => { + expect(applyResumeAfter(users("a", "b", "c"), "b").map((u) => u.userId)).toEqual(["c"]); + }); + + test("returns nothing when the named user is last", () => { + expect(applyResumeAfter(users("a", "b"), "b")).toEqual([]); + }); + + test("throws rather than silently re-importing everyone", () => { + expect(() => applyResumeAfter(users("a"), "zz")).toThrow(CliError); + }); +}); + +describe("run", () => { + const captured = useCaptureLog(); + let originalFetch: typeof globalThis.fetch; + let requests: { method: string; url: string; body: unknown }[]; + + const export2 = [ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d1", + password_hasher: "bcrypt", + }, + { id: "u2", primary_email_address: "b@x.dev" }, + ]; + + beforeAll(() => { + originalFetch = globalThis.fetch; + }); + + beforeEach(() => { + requests = []; + delete process.env.CLERK_MIGRATE_RATE_LIMIT; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ + method: init?.method ?? "GET", + url: input.toString(), + body: init?.body ? JSON.parse(init.body as string) : null, + }); + return new Response(JSON.stringify({ id: "user_created" }), { status: 200 }); + }) as typeof fetch; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + process.exitCode = 0; + }); + + const baseOptions = { + transformer: "clerk", + file: "export.json", + yes: true, + secretKey: "sk_test_x", + }; + + test("imports every user in the file end to end", async () => { + await run(baseOptions); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created).toHaveLength(2); + expect(created[0]?.method).toBe("POST"); + expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "u1", + "u2", + ]); + expect(captured.err).toContain("Imported:"); + }); + + test("writes a timestamped NDJSON log for the run", async () => { + await run(baseOptions); + + const logs = fs.readdirSync(getLogDir()); + expect(logs).toHaveLength(1); + expect(logs[0]).toMatch(/^migration-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + + const entries = fs + .readFileSync(path.join(getLogDir(), logs[0] as string), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(entries.filter((e) => e.status === "success")).toHaveLength(2); + }); + + test("records the run's key and file in .settings", async () => { + await run(baseOptions); + expect(loadSettings()).toEqual({ key: "clerk", file: "export.json" }); + }); + + test("--require-password imports only the users that have one", async () => { + await run({ ...baseOptions, requirePassword: true }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u1"]); + expect(captured.err).toContain("skipping 1 user(s) without a password"); + }); + + test("--resume-after skips everyone up to and including that ID", async () => { + await run({ ...baseOptions, resumeAfter: "u1" }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u2"]); + }); + + test("logs validation failures and imports the rest", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify([...export2, { id: "u3" }])); + + await run(baseOptions); + + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + expect(captured.err).toContain("1 user(s) failed validation"); + }); + + test("warns that --clerk-secret-key is deprecated but still honours it", async () => { + await run({ ...baseOptions, secretKey: undefined, clerkSecretKey: "sk_test_x" }); + + expect(captured.err).toContain("--clerk-secret-key is deprecated"); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + }); + + test("refuses to exceed the development-instance user limit", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify( + Array.from({ length: 501 }, (_, i) => ({ + id: `u${i}`, + primary_email_address: `u${i}@x.dev`, + })), + ), + ); + + await expect(run(baseOptions)).rejects.toThrow(/development instance/); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + }); + + test("aborts before any API call when the hasher is unrecognized", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + await expect(run(baseOptions)).rejects.toThrow(/Invalid password hasher/); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + }); + + test("exits non-zero when some users failed", async () => { + globalThis.fetch = (async () => + new Response(JSON.stringify({ errors: [{ code: "e", message: "taken" }] }), { + status: 422, + })) as unknown as typeof fetch; + + await run(baseOptions); + expect(process.exitCode).toBe(1); + }); + + // Tests run non-TTY, so `isHuman()` is false and the wizard path is never + // reached — the same guard an agent hits. + describe("without --transformer or --file", () => { + test.each([ + [{}, /--transformer and --file /], + [{ transformer: "clerk" }, /--file /], + [{ file: "export.json" }, /--transformer /], + ])("names the missing flags rather than prompting (%p)", async (partial, expected) => { + await expect(run({ ...partial, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + expected, + ); + expect(requests).toHaveLength(0); + }); + + test("explains that it cannot prompt", async () => { + await expect(run({ yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /cannot prompt in agent mode/, + ); + }); + }); + + describe("readiness report", () => { + /** Stubs BAPI plus the FAPI environment lookup the report depends on. */ + function stubInstanceSettings( + settings: { attributes?: object; social?: object } | null, + onUsers?: () => Response, + ) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + if (url.endsWith("/v1/domains")) { + if (!settings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.includes("/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.includes("/v1/environment")) return Response.json({ user_settings: settings }); + return onUsers ? onUsers() : Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; + } + + const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); + + // `-y` means nobody is watching, so the two extra round-trips buy nothing. + test("is skipped for a -y run", async () => { + stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); + + await run(baseOptions); + + expect(requests.some((r) => r.url.endsWith("/v1/domains"))).toBe(false); + expect(captured.err).not.toContain("Migration readiness"); + expect(created()).toHaveLength(2); + }); + + test("renders before any user is created, and flags a required-but-missing field", async () => { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }); + // One user has no email, so a required email address will cost them. + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", username: "bob" }, + ]), + ); + + await run({ ...baseOptions, yes: false }); + + expect(captured.err).toContain("Migration readiness"); + expect(captured.err).toContain("1 user lacks it"); + + // The report was printed before the first POST /v1/users. + const reportIndex = requests.findIndex((r) => r.url.includes("/v1/environment")); + const firstCreate = requests.findIndex((r) => r.url.endsWith("/v1/users")); + expect(reportIndex).toBeGreaterThanOrEqual(0); + expect(reportIndex).toBeLessThan(firstCreate); + }); + + test("degrades to a note when the instance settings cannot be read", async () => { + stubInstanceSettings(null); + + await run({ ...baseOptions, yes: false }); + + expect(captured.err).toContain("Could not read this instance's settings"); + expect(created()).toHaveLength(2); + }); + + test("cross-references supabase providers against the instance", async () => { + stubInstanceSettings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["discord"]}', + }, + ]), + ); + + await run({ ...baseOptions, transformer: "supabase", yes: false }); + + expect(captured.err).toContain("Social connections"); + expect(captured.err).toContain("Discord"); + expect(captured.err).toContain("not enabled in Clerk"); + }); + + // Supabase lists `email` and `phone` in `providers` alongside real social + // connections, and Clerk has no `oauth_email` to enable — so counting them + // as social flagged every password user as a blocking problem. + test("leaves supabase's email and phone pseudo-providers out of the social section", async () => { + stubInstanceSettings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email","discord"]}', + }, + ]), + ); + + await run({ ...baseOptions, transformer: "supabase", yes: false }); + + const social = captured.err.slice(captured.err.indexOf("Social connections")); + expect(social).toContain("Discord"); + expect(social).not.toContain("Email"); + expect(social).not.toContain("Phone"); + }); + }); + + describe("--transformer-file", () => { + const CUSTOM = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email", given: "firstName", pw: "password" }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { if (!user.firstName) delete user.firstName; }, + };`; + + let customFile: string; + let customCounter = 0; + + beforeEach(() => { + // A fresh filename each time: dynamic import() caches by URL, so reusing + // one would silently return a previous test's module. + customFile = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, customFile), CUSTOM); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { account_ref: "mp_1", contact_email: "a@x.dev", given: "Ada", pw: "$2b$10$hash" }, + { account_ref: "mp_2", contact_email: "b@x.dev", given: "", pw: "$2b$10$hash" }, + ]), + ); + }); + + afterEach(() => { + __resetCustomTransformersForTesting(); + }); + + const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); + + test("imports through a user-authored transformer", async () => { + await run({ + file: "export.json", + transformerFile: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_1", + "mp_2", + ]); + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("transformer from"); + }); + + test("applies the custom transformer's defaults and postTransform", async () => { + await run({ + file: "export.json", + transformerFile: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ first_name: "Ada", password_hasher: "bcrypt" }); + // postTransform dropped the empty given name rather than sending "". + expect("first_name" in (bodies[1] ?? {})).toBe(false); + }); + + // No sensible precedence between "the one you wrote" and "the one we ship". + test("conflicts with --transformer rather than picking one", async () => { + await expect( + run({ + transformer: "clerk", + file: "export.json", + transformerFile: customFile, + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/both name a transformer. Pass one or the other/); + expect(created()).toHaveLength(0); + }); + + test("fails before any request when the file is not there", async () => { + await expect( + run({ + file: "export.json", + transformerFile: "./nope.ts", + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/No transformer file at/); + expect(requests).toHaveLength(0); + }); + + test("fails before any request when the file is malformed", async () => { + const bad = `./bad-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, bad), + `export default { key: "x", label: "X", transformer: {} };`, + ); + + await expect( + run({ file: "export.json", transformerFile: bad, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/no source field maps to `userId`/); + expect(requests).toHaveLength(0); + }); + + test("still requires --file", async () => { + await expect( + run({ transformerFile: customFile, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/--file/); + }); + }); + + describe("per-platform imports", () => { + /** One realistic record per platform, in that platform's export shape. */ + const PLATFORMS: [string, unknown, string][] = [ + [ + "auth0", + [ + { + user_id: "auth0|1", + email: "a@x.dev", + email_verified: true, + given_name: "Ada", + family_name: "L", + }, + ], + "auth0|1", + ], + ["authjs", [{ id: "aj1", email: "a@x.dev", email_verified: "2024-01-01T00:00:00Z" }], "aj1"], + [ + "betterauth", + [{ user_id: "ba1", email: "a@x.dev", email_verified: true, password_hash: "$2a$10$h" }], + "ba1", + ], + [ + "supabase", + [ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-06-29 20:25:06+00", + encrypted_password: "$2b$10$h", + }, + ], + "sb1", + ], + ]; + + test.each(PLATFORMS)( + "%s transforms, validates and imports its export", + async (key, records, externalId) => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(records)); + + await run({ ...baseOptions, transformer: key }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created).toHaveLength(1); + expect((created[0]?.body as { external_id: string } | undefined)?.external_id).toBe( + externalId, + ); + }, + ); + + test("firebase imports its wrapped export and builds the scrypt digest", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify({ + users: [ + { + localId: "fb1", + email: "a@x.dev", + emailVerified: true, + passwordHash: "SGFzaA==", + salt: "U2FsdA==", + }, + ], + }), + ); + + await run({ + ...baseOptions, + transformer: "firebase", + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }); + + const body = requests.find((r) => r.url.endsWith("/v1/users"))?.body as Record< + string, + unknown + >; + expect(body).toMatchObject({ + external_id: "fb1", + password_digest: "SGFzaA==$U2FsdA==$SIGNER$Bw==$8$14", + password_hasher: "scrypt_firebase", + }); + }); + + test("a partial firebase flag set fails before anything is read", async () => { + await expect( + run({ ...baseOptions, transformer: "firebase", firebaseSignerKey: "SIGNER" }), + ).rejects.toThrow(/--firebase-salt-separator/); + expect(requests).toHaveLength(0); + }); + + test("an unknown transformer fails listing the valid keys", async () => { + await expect(run({ ...baseOptions, transformer: "okta" })).rejects.toThrow( + /Unknown transformer "okta".*clerk.*supabase/s, + ); + }); + }); + + describe("--skip-unsupported-providers", () => { + const supabaseExport = [ + { + id: "sb_email", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email"]}', + }, + { + id: "sb_discord", + email: "b@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["discord"]}', + }, + { + id: "sb_both", + email: "c@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email","discord"]}', + }, + ]; + + /** Stubs BAPI plus the FAPI environment lookup the check depends on. */ + function stubInstance(enabledSocial: Record | null) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + if (url.endsWith("/v1/domains")) { + if (!enabledSocial) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.includes("/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.includes("/v1/environment")) { + return Response.json({ user_settings: { social: enabledSocial } }); + } + return Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; + } + + const created = () => + requests + .filter((r) => r.url.endsWith("/v1/users")) + .map((r) => (r.body as { external_id: string }).external_id); + + beforeEach(() => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(supabaseExport)); + }); + + test("skips only the user whose sole provider is disabled", async () => { + stubInstance({ oauth_google: { enabled: true }, oauth_discord: { enabled: false } }); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(created()).toEqual(["sb_email", "sb_both"]); + expect(captured.err).toContain("skipping 1 user(s)"); + expect(captured.err).toContain("discord: 1"); + }); + + test("imports everyone when the provider is enabled", async () => { + stubInstance({ oauth_discord: { enabled: true } }); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(created()).toHaveLength(3); + }); + + // A failed lookup must not be read as "nothing is enabled" — that would + // silently drop every social user. + test("imports everyone when the instance config cannot be read", async () => { + stubInstance(null); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(created()).toHaveLength(3); + expect(captured.err).toContain("Could not read the instance's enabled providers"); + }); + + test("is a no-op with a warning on a non-supabase transformer", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); + + await run({ ...baseOptions, skipUnsupportedProviders: true }); + + expect(created()).toHaveLength(2); + expect(captured.err).toContain("only applies to supabase"); + }); + + test("records the flag in .settings", async () => { + stubInstance({ oauth_discord: { enabled: true } }); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(loadSettings().skipUnsupportedProviders).toBe(true); + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts new file mode 100644 index 000000000..0d518fb3c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -0,0 +1,519 @@ +/** + * `clerk migrate run` — non-interactive user import. + * + * Ported from the standalone migration-tool's `src/migrate/cli.ts` + * (`runNonInteractive`), with auth moved onto the CLI's standard secret-key + * resolution chain and every failure raised as a `CliError` instead of + * `console.error` + `process.exit`. + * + * The interactive wizard that a bare `clerk migrate` will launch is a separate + * command; this path is the one an agent or a script drives. + */ + +import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; +import { bold, dim, green, red, yellow } from "../../lib/color.ts"; +import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; +import { log } from "../../lib/log.ts"; +import { confirm } from "../../lib/prompts.ts"; +import { withGutter, withSpinner } from "../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../mode.ts"; +import { importUsers } from "./import-users.ts"; +import { analyzeFields } from "./lib/analysis.ts"; +import { + enabledSocialProviders, + fetchInstanceSettings, + toClerkStrategy, +} from "./lib/clerk-config.ts"; +import { buildReadinessReport, formatReadinessReport } from "./lib/readiness.ts"; +import { DEV_USER_LIMIT, resolveLimits } from "./lib/instance.ts"; +import { getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; +import { loadSettings, saveSettings } from "./lib/settings.ts"; +import { + countSocialProviders, + findDisabledProviders, + findUsersWithOnlyDisabledProviders, + readSupabaseRows, +} from "./lib/supabase-providers.ts"; +import { fileExists, getFileType, loadUsersFromFile } from "./lib/transform.ts"; +import { loadCustomTransformer } from "./transformers/load-custom.ts"; +import { registerCustomTransformer, transformerKeys } from "./transformers/registry.ts"; +import type { FirebaseHashConfig, ImportSummary, User } from "./types.ts"; +import { runWizard, throwAgentFlagsRequired } from "./wizard.ts"; + +export type MigrateRunOptions = { + transformer?: string; + file?: string; + resumeAfter?: string; + requirePassword?: boolean; + yes?: boolean; + secretKey?: string; + /** Deprecated alias for `--secret-key`, kept for existing prompts and docs. */ + clerkSecretKey?: string; + app?: string; + instance?: string; + /** Path to a user-authored transformer, for a platform with no built-in. */ + transformerFile?: string; + /** Supabase: drop users whose only social provider is disabled in Clerk. */ + skipUnsupportedProviders?: boolean; + firebaseSignerKey?: string; + firebaseSaltSeparator?: string; + firebaseRounds?: number; + firebaseMemCost?: number; +}; + +const FIREBASE_FLAGS = [ + ["firebaseSignerKey", "--firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost"], +] as const; + +/** + * Resolves Firebase's four hash parameters from flags, falling back to + * `.settings` when none were passed. + * + * The four are required as a set: a digest built from a partial set is + * well-formed but verifies against nothing, so every migrated user would fail + * to sign in with no error at import time. + * + * @returns The config, or `undefined` when none was supplied — which is fine + * for an export that carries no password hashes. + */ +export function resolveFirebaseHashConfig( + options: MigrateRunOptions, + saved?: FirebaseHashConfig, +): FirebaseHashConfig | undefined { + const provided = FIREBASE_FLAGS.filter(([key]) => options[key] !== undefined); + + if (provided.length === 0) return saved; + + if (provided.length < FIREBASE_FLAGS.length) { + const missing = FIREBASE_FLAGS.filter(([key]) => options[key] === undefined).map( + ([, flag]) => flag, + ); + throwUsageError( + `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + + "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + "https://clerk.com/docs/guides/development/migrating/firebase", + ); + } + + return { + base64_signer_key: options.firebaseSignerKey as string, + base64_salt_separator: options.firebaseSaltSeparator as string, + rounds: options.firebaseRounds as number, + mem_cost: options.firebaseMemCost as number, + }; +} + +/** + * Validates the flags a run needs before anything is read or sent. + * + * @returns The transformer key and file path, both guaranteed present. + */ +export function validateRunOptions(options: MigrateRunOptions): { + transformer: string; + file: string; +} { + const valid = transformerKeys(); + + // A custom transformer has already been loaded and registered by the time + // this runs, so its key is resolvable even though it is not in `valid`. + if (options.transformerFile) { + if (!options.file) { + throwUsageError( + "Missing required option --file (path to a JSON or CSV export).", + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: + "clerk migrate run -y --transformer-file ./my-transformer.ts --file users.json", + description: "Import with a custom transformer", + }, + ], + ); + } + if (!fileExists(options.file)) { + throw new CliError(`File not found: ${options.file}`, { code: ERROR_CODE.FILE_NOT_FOUND }); + } + if (!getFileType(options.file)) { + throwUsageError(`Unsupported file type for ${options.file}. Provide a .json or .csv file.`); + } + return { transformer: options.transformer as string, file: options.file }; + } + + if (!options.transformer) { + throwUsageError( + `Missing required option --transformer. Valid values: ${valid.join(", ")}.`, + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: "clerk migrate run -y --transformer clerk --file users.json", + description: "Import a Clerk export", + }, + ], + ); + } + if (!valid.includes(options.transformer)) { + throwUsageError( + `Unknown transformer "${options.transformer}". Valid values: ${valid.join(", ")}.`, + ); + } + if (!options.file) { + throwUsageError( + "Missing required option --file (path to a JSON or CSV export).", + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: "clerk migrate run -y --transformer clerk --file users.json", + description: "Import a Clerk export", + }, + ], + ); + } + if (!fileExists(options.file)) { + throw new CliError(`File not found: ${options.file}`, { code: ERROR_CODE.FILE_NOT_FOUND }); + } + if (!getFileType(options.file)) { + throwUsageError(`Unsupported file type for ${options.file}. Provide a .json or .csv file.`); + } + + return { transformer: options.transformer, file: options.file }; +} + +/** + * Drops every user up to and including `resumeAfter`. + * + * @throws CliError when the ID is not in the file — silently importing the + * whole set would duplicate everything the previous run already created. + */ +export function applyResumeAfter(users: User[], resumeAfter: string | undefined): User[] { + if (!resumeAfter) return users; + + const index = users.findIndex((user) => user.userId === resumeAfter); + if (index === -1) { + throw new CliError(`Could not find user ID "${resumeAfter}" in the import file.`, { + code: ERROR_CODE.USAGE_ERROR, + }); + } + return users.slice(index + 1); +} + +function formatSummary(summary: ImportSummary, logFile: string): string { + const inFile = summary.totalProcessed + summary.validationFailed; + const lines = [ + `${bold("Total users in file:")} ${inFile}`, + `${green("Imported:")} ${summary.successful}`, + `${red("Failed:")} ${summary.failed}`, + ]; + + if (summary.validationFailed > 0) { + lines.push(`${yellow("Failed validation:")} ${summary.validationFailed}`); + } + if (summary.errorBreakdown.size > 0) { + lines.push("", bold("Error breakdown:")); + for (const [error, count] of summary.errorBreakdown) { + lines.push(` ${count} user${count === 1 ? "" : "s"}: ${error}`); + } + } + lines.push("", dim(`Log: ${logFile}`)); + + return lines.join("\n"); +} + +/** + * Drops users whose only way into Clerk is a social provider the destination + * instance has not enabled. + * + * Only meaningful for Supabase exports — it is the one platform whose export + * records per-user providers. If the instance's configuration cannot be read, + * nobody is dropped: a failed lookup must not be mistaken for "no providers + * are enabled". + */ +async function skipDisabledProviderUsers( + users: User[], + file: string, + transformer: string, + secretKey: string, +): Promise { + if (transformer !== "supabase") { + log.warn(`--skip-unsupported-providers only applies to supabase exports; ignoring.`); + return users; + } + + const settings = await withSpinner("Checking enabled providers", () => + fetchInstanceSettings(secretKey), + ); + const enabled = settings ? enabledSocialProviders(settings) : null; + if (!enabled) { + log.warn( + "Could not read the instance's enabled providers; importing every user. Re-run with --verbose for details.", + ); + return users; + } + + const rows = await readSupabaseRows(file); + const disabled = findDisabledProviders(rows, enabled, toClerkStrategy); + if (disabled.length === 0) { + log.info("Every provider in this export is enabled in Clerk; no users skipped."); + return users; + } + + const { excludedIds, byProvider } = findUsersWithOnlyDisabledProviders(rows, disabled); + if (excludedIds.size === 0) { + log.info( + `${disabled.join(", ")} not enabled in Clerk, but every user has another way to sign in; none skipped.`, + ); + return users; + } + + const breakdown = Object.entries(byProvider) + .map(([provider, count]) => `${provider}: ${count}`) + .join(", "); + log.warn( + `--skip-unsupported-providers: skipping ${excludedIds.size} user(s) whose only provider is not enabled in Clerk (${breakdown}).`, + ); + + return users.filter((user) => !excludedIds.has(user.userId)); +} + +/** + * Prints the Migration Readiness report: what the file contains, cross- + * referenced against what the destination instance accepts. + * + * Rendered immediately before the confirmation prompt, so declining that + * prompt aborts with nothing written to Clerk. + * + * Skipped only for `-y`, which says "don't ask, don't lecture" and should not + * pay for two extra network round-trips. Agent mode still gets it: an agent + * driving a migration can act on "this field is required and 40 users lack it" + * exactly as a human would. + */ +async function showReadinessReport(input: { + users: User[]; + file: string; + transformer: string; + secretKey: string; + validationFailed: number; + skipReport: boolean; +}): Promise { + if (input.skipReport) return; + + const settings = await withSpinner("Checking instance settings", () => + fetchInstanceSettings(input.secretKey), + ); + + // Only Supabase exports record per-user providers, so only they can be + // cross-referenced against the instance's social connections. + let providerCounts: Record | undefined; + if (input.transformer === "supabase") { + try { + providerCounts = countSocialProviders(await readSupabaseRows(input.file)); + } catch (error) { + log.debug(`migrate: could not read providers for the readiness report: ${String(error)}`); + } + } + + const report = buildReadinessReport({ + analysis: analyzeFields(input.users), + settings, + validationFailed: input.validationFailed, + providerCounts, + }); + + log.blank(); + for (const line of formatReadinessReport(report)) log.info(line); + log.blank(); +} + +/** + * Fills in a missing `--transformer`/`--file` interactively, or explains what + * to pass. + * + * Agent mode is the CLI's existing non-interactive signal, so an agent that + * runs bare `clerk migrate` gets a usage error naming the flags rather than a + * prompt it cannot answer. + */ +async function resolveMissingOptions(options: MigrateRunOptions): Promise { + const missing = { transformer: !options.transformer, file: !options.file }; + if (!missing.transformer && !missing.file) return options; + + if (isAgent() || !isHuman()) { + throwAgentFlagsRequired(missing); + } + + // A partial Firebase flag set is a usage error whether or not the wizard is + // filling in the rest, so it is checked before any prompt. + const firebaseHashConfig = resolveFirebaseHashConfig(options); + const answers = await runWizard({ + transformer: options.transformer, + file: options.file, + firebaseHashConfig, + }); + + return { + ...options, + transformer: answers.transformer, + file: answers.file, + ...(answers.firebaseHashConfig + ? { + firebaseSignerKey: answers.firebaseHashConfig.base64_signer_key, + firebaseSaltSeparator: answers.firebaseHashConfig.base64_salt_separator, + firebaseRounds: answers.firebaseHashConfig.rounds, + firebaseMemCost: answers.firebaseHashConfig.mem_cost, + } + : {}), + }; +} + +/** + * Loads and registers a `--transformer-file`, so the rest of the run treats it + * exactly like a built-in. + * + * @returns The options with `transformer` set to the loaded entry's key. + */ +async function applyCustomTransformer(options: MigrateRunOptions): Promise { + if (!options.transformerFile) return options; + + // Both name a transformer, and there is no sensible precedence between "the + // one you wrote" and "the one we ship" — say so rather than picking. + if (options.transformer) { + throwUsageError( + "--transformer and --transformer-file both name a transformer. Pass one or the other.", + undefined, + undefined, + [ + { + command: "clerk migrate run -y --transformer-file ./my-transformer.ts --file users.json", + description: "Use a transformer you wrote", + }, + { + command: "clerk migrate run -y --transformer clerk --file users.json", + description: "Use a built-in transformer", + }, + ], + ); + } + + const custom = await loadCustomTransformer(options.transformerFile); + registerCustomTransformer(custom); + log.info(`Loaded the \`${custom.key}\` transformer from ${options.transformerFile}.`); + + return { ...options, transformer: custom.key }; +} + +export async function run(rawOptions: MigrateRunOptions): Promise { + if (rawOptions.clerkSecretKey) { + log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); + } + + rawOptions = await applyCustomTransformer(rawOptions); + const options = await resolveMissingOptions(rawOptions); + const secretKeyOption = options.secretKey ?? options.clerkSecretKey; + + const { transformer, file } = validateRunOptions(options); + const saved = loadSettings(); + const firebaseHashConfig = resolveFirebaseHashConfig(options, saved.firebaseHashConfig); + + await withGutter("Migrating users to Clerk", async () => { + const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const limits = resolveLimits(secretKey); + const dateTime = getDateTimeStamp(); + const logFile = getLogFilePath("migration", dateTime); + + const { users: loaded, validationFailed } = await withSpinner( + `Loading users from ${file}`, + () => loadUsersFromFile(file, transformer, dateTime, { context: { firebaseHashConfig } }), + "Users loaded", + ); + + let users = applyResumeAfter(loaded, options.resumeAfter); + if (options.resumeAfter) { + log.info(`Resuming after ${options.resumeAfter} (${loaded.length - users.length} skipped).`); + } + + if (options.skipUnsupportedProviders) { + users = await skipDisabledProviderUsers(users, file, transformer, secretKey); + } + + if (options.requirePassword) { + const withPassword = users.filter((user) => Boolean(user.password)); + const dropped = users.length - withPassword.length; + if (dropped > 0) { + log.info(`--require-password: skipping ${dropped} user(s) without a password.`); + } + users = withPassword; + } + + if (validationFailed > 0) { + log.warn( + `${validationFailed} user(s) failed validation and will be skipped. See ${logFile}.`, + ); + } + + if (users.length === 0) { + log.warn("No users left to import."); + return; + } + + if (limits.instanceType === "dev" && users.length > DEV_USER_LIMIT) { + throw new CliError( + `Cannot import ${users.length} users into a development instance — the limit is ${DEV_USER_LIMIT}.\n` + + "Target a production instance, or reduce the import file.", + { code: ERROR_CODE.USAGE_ERROR }, + ); + } + + log.info( + `Importing ${users.length} user(s) via the ${transformer} transformer into ` + + `${target ?? "the resolved instance"} (${limits.instanceType}).`, + ); + + await showReadinessReport({ + users, + file, + transformer, + secretKey, + validationFailed, + skipReport: Boolean(options.yes), + }); + + if (!options.yes && isHuman() && !isAgent()) { + const proceed = await confirm({ + message: `Import ${users.length} user(s)?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + saveSettings({ + key: transformer, + file, + ...(options.skipUnsupportedProviders ? { skipUnsupportedProviders: true } : {}), + ...(firebaseHashConfig ? { firebaseHashConfig } : {}), + }); + + const summary = await withSpinner( + `Importing users: [0/${users.length}]`, + (spinner) => + importUsers({ + users, + secretKey, + limits, + dateTime, + skipPasswordRequirement: !options.requirePassword, + validationFailed, + spinner, + }), + "Import complete", + ); + + log.raw(formatSummary(summary, logFile)); + + if (summary.failed > 0) process.exitCode = 1; + }); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/auth0.ts b/packages/cli-core/src/commands/migrate/transformers/auth0.ts new file mode 100644 index 000000000..c1595d139 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/auth0.ts @@ -0,0 +1,43 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification } from "./shared.ts"; + +/** + * Auth0 → Clerk transformer. + * + * Works with Auth0's Export Users API. `user_id` is a `provider|id` string + * (`auth0|abc123`, `github|12345`) and is carried through as the Clerk user's + * `external_id`. + * + * Auth0 does not include password hashes in a standard export — they have to + * be requested from Auth0 support. When present they are bcrypt (`$2a$`/`$2b$`, + * 10 rounds), which is why `passwordHasher` defaults to `bcrypt`. + */ +const auth0Transformer = { + key: "auth0", + label: "Auth0", + description: + "Works with Auth0's Export Users API. Password hashes require a support request to Auth0.", + transformer: { + user_id: "userId", + email: "email", + email_verified: "emailVerified", + username: "username", + given_name: "firstName", + family_name: "lastName", + phone_number: "phone", + phone_verified: "phoneVerified", + passwordHash: "password", + user_metadata: "publicMetadata", + app_metadata: "privateMetadata", + created_at: "createdAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "boolean"); + routeByVerification(user, "phone", "phoneVerified", "boolean"); + }, + defaults: { + passwordHasher: "bcrypt" as const, + }, +} satisfies TransformerRegistryEntry; + +export default auth0Transformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/authjs.ts b/packages/cli-core/src/commands/migrate/transformers/authjs.ts new file mode 100644 index 000000000..3391b1804 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/authjs.ts @@ -0,0 +1,38 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, splitName } from "./shared.ts"; + +/** + * Auth.js (formerly NextAuth) → Clerk transformer. + * + * Auth.js has no export tool and no fixed user table, so this assumes the + * common shape: `SELECT id, name, email, email_verified, created_at FROM users`. + * A different schema means editing the mapping below or supplying a custom + * transformer file. + * + * `email_verified` is a nullable timestamp rather than a boolean — any value + * means verified. + * + * No password default: Auth.js's core is passwordless (OAuth and email links), + * so users arrive without a digest and are imported with + * `skip_password_requirement`. + */ +const authjsTransformer = { + key: "authjs", + label: "Auth.js (NextAuth)", + description: + "Assumes an export of `SELECT id, name, email, email_verified, created_at FROM users`. `name` is split into firstName and lastName.", + transformer: { + id: "userId", + email: "email", + email_verified: "emailVerified", + name: "name", + created_at: "createdAt", + updated_at: "updatedAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "timestamp"); + splitName(user); + }, +} satisfies TransformerRegistryEntry; + +export default authjsTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/betterauth.ts b/packages/cli-core/src/commands/migrate/transformers/betterauth.ts new file mode 100644 index 000000000..f32d2f31c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/betterauth.ts @@ -0,0 +1,46 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, splitName } from "./shared.ts"; + +/** + * Better Auth → Clerk transformer. + * + * Works with `clerk migrate export betterauth`, which joins the user table + * with the credential account row to pick up the bcrypt `password_hash`. + * + * Better Auth plugins add columns Clerk has no equivalent for + * (`display_username`, `role`, `ban_reason`, `two_factor_enabled`). They need + * no handling: the schema strips anything it does not declare. `banned` is the + * exception, because that one *is* a Clerk field. + */ +const betterAuthTransformer = { + key: "betterauth", + label: "Better Auth", + description: + "Works with the Better Auth export. Supports bcrypt passwords and the admin plugin's banned flag.", + transformer: { + user_id: "userId", + email: "email", + email_verified: "emailVerified", + name: "name", + password_hash: "password", + username: "username", + phone_number: "phone", + phone_number_verified: "phoneVerified", + created_at: "createdAt", + updated_at: "updatedAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "boolean"); + routeByVerification(user, "phone", "phoneVerified", "boolean"); + splitName(user); + + // Only carry `banned` when it is actually true — Better Auth writes false + // for every user that was never banned, and sending that to Clerk is noise. + if (user.banned !== true) delete user.banned; + }, + defaults: { + passwordHasher: "bcrypt" as const, + }, +} satisfies TransformerRegistryEntry; + +export default betterAuthTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/clerk.ts b/packages/cli-core/src/commands/migrate/transformers/clerk.ts new file mode 100644 index 000000000..8fb094839 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/clerk.ts @@ -0,0 +1,45 @@ +import type { TransformerRegistryEntry } from "../types.ts"; + +/** + * Clerk → Clerk transformer, for moving users between Clerk instances + * (typically development → production). + * + * Maps the Dashboard's user export format onto the import schema. + */ +const clerkTransformer = { + key: "clerk", + label: "Clerk", + description: + "Migrate between Clerk instances (e.g. development to production, or to another Clerk application). Export your users from the Clerk Dashboard first.", + transformer: { + id: "userId", + primary_email_address: "email", + verified_email_addresses: "emailAddresses", + unverified_email_addresses: "unverifiedEmailAddresses", + first_name: "firstName", + last_name: "lastName", + password_digest: "password", + password_hasher: "passwordHasher", + primary_phone_number: "phone", + verified_phone_numbers: "phoneNumbers", + unverified_phone_numbers: "unverifiedPhoneNumbers", + username: "username", + totp_secret: "totpSecret", + backup_codes_enabled: "backupCodesEnabled", + backup_codes: "backupCodes", + public_metadata: "publicMetadata", + unsafe_metadata: "unsafeMetadata", + private_metadata: "privateMetadata", + // Account state a Dashboard export carries and `POST /v1/users` accepts. + // Unmapped, these survive the export and are then silently stripped at + // validation — losing original signup dates on a dev → prod migration. + created_at: "createdAt", + legal_accepted_at: "legalAcceptedAt", + banned: "banned", + create_organization_enabled: "createOrganizationEnabled", + create_organizations_limit: "createOrganizationsLimit", + delete_self_enabled: "deleteSelfEnabled", + }, +} satisfies TransformerRegistryEntry; + +export default clerkTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/firebase.ts b/packages/cli-core/src/commands/migrate/transformers/firebase.ts new file mode 100644 index 000000000..73966c1b2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/firebase.ts @@ -0,0 +1,121 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import type { PreTransformResult, TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, splitName, toIsoDate } from "./shared.ts"; + +/** + * Column order of `firebase auth:export --format=csv`, which writes no header + * row. Without these the CSV parser would treat the first user as the header. + */ +const FIREBASE_CSV_HEADERS = + "localId,email,emailVerified,passwordHash,passwordSalt,displayName,photoUrl," + + "googleId,googleEmail,googleDisplayName,googlePhotoUrl," + + "facebookId,facebookEmail,facebookDisplayName,facebookPhotoUrl," + + "twitterId,twitterEmail,twitterDisplayName,twitterPhotoUrl," + + "githubId,githubEmail,githubDisplayName,githubPhotoUrl," + + "createdAt,lastSignedInAt,phoneNumber,disabled,customAttributes,providerUserInfo"; + +/** + * Firebase → Clerk transformer. + * + * Handles both shapes `firebase auth:export` produces: a headerless CSV, and + * JSON wrapped in `{ users: [...] }`. + * + * Firebase's scrypt is a modified variant, so Clerk needs the project's four + * hash parameters alongside each digest. They arrive on the run's + * {@link TransformContext} from `--firebase-*` flags or saved `.settings`. + * + * See https://clerk.com/docs/guides/development/migrating/firebase + */ +const firebaseTransformer = { + key: "firebase", + label: "Firebase", + description: + "Works with `firebase auth:export` (CSV or JSON). Requires the project's four password hash parameters to migrate passwords.", + + preTransform: (filePath: string, fileType: string): PreTransformResult => { + if (fileType === "text/csv") { + // Written to the OS temp dir rather than the user's cwd: this is a + // parsing artifact, not a migration output like ./logs. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-firebase-")); + const withHeaders = path.join(tmpDir, path.basename(filePath)); + fs.writeFileSync( + withHeaders, + `${FIREBASE_CSV_HEADERS}\n${fs.readFileSync(filePath, "utf-8")}`, + ); + return { filePath: withHeaders }; + } + + if (fileType === "application/json") { + const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + if (Array.isArray(parsed)) return { filePath, data: parsed as Record[] }; + + const users = (parsed as { users?: unknown })?.users; + if (Array.isArray(users)) return { filePath, data: users as Record[] }; + + throw new CliError( + "Invalid Firebase JSON export: expected `{ users: [...] }` or an array of users.", + { code: ERROR_CODE.INVALID_JSON }, + ); + } + + return { filePath }; + }, + + transformer: { + localId: "userId", + email: "email", + emailVerified: "emailVerified", + passwordHash: "passwordHash", + passwordSalt: "salt", + phoneNumber: "phone", + displayName: "name", + }, + + postTransform: (user, context) => { + const passwordHash = user.passwordHash; + const salt = user.salt; + + if (passwordHash && salt) { + const config = context.firebaseHashConfig; + if (!config) { + throw new CliError( + "This export contains Firebase password hashes, which need the project's hash parameters to import.\n" + + "Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters, then pass:\n" + + " --firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost", + { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: "https://clerk.com/docs/guides/development/migrating/firebase", + }, + ); + } + + // Clerk's scrypt_firebase hasher expects every parameter inline: + // hash$salt$signerKey$saltSeparator$rounds$memCost + user.password = [ + passwordHash, + salt, + config.base64_signer_key, + config.base64_salt_separator, + config.rounds, + config.mem_cost, + ].join("$"); + + delete user.passwordHash; + delete user.salt; + } + + routeByVerification(user, "email", "emailVerified", "boolean"); + // Firebase exports timestamps as Unix milliseconds, often as strings. + user.createdAt = toIsoDate(user.createdAt, true); + splitName(user); + }, + + defaults: { + passwordHasher: "scrypt_firebase" as const, + }, +} satisfies TransformerRegistryEntry; + +export default firebaseTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/list.test.ts b/packages/cli-core/src/commands/migrate/transformers/list.test.ts new file mode 100644 index 000000000..9f432e1d9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/list.test.ts @@ -0,0 +1,116 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { list } from "./list.ts"; +import { transformers } from "./registry.ts"; + +const captured = useCaptureLog(); + +// eslint-disable-next-line no-control-regex +const stripAnsi = (value: string) => value.replace(/\[[0-9;]*m/g, ""); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-tlist-"))); + process.chdir(workDir); + fs.writeFileSync( + path.join(workDir, "custom.ts"), + `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId" }, + };`, + ); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("human output", () => { + test.each([...transformers])("lists the $key transformer with its label", async (transformer) => { + await list(); + expect(captured.err).toContain(transformer.key); + expect(captured.err).toContain(transformer.label); + }); + + // `log.info` auto-highlights backticked spans, so the rendered description + // carries colour codes the source string does not. + test.each([...transformers])("includes the $key description", async (transformer) => { + await list(); + expect(stripAnsi(captured.err)).toContain(transformer.description); + }); + + test("counts the built-ins", async () => { + await list(); + expect(captured.err).toContain(`${transformers.length} built-in transformers`); + }); + + // A compiled binary has no source tree to grep, so the way to extend it has + // to be discoverable from the list itself. + test("says how to add one when none is loaded", async () => { + await list(); + expect(captured.err).toContain("--transformer-file"); + }); + + test("appends a custom transformer and names its source", async () => { + await list({ transformerFile: "./custom.ts" }); + + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("custom — ./custom.ts"); + expect(captured.err).toContain("plus 1 loaded from --transformer-file"); + }); + + test("drops the how-to hint once one is loaded", async () => { + await list({ transformerFile: "./custom.ts" }); + expect(captured.err).not.toContain("Migrating from something else?"); + }); +}); + +describe("--json", () => { + test("emits every built-in on stdout", async () => { + await list({ json: true }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed).toHaveLength(transformers.length); + expect(parsed.map((entry) => entry.key)).toEqual(transformers.map((entry) => entry.key)); + }); + + test("reports key, label, description and the userId source field", async () => { + await list({ json: true }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed[0]).toMatchObject({ + key: "clerk", + label: "Clerk", + built_in: true, + maps_to_user_id: "id", + }); + }); + + test("marks a custom transformer as not built in", async () => { + await list({ json: true, transformerFile: "./custom.ts" }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed.at(-1)).toMatchObject({ + key: "myplatform", + built_in: false, + source: "./custom.ts", + maps_to_user_id: "account_ref", + }); + }); +}); + +describe("a bad --transformer-file", () => { + test("fails rather than listing only the built-ins", async () => { + await expect(list({ transformerFile: "./nope.ts" })).rejects.toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/list.ts b/packages/cli-core/src/commands/migrate/transformers/list.ts new file mode 100644 index 000000000..349a53a8f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/list.ts @@ -0,0 +1,67 @@ +/** + * `clerk migrate transformers list` — which source platforms are available. + * + * New in the CLI. The standalone tool's interactive picker was the only place + * these were listed, which was fine when the user had the source tree to grep. + * A compiled binary's users have neither, so the list is a command. + */ + +import { bold, cyan, dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import type { TransformerRegistryEntry } from "../types.ts"; +import { loadCustomTransformer } from "./load-custom.ts"; +import { transformers } from "./registry.ts"; + +export type TransformersListOptions = { + json?: boolean; + transformerFile?: string; +}; + +type Listed = TransformerRegistryEntry & { builtIn: boolean; source?: string }; + +function toJson(entries: Listed[]) { + return entries.map((entry) => ({ + key: entry.key, + label: entry.label, + description: entry.description, + built_in: entry.builtIn, + ...(entry.source ? { source: entry.source } : {}), + maps_to_user_id: + Object.entries(entry.transformer).find(([, target]) => target === "userId")?.[0] ?? null, + })); +} + +export async function list(options: TransformersListOptions = {}): Promise { + const entries: Listed[] = transformers.map((entry) => ({ ...entry, builtIn: true })); + + if (options.transformerFile) { + const custom = await loadCustomTransformer(options.transformerFile); + entries.push({ ...custom, builtIn: false, source: options.transformerFile }); + } + + if (options.json) { + log.data(JSON.stringify(toJson(entries), null, 2)); + return; + } + + for (const entry of entries) { + const suffix = entry.builtIn ? "" : ` ${dim(`(custom — ${entry.source})`)}`; + log.info(`${cyan(bold(entry.key))} ${entry.label}${suffix}`); + log.info(` ${dim(entry.description)}`); + log.info(""); + } + + const custom = entries.length - transformers.length; + log.info( + dim( + `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + + (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), + ), + ); + + if (custom === 0) { + log.info( + dim("Migrating from something else? Write a transformer and pass --transformer-file."), + ); + } +} diff --git a/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts b/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts new file mode 100644 index 000000000..5dff6ad0b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts @@ -0,0 +1,222 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { loadCustomTransformer, validateTransformer } from "./load-custom.ts"; +import { __resetCustomTransformersForTesting } from "./registry.ts"; + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-custom-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +afterEach(() => { + __resetCustomTransformersForTesting(); +}); + +/** + * Writes a transformer file with a unique name. + * + * Names must not repeat: a dynamic `import()` caches by URL, so reusing one + * would silently return the previous test's module. + */ +function writeTransformer(source: string, ext = "ts"): string { + const name = `custom-${counter++}.${ext}`; + fs.writeFileSync(path.join(workDir, name), source); + return `./${name}`; +} + +const VALID = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email" }, +};`; + +describe("loadCustomTransformer", () => { + test("loads a user-authored TypeScript transformer", async () => { + const entry = await loadCustomTransformer(writeTransformer(VALID)); + + expect(entry).toMatchObject({ + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId", contact_email: "email" }, + }); + }); + + test("loads plain JavaScript too", async () => { + const entry = await loadCustomTransformer(writeTransformer(VALID, "js")); + expect(entry.key).toBe("myplatform"); + }); + + // The file is the user's own code; the CLI must transpile whatever they wrote. + test("transpiles TypeScript syntax the runtime has to strip", async () => { + const entry = await loadCustomTransformer( + writeTransformer(` + interface Entry { key: string; label: string; transformer: Record } + const mapping = { my_id: "userId" } as const; + const custom: Entry = { key: "tsplatform", label: "TS", transformer: { ...mapping } }; + export default custom satisfies Entry; + `), + ); + expect(entry.key).toBe("tsplatform"); + }); + + test("carries the optional hooks through", async () => { + const entry = await loadCustomTransformer( + writeTransformer(`export default { + key: "hooked", label: "Hooked", + transformer: { id: "userId" }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { user.firstName = "set"; }, + };`), + ); + + expect(entry.defaults).toEqual({ passwordHasher: "bcrypt" }); + const user: Record = {}; + entry.postTransform?.(user, {}); + expect(user.firstName).toBe("set"); + }); + + test("supplies a description when the author omitted one", async () => { + const entry = await loadCustomTransformer( + writeTransformer( + `export default { key: "bare", label: "Bare", transformer: { id: "userId" } };`, + ), + ); + expect(entry.description).toBe("Custom transformer"); + }); + + test("reports a path that is not there", async () => { + await expect(loadCustomTransformer("./nope.ts")).rejects.toThrow(/No transformer file at/); + }); + + test("reports a directory given instead of a file", async () => { + fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); + await expect(loadCustomTransformer("./adir")).rejects.toThrow(/is a directory/); + }); + + test("reports a file that does not parse, quoting the syntax error", async () => { + await expect( + loadCustomTransformer(writeTransformer("export default { key: ,,, }")), + ).rejects.toThrow(/Could not load/); + }); + + test("reports a file that throws while loading", async () => { + await expect( + loadCustomTransformer(writeTransformer(`throw new Error("boom"); export default {};`)), + ).rejects.toThrow(/Could not load .*boom/s); + }); + + test("points at a named export when the default is missing", async () => { + const file = writeTransformer( + `export const myPlatform = { key: "x", label: "X", transformer: { a: "userId" } };`, + ); + + await expect(loadCustomTransformer(file)).rejects.toThrow( + /has no default export.*`myPlatform`.*did you mean `export default`/s, + ); + }); + + test("reports a missing default with no named exports to suggest", async () => { + await expect(loadCustomTransformer(writeTransformer("const unused = 1;"))).rejects.toThrow( + /has no default export\.$/m, + ); + }); +}); + +describe("validateTransformer", () => { + const valid = { + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId" }, + }; + + test("accepts a minimal valid entry", () => { + expect(validateTransformer(valid, "f.ts").key).toBe("myplatform"); + }); + + test.each([ + ["a null default export", null, /is null, not an object/], + ["a number default export", 42, /is number, not an object/], + ["a string default export", "nope", /is string, not an object/], + ])("rejects %s", (_label, value, expected) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(expected); + }); + + test.each([ + ["key", { ...valid, key: undefined }], + ["key", { ...valid, key: "" }], + ["key", { ...valid, key: " " }], + ["key", { ...valid, key: 7 }], + ["label", { ...valid, label: undefined }], + ["label", { ...valid, label: "" }], + ])("rejects a bad %s naming the field", (field, value) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test("rejects a non-string description", () => { + expect(() => validateTransformer({ ...valid, description: 7 }, "f.ts")).toThrow( + /`description` must be a string/, + ); + }); + + test.each([ + ["missing", { ...valid, transformer: undefined }], + ["null", { ...valid, transformer: null }], + ["an array", { ...valid, transformer: [] }], + ["a string", { ...valid, transformer: "id" }], + ])("rejects a transformer mapping that is %s", (_label, value) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(/`transformer`|`transformer\./); + }); + + test("names the offending entry when a mapping target is not a field name", () => { + expect(() => + validateTransformer({ ...valid, transformer: { account_ref: "userId", bad: 7 } }, "f.ts"), + ).toThrow(/`transformer.bad` must map to a Clerk field name, got number/); + }); + + // Without it the import runs to completion and creates every user with no + // external_id — which is what makes a migration reversible. + test("rejects a mapping with no userId target", () => { + expect(() => validateTransformer({ ...valid, transformer: { a: "email" } }, "f.ts")).toThrow( + /no source field maps to `userId`/, + ); + }); + + test.each([ + ["defaults", { ...valid, defaults: "nope" }], + ["preTransform", { ...valid, preTransform: "nope" }], + ["postTransform", { ...valid, postTransform: 7 }], + ])("rejects a %s of the wrong type", (field, value) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test.each([["clerk"], ["auth0"], ["supabase"]])( + "rejects %s, which would shadow a built-in", + (key) => { + expect(() => validateTransformer({ ...valid, key }, "f.ts")).toThrow( + /already a built-in transformer/, + ); + }, + ); + + test("names the file in every message, so the author knows which one", () => { + expect(() => validateTransformer({}, "./their-file.ts")).toThrow(/\.\/their-file\.ts/); + }); + + test("raises CliError, so the global handler formats it", () => { + expect(() => validateTransformer({}, "f.ts")).toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/load-custom.ts b/packages/cli-core/src/commands/migrate/transformers/load-custom.ts new file mode 100644 index 000000000..1ac82a719 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/load-custom.ts @@ -0,0 +1,164 @@ +/** + * Loading a user-authored transformer at runtime. + * + * In the standalone migration-tool, supporting a new platform meant adding a + * file to `src/transformers/` and one line to the registry — the user had the + * source tree. A compiled binary has neither a source tree to edit nor a way + * for an end user to rebuild it, so `--transformer-file` restores that + * extensibility by importing a file from the user's own project instead. + * + * **Verified before this was built on:** a `bun build --compile` executable can + * `import()` an arbitrary external `.ts` file at runtime, including TypeScript + * that needs transpiling. Bun's transpiler is part of the runtime, not only the + * bundler. Confirmed with a throwaway compiled binary on darwin-arm64, + * linux-arm64 (glibc), linux-arm64-musl and linux-x64. + * + * The file is user-supplied code the CLI executes, so its shape is validated + * up front and rejected with a specific message rather than crashing deep in + * the transform pipeline on a missing field. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import type { TransformerRegistryEntry } from "../types.ts"; +import { transformers } from "./registry.ts"; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; + +function invalid(problem: string, file: string): never { + throw new CliError(`${file} is not a valid transformer: ${problem}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); +} + +/** + * Checks a loaded value against the registry entry shape. + * + * Every failure names the specific field and what was wrong with it — the + * author is writing this file by hand against a shape they cannot see. + * + * @param file - Path as the user typed it, for the error message. + */ +export function validateTransformer(value: unknown, file: string): TransformerRegistryEntry { + if (value === null || typeof value !== "object") { + invalid(`the default export is ${value === null ? "null" : typeof value}, not an object`, file); + } + + const entry = value as Record; + + for (const field of ["key", "label"] as const) { + if (typeof entry[field] !== "string" || entry[field].trim().length === 0) { + invalid(`\`${field}\` must be a non-empty string`, file); + } + } + + if (entry.description !== undefined && typeof entry.description !== "string") { + invalid("`description` must be a string when present", file); + } + + // Arrays are objects, and an author who wrote `transformer: []` should hear + // that rather than the downstream "no field maps to userId". + if ( + entry.transformer === null || + typeof entry.transformer !== "object" || + Array.isArray(entry.transformer) + ) { + invalid("`transformer` must be an object mapping source fields to Clerk fields", file); + } + + const mapping = entry.transformer as Record; + for (const [source, target] of Object.entries(mapping)) { + if (typeof target !== "string" || target.length === 0) { + invalid( + `\`transformer.${source}\` must map to a Clerk field name, got ${typeof target}`, + file, + ); + } + } + + // Without this the import runs to completion and creates every user with no + // external_id, which is what makes a migration re-runnable and reversible. + if (!Object.values(mapping).includes("userId")) { + invalid( + "no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id", + file, + ); + } + + if ( + entry.defaults !== undefined && + (entry.defaults === null || typeof entry.defaults !== "object" || Array.isArray(entry.defaults)) + ) { + invalid("`defaults` must be an object when present", file); + } + + for (const hook of ["preTransform", "postTransform"] as const) { + if (entry[hook] !== undefined && typeof entry[hook] !== "function") { + invalid(`\`${hook}\` must be a function when present`, file); + } + } + + if (transformers.some((builtIn) => builtIn.key === entry.key)) { + invalid( + `\`key\` is "${String(entry.key)}", which is already a built-in transformer. Choose another key`, + file, + ); + } + + return { + ...(entry as unknown as TransformerRegistryEntry), + description: (entry.description as string | undefined) ?? "Custom transformer", + }; +} + +/** + * Imports and validates a user-authored transformer. + * + * @throws CliError when the path is missing, the module fails to load, or the + * exported value does not match the registry entry shape. + */ +export async function loadCustomTransformer(file: string): Promise { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No transformer file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + if (fs.statSync(resolved).isDirectory()) { + throw new CliError(`${resolved} is a directory, not a transformer file.`, { + code: ERROR_CODE.USAGE_ERROR, + }); + } + + let module: Record; + try { + // A file URL rather than a bare path: an absolute POSIX path happens to + // work, but a Windows path (`C:\...`) is not a valid import specifier. + module = (await import(Bun.pathToFileURL(resolved).href)) as Record; + } catch (error) { + throw new CliError( + `Could not load ${file}: ${(error as Error).message}\n` + + "The file must be valid JavaScript or TypeScript that this CLI can import.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + if (module.default === undefined) { + // Point at what they probably meant rather than just restating the rule. + const named = Object.keys(module).filter((key) => key !== "default"); + const hint = + named.length > 0 + ? ` Found named export${named.length === 1 ? "" : "s"} ${named.map((n) => `\`${n}\``).join(", ")} — did you mean \`export default\`?` + : ""; + throw new CliError(`${file} has no default export.${hint}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + } + + return validateTransformer(module.default, file); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/registry.ts b/packages/cli-core/src/commands/migrate/transformers/registry.ts new file mode 100644 index 000000000..6279577a0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/registry.ts @@ -0,0 +1,74 @@ +/** + * Transformer registry. + * + * `migrate run` reads this array to resolve `--transformer` and to list the + * valid choices in help output and tab-completion. + * + * To add a platform: create `transformers/.ts` exporting a + * `TransformerRegistryEntry`, then add it to the array below. + */ + +import type { TransformerRegistryEntry } from "../types.ts"; +import auth0Transformer from "./auth0.ts"; +import authjsTransformer from "./authjs.ts"; +import betterAuthTransformer from "./betterauth.ts"; +import clerkTransformer from "./clerk.ts"; +import firebaseTransformer from "./firebase.ts"; +import supabaseTransformer from "./supabase.ts"; + +export const transformers: TransformerRegistryEntry[] = [ + clerkTransformer, + auth0Transformer, + authjsTransformer, + betterAuthTransformer, + firebaseTransformer, + supabaseTransformer, +]; + +/** + * Transformers loaded from a user's `--transformer-file` for this invocation. + * + * Kept beside the built-ins rather than pushed into them, so the shipped list + * is never mutated and `--transformer`'s choices stay exactly the built-in + * keys. One CLI invocation loads at most one, so this holding a single entry is + * the normal case; the array shape just avoids a special case in the lookups. + */ +const customTransformers: TransformerRegistryEntry[] = []; + +export function registerCustomTransformer(entry: TransformerRegistryEntry): void { + customTransformers.push(entry); +} + +/** Test-only: drops anything a previous test registered. */ +export function __resetCustomTransformersForTesting(): void { + customTransformers.length = 0; +} + +/** Built-ins plus whatever `--transformer-file` loaded. */ +export function allTransformers(): TransformerRegistryEntry[] { + return [...transformers, ...customTransformers]; +} + +/** + * The built-in keys, for `--transformer`'s choices and tab-completion. + * + * Deliberately excludes custom transformers: they are selected by path via + * `--transformer-file`, and Commander resolves these choices once at + * registration time, before any file could have been loaded. + */ +export function transformerKeys(): string[] { + return transformers.map((entry) => entry.key); +} + +/** + * Looks up a transformer by key, custom ones included. + * + * @throws Error when no transformer is registered under that key. + */ +export function getTransformer(key: string): TransformerRegistryEntry { + const transformer = allTransformers().find((entry) => entry.key === key); + if (!transformer) { + throw new Error(`Transformer not found for key: ${key}`); + } + return transformer; +} diff --git a/packages/cli-core/src/commands/migrate/transformers/shared.ts b/packages/cli-core/src/commands/migrate/transformers/shared.ts new file mode 100644 index 000000000..c4040230a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/shared.ts @@ -0,0 +1,93 @@ +/** + * Helpers shared by more than one transformer. + * + * Every source platform records verification as a sibling field of the + * identifier, and several ship a single `name` string where Clerk wants a + * first/last pair — so both live here rather than being copied five times. + */ + +/** + * How a platform records that an identifier is verified. + * + * - `boolean` — a true/false flag (Auth0, Better Auth, Firebase). A CSV export + * turns these into the *strings* `"true"`/`"false"`, so `"false"` must not + * be mistaken for a truthy value. + * - `timestamp` — a nullable confirmation time (Auth.js `email_verified`, + * Supabase `email_confirmed_at`). Any real value means verified. + */ +export type VerificationStyle = "boolean" | "timestamp"; + +/** CSV exports write SQL NULL as one of these rather than an empty cell. */ +const NULLISH_STRINGS = new Set(["", "null", "nil", "undefined", "\\n"]); + +export function isVerified(value: unknown, style: VerificationStyle): boolean { + if (value === null || value === undefined) return false; + + if (style === "boolean") { + return value === true || value === 1 || value === "true" || value === "1"; + } + + if (value instanceof Date) return !Number.isNaN(value.getTime()); + if (typeof value === "number") return true; + return typeof value === "string" && !NULLISH_STRINGS.has(value.trim().toLowerCase()); +} + +/** + * Routes an identifier to its verified or unverified field, then drops the + * platform's verification marker. + * + * An unverified identifier must not go on `POST /v1/users`'s primary field: + * Clerk creates those verified, which would silently promote an address the + * source platform never confirmed. + */ +export function routeByVerification( + user: Record, + field: "email" | "phone", + verifiedField: string, + style: VerificationStyle, +): void { + const value = user[field]; + if (value && !isVerified(user[verifiedField], style)) { + user[field === "email" ? "unverifiedEmailAddresses" : "unverifiedPhoneNumbers"] = value; + delete user[field]; + } + delete user[verifiedField]; +} + +/** + * Splits a single display name into `firstName` and `lastName`. + * + * Only splits when there are at least two words — a one-word name would + * otherwise produce a first name with no last name, which several instance + * configurations reject. + */ +export function splitName(user: Record, field = "name"): void { + const name = user[field]; + if (!name || typeof name !== "string") return; + + const parts = name.trim().split(/\s+/); + if (parts.length > 1) { + user.firstName = parts[0]; + user.lastName = parts.slice(1).join(" "); + } + delete user[field]; +} + +/** + * Converts a source timestamp to ISO 8601, leaving it untouched when it does + * not parse so the schema reports it as a validation failure with the original + * value visible in the log. + * + * @param epochMillis - Treat a bare number (or numeric string) as Unix + * milliseconds, which is how Firebase exports timestamps. + */ +export function toIsoDate(value: unknown, epochMillis = false): unknown { + if (value === undefined || value === null || value === "") return value; + + const parsed = + epochMillis && (typeof value === "number" || /^\d+$/.test(String(value))) + ? new Date(Number(value)) + : new Date(String(value)); + + return Number.isNaN(parsed.getTime()) ? value : parsed.toISOString(); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/supabase.ts b/packages/cli-core/src/commands/migrate/transformers/supabase.ts new file mode 100644 index 000000000..1efde4b6a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/supabase.ts @@ -0,0 +1,70 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, toIsoDate } from "./shared.ts"; + +/** + * Supabase Auth → Clerk transformer. + * + * Works with a `auth.users` export, per + * https://supabase.com/docs/guides/auth/managing-user-data#exporting-users + * + * Supabase records verification as a nullable confirmation timestamp + * (`email_confirmed_at`) rather than a boolean, and stores timestamps in + * PostgreSQL's format (`2024-06-29 20:25:06.126079+00`). + */ + +/** Discord writes display names as `name#0`; the suffix reads as a URL to Clerk. */ +const DISCORD_DISCRIMINATOR = /#\d+$/; + +function stripDiscriminator(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + return value.replace(DISCORD_DISCRIMINATOR, "").trim() || undefined; +} + +const supabaseTransformer = { + key: "supabase", + label: "Supabase", + description: + "Works with a Supabase `auth.users` export. Use --skip-unsupported-providers to drop users whose only social provider is not enabled in Clerk.", + transformer: { + id: "userId", + email: "email", + email_confirmed_at: "emailConfirmedAt", + first_name: "firstName", + last_name: "lastName", + encrypted_password: "password", + phone: "phone", + phone_confirmed_at: "phoneConfirmedAt", + raw_user_meta_data: "publicMetadata", + created_at: "createdAt", + }, + postTransform: (user) => { + user.createdAt = toIsoDate(user.createdAt); + routeByVerification(user, "email", "emailConfirmedAt", "timestamp"); + routeByVerification(user, "phone", "phoneConfirmedAt", "timestamp"); + + // A basic SQL export has no first_name/last_name columns; the name lives in + // user metadata instead, under whichever key the provider happened to use. + if (!user.firstName && user.publicMetadata && typeof user.publicMetadata === "object") { + const meta = user.publicMetadata as Record; + const displayName = stripDiscriminator(meta.display_name ?? meta.first_name ?? meta.name); + if (displayName) { + const parts = displayName.split(/\s+/); + user.firstName = parts[0]; + if (parts.length > 1 && !user.lastName) user.lastName = parts.slice(1).join(" "); + } + } + + for (const field of ["firstName", "lastName"] as const) { + if (typeof user[field] === "string") { + const cleaned = stripDiscriminator(user[field]); + if (cleaned) user[field] = cleaned; + else delete user[field]; + } + } + }, + defaults: { + passwordHasher: "bcrypt" as const, + }, +} satisfies TransformerRegistryEntry; + +export default supabaseTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts b/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts new file mode 100644 index 000000000..ae492f01d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts @@ -0,0 +1,405 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { loadUsersFromFile, transformUsers } from "../lib/transform.ts"; +import type { FirebaseHashConfig } from "../types.ts"; +import { getTransformer, transformerKeys, transformers } from "./registry.ts"; +import { isVerified } from "./shared.ts"; + +const DATE_TIME = "2026-01-01T00:00:00"; + +const FIREBASE_HASH: FirebaseHashConfig = { + base64_signer_key: "SIGNERKEY==", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, +}; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-transformers-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +/** Writes `records` to a uniquely-named file and loads it through `key`. */ +async function load(key: string, records: unknown, ext = "json", context = {}) { + const file = `${key}-${Math.abs(JSON.stringify(records).length)}-${ext}.${ext}`; + fs.writeFileSync( + path.join(workDir, file), + typeof records === "string" ? records : JSON.stringify(records), + ); + return loadUsersFromFile(file, key, DATE_TIME, { context }); +} + +const one = (key: string, record: Record, context = {}) => + transformUsers([record], key, DATE_TIME, { validate: false, context }).transformedData[0] as + | Record + | undefined; + +describe("registry", () => { + test("registers all six platforms", () => { + expect(transformerKeys()).toEqual([ + "clerk", + "auth0", + "authjs", + "betterauth", + "firebase", + "supabase", + ]); + }); + + test.each([...transformers])("$key maps a source field to userId", (transformer) => { + expect(Object.values(transformer.transformer)).toContain("userId"); + }); + + test.each([...transformers])("$key carries a label and description", (transformer) => { + expect(transformer.label.length).toBeGreaterThan(0); + expect(transformer.description.length).toBeGreaterThan(0); + }); + + test("throws for an unregistered key", () => { + expect(() => getTransformer("okta")).toThrow(/Transformer not found/); + }); +}); + +describe("isVerified", () => { + // A CSV export stringifies everything, so the boolean style must read + // "false" as false. Treating it as truthy would mark unconfirmed addresses + // verified on import — the exact thing the routing exists to prevent. + test.each([ + [true, true], + ["true", true], + [1, true], + ["1", true], + [false, false], + ["false", false], + [0, false], + ["0", false], + ["", false], + [null, false], + [undefined, false], + ])("boolean style: %p -> %p", (value, expected) => { + expect(isVerified(value, "boolean")).toBe(expected); + }); + + // The timestamp style is presence-based: any real confirmation time counts, + // and SQL NULL arrives from a CSV export as one of several spellings. + test.each([ + ["2024-06-29 20:25:06+00", true], + ["2024-01-15T10:30:00.000Z", true], + ["", false], + [" ", false], + ["null", false], + ["NULL", false], + ["\\N", false], + [null, false], + [undefined, false], + ])("timestamp style: %p -> %p", (value, expected) => { + expect(isVerified(value, "timestamp")).toBe(expected); + }); +}); + +describe("auth0", () => { + const base = { user_id: "auth0|abc", email: "a@x.dev", passwordHash: "$2b$10$hash" }; + + test("maps identity, name and metadata onto the Clerk schema", async () => { + const { users } = await load("auth0", [ + { ...base, email_verified: true, given_name: "Ada", family_name: "Lovelace" }, + ]); + expect(users[0]).toMatchObject({ + userId: "auth0|abc", + email: "a@x.dev", + firstName: "Ada", + lastName: "Lovelace", + password: "$2b$10$hash", + passwordHasher: "bcrypt", + }); + }); + + test.each([ + [true, "email", undefined], + [false, undefined, "a@x.dev"], + [undefined, undefined, "a@x.dev"], + ])("email_verified=%p routes the address correctly", (verified, kept, unverified) => { + const user = one("auth0", { ...base, email_verified: verified }); + expect(user?.email).toBe(kept ? "a@x.dev" : undefined); + expect(user?.unverifiedEmailAddresses).toBe(unverified); + }); + + test("routes an unverified phone away from the primary field", () => { + const user = one("auth0", { ...base, phone_number: "+15555550100", phone_verified: false }); + expect(user?.phone).toBeUndefined(); + expect(user?.unverifiedPhoneNumbers).toBe("+15555550100"); + }); + + test("drops the platform's verification markers", () => { + const user = one("auth0", { ...base, email_verified: true, phone_verified: true }); + expect("emailVerified" in (user ?? {})).toBe(false); + expect("phoneVerified" in (user ?? {})).toBe(false); + }); + + test("keeps user_metadata public and app_metadata private", async () => { + const { users } = await load("auth0", [ + { + ...base, + email_verified: true, + user_metadata: { theme: "dark" }, + app_metadata: { plan: "pro" }, + }, + ]); + expect(users[0]?.publicMetadata).toEqual({ theme: "dark" }); + expect(users[0]?.privateMetadata).toEqual({ plan: "pro" }); + }); +}); + +describe("authjs", () => { + const base = { id: "cuid1", email: "a@x.dev" }; + + test("treats a confirmation timestamp as verified", () => { + const user = one("authjs", { ...base, email_verified: "2024-01-15T10:30:00.000Z" }); + expect(user?.email).toBe("a@x.dev"); + expect(user?.unverifiedEmailAddresses).toBeUndefined(); + }); + + test.each([[null], [""], [undefined]])("treats email_verified=%p as unverified", (value) => { + const user = one("authjs", { ...base, email_verified: value }); + expect(user?.unverifiedEmailAddresses).toBe("a@x.dev"); + }); + + test.each([ + ["Jane Doe", "Jane", "Doe"], + ["Mary Jane Watson", "Mary", "Jane Watson"], + [" Ada Lovelace ", "Ada", "Lovelace"], + ])("splits %p into %p / %p", (name, firstName, lastName) => { + const user = one("authjs", { ...base, name }); + expect(user?.firstName).toBe(firstName); + expect(user?.lastName).toBe(lastName); + }); + + test("leaves a single-word name unsplit rather than inventing a last name", () => { + const user = one("authjs", { ...base, name: "Prince" }); + expect(user?.firstName).toBeUndefined(); + expect(user?.lastName).toBeUndefined(); + expect("name" in (user ?? {})).toBe(false); + }); + + test("imports without a password, since Auth.js core is passwordless", async () => { + const { users } = await load("authjs", [{ ...base, email_verified: "2024-01-01" }]); + expect(users[0]?.password).toBeUndefined(); + expect(users[0]?.passwordHasher).toBeUndefined(); + }); +}); + +describe("betterauth", () => { + const base = { user_id: "ba1", email: "a@x.dev", email_verified: true }; + + test("maps the credential hash and defaults the hasher to bcrypt", async () => { + const { users } = await load("betterauth", [{ ...base, password_hash: "$2a$10$hash" }]); + expect(users[0]).toMatchObject({ password: "$2a$10$hash", passwordHasher: "bcrypt" }); + }); + + test("routes an unverified phone", () => { + const user = one("betterauth", { + ...base, + phone_number: "+15555550100", + phone_number_verified: false, + }); + expect(user?.unverifiedPhoneNumbers).toBe("+15555550100"); + }); + + test.each([ + [true, true], + [false, undefined], + [undefined, undefined], + ])("banned=%p is carried through as %p", (banned, expected) => { + expect(one("betterauth", { ...base, banned })?.banned).toBe(expected as boolean | undefined); + }); + + test("drops plugin-only columns during validation", async () => { + const { users } = await load("betterauth", [ + { ...base, role: "admin", display_username: "ADA", two_factor_enabled: true }, + ]); + const user = users[0] as Record; + expect("role" in user).toBe(false); + expect("display_username" in user).toBe(false); + expect("two_factor_enabled" in user).toBe(false); + }); +}); + +describe("firebase", () => { + const base = { localId: "fb1", email: "a@x.dev", emailVerified: true }; + const withHash = { ...base, passwordHash: "SGFzaA==", salt: "U2FsdA==" }; + + test("builds the scrypt digest Clerk expects, parameters inline", async () => { + const { users } = await load("firebase", { users: [withHash] }, "json", { + firebaseHashConfig: FIREBASE_HASH, + }); + expect(users[0]?.password).toBe("SGFzaA==$U2FsdA==$SIGNERKEY==$Bw==$8$14"); + expect(users[0]?.passwordHasher).toBe("scrypt_firebase"); + }); + + test("refuses to import hashes without the project's hash parameters", async () => { + await expect(load("firebase", { users: [withHash] })).rejects.toThrow( + /Firebase password hashes/, + ); + }); + + test("imports a passwordless export with no hash parameters at all", async () => { + const { users } = await load("firebase", { users: [base] }); + expect(users).toHaveLength(1); + expect(users[0]?.password).toBeUndefined(); + }); + + test("unwraps the { users: [...] } export shape", async () => { + const { users } = await load("firebase", { users: [base, { ...base, localId: "fb2" }] }); + expect(users.map((u) => u.userId)).toEqual(["fb1", "fb2"]); + }); + + test("accepts a bare array too", async () => { + const { users } = await load("firebase", [base]); + expect(users).toHaveLength(1); + }); + + test("rejects a JSON export that is neither", async () => { + await expect(load("firebase", { records: [] })).rejects.toThrow(CliError); + }); + + test("prepends headers to a headerless CSV export", async () => { + const csv = "fb9,a@x.dev,true,,,Ada Lovelace,,,,,,,,,,,,,,,,,,1704067200000,,,,,\n"; + const { users } = await load("firebase", csv, "csv"); + expect(users[0]).toMatchObject({ userId: "fb9", email: "a@x.dev", firstName: "Ada" }); + }); + + test.each([ + ["1704067200000", "2024-01-01T00:00:00.000Z"], + [1704067200000, "2024-01-01T00:00:00.000Z"], + ])("converts the Unix-millisecond createdAt %p", (createdAt, expected) => { + expect(one("firebase", { ...base, createdAt })?.createdAt).toBe(expected); + }); + + test.each([ + [true, true], + ["true", true], + [false, false], + ["false", false], + ])("emailVerified=%p keeps the address primary: %p", (emailVerified, verified) => { + const user = one("firebase", { ...base, emailVerified }); + expect(user?.email !== undefined).toBe(verified); + }); +}); + +describe("supabase", () => { + const base = { id: "sb1", email: "a@x.dev", email_confirmed_at: "2024-06-29 20:25:06.126079+00" }; + + test("maps the bcrypt password and converts the PostgreSQL timestamp", async () => { + const { users } = await load("supabase", [ + { ...base, encrypted_password: "$2b$10$hash", created_at: "2024-06-29 20:25:06.126079+00" }, + ]); + expect(users[0]).toMatchObject({ + password: "$2b$10$hash", + passwordHasher: "bcrypt", + createdAt: "2024-06-29T20:25:06.126Z", + }); + }); + + test.each([ + ["2024-06-29 20:25:06+00", true], + [null, false], + ["", false], + ])("email_confirmed_at=%p means verified: %p", (confirmedAt, verified) => { + const user = one("supabase", { ...base, email_confirmed_at: confirmedAt }); + expect(user?.email !== undefined).toBe(verified); + }); + + test("falls back to user metadata for a missing first name", () => { + const user = one("supabase", { + ...base, + raw_user_meta_data: { display_name: "Ada Lovelace" }, + }); + expect(user?.firstName).toBe("Ada"); + expect(user?.lastName).toBe("Lovelace"); + }); + + test("prefers explicit name columns over metadata", () => { + const user = one("supabase", { + ...base, + first_name: "Grace", + raw_user_meta_data: { display_name: "Ada Lovelace" }, + }); + expect(user?.firstName).toBe("Grace"); + }); + + test.each([ + ["ada#0", "ada"], + ["ada#1234", "ada"], + ])("strips the Discord discriminator from %p", (displayName, expected) => { + const user = one("supabase", { ...base, raw_user_meta_data: { display_name: displayName } }); + expect(user?.firstName).toBe(expected); + }); + + test("drops a name that was nothing but a discriminator", () => { + const user = one("supabase", { ...base, first_name: "#0" }); + expect(user?.firstName).toBeUndefined(); + }); +}); + +describe("invalid records", () => { + const INVALID: [string, Record][] = [ + ["auth0", { user_id: "a1" }], + ["authjs", { id: "a2" }], + ["betterauth", { user_id: "a3" }], + ["firebase", { localId: "a4" }], + ["supabase", { id: "a5" }], + ]; + + test.each(INVALID)( + "%s logs a user with no identifier instead of crashing", + async (key, record) => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); + + const { users, validationFailed } = await load(key, [ + record, + { ...record, ...identifierFor(key) }, + ]); + + expect(validationFailed).toBe(1); + expect(users).toHaveLength(1); + + const logged = fs + .readdirSync(getLogDir()) + .flatMap((name) => + fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n"), + ) + .map((line) => JSON.parse(line) as Record); + expect(logged.some((entry) => entry.status === "fail")).toBe(true); + }, + ); + + test.each(INVALID)("%s logs a malformed email rather than sending it", async (key, record) => { + const { users, validationFailed } = await load(key, [ + { ...record, ...identifierFor(key, "not-an-email") }, + ]); + expect(validationFailed).toBe(1); + expect(users).toHaveLength(0); + }); +}); + +/** The per-platform source field that becomes a Clerk identifier. */ +function identifierFor(key: string, email = "ok@x.dev"): Record { + if (key === "auth0") return { email, email_verified: true }; + if (key === "authjs") return { email, email_verified: "2024-01-01" }; + if (key === "betterauth") return { email, email_verified: true }; + if (key === "firebase") return { email, emailVerified: true }; + return { email, email_confirmed_at: "2024-01-01 00:00:00+00" }; +} diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts new file mode 100644 index 000000000..40ba3a5ec --- /dev/null +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -0,0 +1,193 @@ +/** + * Shared types for `clerk migrate`. + * + * Ported from the standalone migration-tool's `src/types.ts`. The Clerk API + * error shape is declared locally rather than imported from `@clerk/types`, + * because this command family talks to BAPI through `lib/bapi.ts` instead of + * `@clerk/backend`. + */ + +import type * as z from "zod"; +import type { userSchema } from "./validator.ts"; + +/** + * Password hashing algorithms Clerk can verify on import. + * + * When migrating users with existing passwords, the source platform's hasher + * must be named so Clerk can validate the digest instead of rejecting it. + */ +export const PASSWORD_HASHERS = [ + "argon2i", + "argon2id", + "awscognito", + "bcrypt", + "bcrypt_peppered", + "bcrypt_sha256_django", + "hmac_sha256_utf16_b64", + "md5", + "md5_salted", + "pbkdf2_sha1", + "pbkdf2_sha256", + "pbkdf2_sha256_django", + "pbkdf2_sha512", + "pbkdf2_sha512_hex", + "scrypt_firebase", + "scrypt_werkzeug", + "sha256", + "sha256_salted", + "md5_phpass", + "ldap_ssha", + "sha512_symfony", +] as const; + +/** A user that has passed schema validation and is ready to import. */ +export type User = z.infer; + +/** Union of all registered transformer keys (e.g. `"clerk"`). */ +export type TransformerKey = string; + +/** + * One error entry as returned in a Clerk API error response body. + * + * Local mirror of `@clerk/types`' `ClerkAPIError` covering only the fields the + * migration logs read. + */ +export type ClerkApiError = { + code: string; + message: string; + longMessage?: string; +}; + +/** A failed user-creation attempt, as handed to the error logger. */ +export type ErrorPayload = { + userId: string; + status: string; + errors: ClerkApiError[]; +}; + +/** A user that failed schema validation before any API call was made. */ +export type ValidationErrorPayload = { + error: string; + path: (string | number)[]; + userId: string; + row: number; +}; + +/** A formatted error line as written to the NDJSON log. */ +export type ErrorLog = { + type: string; + userId: string; + status: string; + error: string | undefined; +}; + +/** One import attempt as written to the NDJSON log. */ +export type ImportLogEntry = { + userId: string; + status: "success" | "error"; + clerkUserId?: string; + error?: string; + code?: string; +}; + +/** One exported user as written to the NDJSON log. */ +export type ExportLogEntry = { + /** The source platform's ID for this user. */ + userId: string; + status: "success" | "error"; + error?: string; +}; + +/** One deletion attempt as written to the NDJSON log. */ +export type DeleteLogEntry = { + /** The source platform's ID — the Clerk user's `external_id`. */ + userId: string; + clerkUserId?: string; + status: "success" | "error"; + error?: string; + code?: string; +}; + +/** Totals for a completed import run. */ +export type ImportSummary = { + totalProcessed: number; + successful: number; + failed: number; + validationFailed: number; + errorBreakdown: Map; +}; + +/** + * Per-directory migration state, persisted to a cwd-relative `.settings` file. + * + * Deliberately not routed through `~/.config/clerk/config.json`: that file is + * keyed by linked-project identity, which is a different concept from "which + * file did I last migrate with". + */ +export type Settings = { + key?: string; + file?: string; + skipUnsupportedProviders?: boolean; + firebaseHashConfig?: FirebaseHashConfig; +}; + +/** + * Firebase's scrypt parameters, needed to rebuild a password hash Clerk can + * verify. + * + * Found in the Firebase console under Authentication → Users → (⋮) → Password + * hash parameters. All four are required together; a partial set produces a + * digest that silently fails every sign-in. + */ +export type FirebaseHashConfig = { + base64_signer_key: string; + base64_salt_separator: string; + rounds: number; + mem_cost: number; +}; + +/** + * Per-run values a transformer may need but cannot read from the user record. + * + * Passed to `postTransform` rather than held in module state so two runs in one + * process — or two test files — cannot see each other's configuration. + */ +export type TransformContext = { + firebaseHashConfig?: FirebaseHashConfig; +}; + +/** + * Result of a transformer's `preTransform` hook. + * + * @property filePath - Path to read from; may differ from the input (e.g. a + * temp file with generated CSV headers). + * @property data - Users already extracted from a wrapper object, when the + * source format nests them. + */ +export type PreTransformResult = { + filePath: string; + data?: Record[]; +}; + +/** + * A platform transformer: how to get from one source export shape to Clerk's + * import shape. + * + * @property transformer - Source field path → Clerk field name. + * @property defaults - Values merged into every user from this platform. + * @property preTransform - Runs before field mapping. + * @property postTransform - Mutates a user after field mapping, given the + * run's {@link TransformContext}. + */ +export type TransformerRegistryEntry = { + key: string; + label: string; + description: string; + transformer: Record; + defaults?: Record; + preTransform?: ( + filePath: string, + fileType: string, + ) => PreTransformResult | Promise; + postTransform?: (user: Record, context: TransformContext) => void; +}; diff --git a/packages/cli-core/src/commands/migrate/validator.test.ts b/packages/cli-core/src/commands/migrate/validator.test.ts new file mode 100644 index 000000000..d398a4dc9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/validator.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, test } from "bun:test"; +import { PASSWORD_HASHERS } from "./types.ts"; +import { userSchema } from "./validator.ts"; + +const base = { userId: "user_1", email: "a@example.com" }; + +describe("userSchema identifiers", () => { + const IDENTIFIER_CASES = [ + ["email", { email: "a@example.com" }, true], + ["emailAddresses array", { emailAddresses: ["a@example.com"] }, true], + ["unverified email", { unverifiedEmailAddresses: ["a@example.com"] }, true], + ["phone", { phone: "+15555550100" }, true], + ["unverified phone", { unverifiedPhoneNumbers: ["+15555550100"] }, true], + ["username", { username: "alice" }, true], + ["nothing", {}, false], + ["empty email array", { email: [] }, false], + ["empty username", { username: "" }, false], + ] as const; + + test.each([...IDENTIFIER_CASES])( + "accepts a user identified by %s: %p -> %p", + (_label, fields, ok) => { + expect(userSchema.safeParse({ userId: "user_1", ...fields }).success).toBe(ok); + }, + ); + + test("reports the identifier failure against the email path", () => { + const result = userSchema.safeParse({ userId: "user_1" }); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues[0]?.path).toEqual(["email"]); + }); +}); + +describe("userSchema passwords", () => { + test("rejects a password without a hasher", () => { + const result = userSchema.safeParse({ ...base, password: "digest" }); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues[0]?.path).toEqual(["passwordHasher"]); + }); + + test("accepts a password with a valid hasher", () => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: "bcrypt" }).success, + ).toBe(true); + }); + + test("rejects an unknown hasher", () => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: "rot13" }).success, + ).toBe(false); + }); + + test.each([...PASSWORD_HASHERS])("accepts the %s hasher", (hasher) => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: hasher }).success, + ).toBe(true); + }); +}); + +describe("userSchema field types", () => { + const FIELD_CASES = [ + ["valid email", { email: "a@example.com" }, true], + ["malformed email", { email: "not-an-email" }, false], + ["email array with one bad entry", { email: ["a@example.com", "nope"] }, false], + ["userId missing", { userId: undefined }, false], + ["valid createdAt", { createdAt: "2024-01-01T00:00:00Z" }, true], + ["unparseable createdAt", { createdAt: "yesterday" }, false], + ["integer org limit", { createOrganizationsLimit: 3 }, true], + ["fractional org limit", { createOrganizationsLimit: 1.5 }, false], + ["metadata object", { publicMetadata: { plan: "pro" } }, true], + ["metadata string", { publicMetadata: "pro" }, false], + ["backupCodes array", { backupCodes: ["a", "b"] }, true], + ] as const; + + test.each([...FIELD_CASES])("%s -> %p", (_label, fields, ok) => { + expect(userSchema.safeParse({ ...base, ...fields }).success).toBe(ok); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/validator.ts b/packages/cli-core/src/commands/migrate/validator.ts new file mode 100644 index 000000000..2ebc31166 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/validator.ts @@ -0,0 +1,99 @@ +/** + * Zod schema every user is validated against before it reaches BAPI. + * + * Ported from the standalone migration-tool's `src/migrate/validator.ts`. + * + * ============================================================================ + * ONLY EDIT THIS IF YOU ARE ADDING A NEW FIELD. + * Adding support for a new source platform means adding a transformer, not + * touching the schema. + * ============================================================================ + */ + +import * as z from "zod"; +import { PASSWORD_HASHERS } from "./types.ts"; + +const metadataSchema = z.record(z.string(), z.unknown()); + +const dateStringSchema = z.string().refine((value) => !Number.isNaN(new Date(value).getTime()), { + message: "Expected a valid date string", +}); + +/** Zod enum of the password hashers Clerk accepts on import. */ +export const passwordHasherEnum = z.enum(PASSWORD_HASHERS); + +/** + * Validates user data before sending it to Clerk. + * + * Everything is optional except: + * - `userId`, required for tracking, logging and `--resume-after` + * - `passwordHasher`, required whenever `password` is present + * - at least one identifier (email, phone or username) + * + * Identifier fields accept either a single value or an array. + */ +export const userSchema = z + .object({ + userId: z.string(), + // Email fields + email: z.union([z.email(), z.array(z.email())]).optional(), + emailAddresses: z.union([z.email(), z.array(z.email())]).optional(), + unverifiedEmailAddresses: z.union([z.email(), z.array(z.email())]).optional(), + // Phone fields + phone: z.union([z.string(), z.array(z.string())]).optional(), + phoneNumbers: z.union([z.string(), z.array(z.string())]).optional(), + unverifiedPhoneNumbers: z.union([z.string(), z.array(z.string())]).optional(), + // User info + username: z.string().optional(), + firstName: z.string().optional(), + lastName: z.string().optional(), + // Password + password: z.string().optional(), + passwordHasher: passwordHasherEnum.optional(), + // 2FA + totpSecret: z.string().optional(), + backupCodesEnabled: z.boolean().optional(), + backupCodes: z.array(z.string()).optional(), + // Metadata + unsafeMetadata: metadataSchema.optional(), + publicMetadata: metadataSchema.optional(), + privateMetadata: metadataSchema.optional(), + // Additional Clerk API fields + banned: z.boolean().optional(), + bypassClientTrust: z.boolean().optional(), + createOrganizationEnabled: z.boolean().optional(), + createOrganizationsLimit: z.number().int().optional(), + createdAt: dateStringSchema.optional(), + deleteSelfEnabled: z.boolean().optional(), + legalAcceptedAt: dateStringSchema.optional(), + skipLegalChecks: z.boolean().optional(), + skipPasswordChecks: z.boolean().optional(), + }) + .refine((data) => !data.password || data.passwordHasher, { + message: "passwordHasher is required when password is provided", + path: ["passwordHasher"], + }) + .refine( + (data) => { + const hasValue = (field: unknown): boolean => { + if (!field) return false; + if (typeof field === "string") return field.length > 0; + if (Array.isArray(field)) return field.length > 0; + return false; + }; + return ( + hasValue(data.email) || + hasValue(data.emailAddresses) || + hasValue(data.unverifiedEmailAddresses) || + hasValue(data.phone) || + hasValue(data.phoneNumbers) || + hasValue(data.unverifiedPhoneNumbers) || + hasValue(data.username) + ); + }, + { + message: + "User must have at least one identifier (email, phone, unverified email, unverified phone, or username)", + path: ["email"], + }, + ); diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts new file mode 100644 index 000000000..f3633665d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -0,0 +1,267 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +useCaptureLog(); + +type Prompt = { message: string; default?: string; validate?: (v?: string) => string | undefined }; +type SelectPrompt = Prompt & { choices: { name: string; value: string }[] }; + +// Registered at file top, before the wizard (or anything it imports) loads. +// This file is the only consumer of the mocked prompt modules. +const mockSelect = mock(async (_config: SelectPrompt) => undefined as unknown); +const mockText = mock(async (_config: Prompt) => "" as unknown); + +mock.module("../../lib/listage.ts", () => ({ + ...listageStubs, + select: (config: SelectPrompt) => mockSelect(config), +})); + +mock.module("../../lib/prompts.ts", () => ({ + confirm: async () => true, + text: (config: Prompt) => mockText(config), + password: async () => "", + editor: async () => "{}", +})); + +const { runWizard, throwAgentFlagsRequired } = await import("./wizard.ts"); +const { saveSettings } = await import("./lib/settings.ts"); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-"))); + process.chdir(workDir); + fs.writeFileSync(path.join(workDir, "users.json"), "[]"); + fs.writeFileSync(path.join(workDir, "other.csv"), ""); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + mockSelect.mockReset(); + mockText.mockReset(); + fs.rmSync(path.join(workDir, ".settings"), { force: true }); +}); + +/** The config object the wizard passed to its Nth `text`/`select` prompt. */ +const textCall = (index: number): Prompt | undefined => mockText.mock.calls[index]?.[0]; +const selectCall = (index: number): SelectPrompt | undefined => mockSelect.mock.calls[index]?.[0]; + +describe("transformer picker", () => { + test("is built from the registry, so every platform appears", async () => { + mockSelect.mockResolvedValue("auth0"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.choices.map((choice) => choice.value)).toEqual([ + "clerk", + "auth0", + "authjs", + "betterauth", + "firebase", + "supabase", + ]); + }); + + test("labels each choice with the transformer's display name", async () => { + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.choices.map((choice) => choice.name)).toContain("Better Auth"); + }); + + test("is skipped when --transformer was already passed", async () => { + mockText.mockResolvedValue("users.json"); + + const result = await runWizard({ transformer: "clerk" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(result.transformer).toBe("clerk"); + }); +}); + +describe("defaults from the previous run", () => { + test("pre-selects the last transformer and pre-fills the last file", async () => { + saveSettings({ key: "supabase", file: "other.csv" }); + mockSelect.mockResolvedValue("supabase"); + mockText.mockResolvedValue("other.csv"); + + await runWizard({}); + + expect(selectCall(0)?.default).toBe("supabase"); + expect(textCall(0)?.default).toBe("other.csv"); + }); + + test("offers no default when nothing has been saved", async () => { + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.default).toBeUndefined(); + expect(textCall(0)?.default).toBeUndefined(); + }); + + // A saved key from a build that has since dropped that transformer would + // otherwise pre-select a value the picker cannot offer. + test("ignores a saved transformer that is no longer registered", async () => { + saveSettings({ key: "okta" }); + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.default).toBeUndefined(); + }); +}); + +describe("file prompt validation", () => { + const validate = async () => { + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + await runWizard({}); + return textCall(0)?.validate; + }; + + test.each([ + ["users.json", undefined], + ["other.csv", undefined], + ])("accepts %s", async (file, expected) => { + expect((await validate())?.(file)).toBe(expected as undefined); + }); + + test("rejects an empty answer", async () => { + expect((await validate())?.("")).toMatch(/required/); + }); + + test("rejects a file that does not exist", async () => { + expect((await validate())?.("missing.json")).toMatch(/File not found/); + }); + + test("rejects an unsupported extension", async () => { + fs.writeFileSync(path.join(workDir, "notes.txt"), ""); + expect((await validate())?.("notes.txt")).toMatch(/\.json or \.csv/); + }); +}); + +describe("firebase hash parameters", () => { + test("are asked for when the firebase transformer is picked", async () => { + mockSelect.mockResolvedValue("firebase"); + mockText + .mockResolvedValueOnce("users.json") + .mockResolvedValueOnce("SIGNER") + .mockResolvedValueOnce("Bw==") + .mockResolvedValueOnce("8") + .mockResolvedValueOnce("14"); + + const result = await runWizard({}); + + expect(result.firebaseHashConfig).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + // Pressing enter through the signer key is how a user says "this export has + // no passwords" — the remaining three would be meaningless without it. + test("stop being asked when the signer key is left blank", async () => { + mockSelect.mockResolvedValue("firebase"); + mockText.mockResolvedValueOnce("users.json").mockResolvedValueOnce(" "); + + const result = await runWizard({}); + + expect(result.firebaseHashConfig).toBeUndefined(); + expect(mockText).toHaveBeenCalledTimes(2); + }); + + test("are pre-filled from the previous run", async () => { + saveSettings({ + firebaseHashConfig: { + base64_signer_key: "SAVED", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }, + }); + mockSelect.mockResolvedValue("firebase"); + mockText + .mockResolvedValueOnce("users.json") + .mockResolvedValueOnce("SAVED") + .mockResolvedValueOnce("Bw==") + .mockResolvedValueOnce("8") + .mockResolvedValueOnce("14"); + + await runWizard({}); + + expect(textCall(1)?.default).toBe("SAVED"); + expect(textCall(3)?.default).toBe("8"); + }); + + test("are not asked for on a non-firebase transformer", async () => { + mockSelect.mockResolvedValue("auth0"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(mockText).toHaveBeenCalledTimes(1); + }); + + test("are not asked for when the flags already supplied them", async () => { + mockSelect.mockResolvedValue("firebase"); + mockText.mockResolvedValue("users.json"); + + const config = { + base64_signer_key: "FLAG", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }; + const result = await runWizard({ firebaseHashConfig: config }); + + expect(mockText).toHaveBeenCalledTimes(1); + expect(result.firebaseHashConfig).toEqual(config); + }); + + test.each([["0"], ["-1"], ["1.5"], ["many"]])("rejects %p as a rounds value", async (value) => { + mockSelect.mockResolvedValue("firebase"); + mockText + .mockResolvedValueOnce("users.json") + .mockResolvedValueOnce("SIGNER") + .mockResolvedValueOnce("Bw==") + .mockResolvedValueOnce("8") + .mockResolvedValueOnce("14"); + + await runWizard({}); + + expect(textCall(3)?.validate?.(value)).toMatch(/positive whole number/); + }); +}); + +describe("throwAgentFlagsRequired", () => { + test.each([ + [{ transformer: true, file: true }, /--transformer and --file /], + [{ transformer: true, file: false }, /--transformer \./], + [{ transformer: false, file: true }, /--file \./], + ])("names only the flags that are missing (%p)", (missing, expected) => { + expect(() => throwAgentFlagsRequired(missing)).toThrow(expected); + }); + + test("says why it cannot prompt", () => { + expect(() => throwAgentFlagsRequired({ transformer: true, file: true })).toThrow( + /cannot prompt in agent mode/, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts new file mode 100644 index 000000000..3082a1b63 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -0,0 +1,167 @@ +/** + * The interactive path behind a bare `clerk migrate`. + * + * Ported from the standalone migration-tool's `src/migrate/cli.ts` interactive + * flow. Every answer is pre-filled from the previous run's `.settings`, so a + * repeat migration is mostly pressing enter. + * + * Agent mode never reaches here — `run` raises a usage error naming the flags + * instead, because an agent cannot answer a prompt. + */ + +import { throwUsageError } from "../../lib/errors.ts"; +import { select } from "../../lib/listage.ts"; +import { log } from "../../lib/log.ts"; +import { text } from "../../lib/prompts.ts"; +import { loadSettings } from "./lib/settings.ts"; +import { fileExists, getFileType } from "./lib/transform.ts"; +import { transformers } from "./transformers/registry.ts"; +import type { FirebaseHashConfig } from "./types.ts"; + +export type WizardResult = { + transformer: string; + file: string; + firebaseHashConfig?: FirebaseHashConfig; +}; + +/** Trims a description down to a single readable hint line. */ +function hint(description: string): string { + const firstSentence = description.split(". ")[0] ?? description; + return firstSentence.length > 96 ? `${firstSentence.slice(0, 93)}...` : firstSentence; +} + +async function pickTransformer(defaultKey: string | undefined): Promise { + // Built from the registry, so a new platform appears here with no second + // place to update. + return select({ + message: "Which platform are you migrating from?", + choices: transformers.map((entry) => ({ + name: entry.label, + value: entry.key, + description: hint(entry.description), + })), + default: defaultKey && transformers.some((t) => t.key === defaultKey) ? defaultKey : undefined, + }); +} + +async function askFile(defaultFile: string | undefined): Promise { + return text({ + message: "Path to the exported user file (JSON or CSV)", + default: defaultFile, + validate: (value) => { + const file = value?.trim(); + if (!file) return "A file path is required"; + if (!fileExists(file)) return `File not found: ${file}`; + if (!getFileType(file)) return "Provide a .json or .csv file"; + return undefined; + }, + }); +} + +/** + * Collects Firebase's four hash parameters. + * + * Asked as a set because a partial set produces a digest that verifies against + * nothing. Pressing enter through all four leaves the config unset, which is + * correct for an export with no password hashes. + */ +async function askFirebaseHashConfig( + saved: FirebaseHashConfig | undefined, +): Promise { + log.info( + "Firebase password hashes need the project's hash parameters. Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + ); + log.info(dimIfSaved(saved)); + + const signerKey = ( + await text({ + message: "base64 signer key (leave blank if this export has no passwords)", + default: saved?.base64_signer_key, + }) + ).trim(); + if (!signerKey) return undefined; + + const saltSeparator = ( + await text({ + message: "base64 salt separator", + default: saved?.base64_salt_separator, + validate: (value) => (value?.trim() ? undefined : "Required alongside the signer key"), + }) + ).trim(); + + const rounds = await askNumber("rounds", saved?.rounds); + const memCost = await askNumber("mem cost", saved?.mem_cost); + + return { + base64_signer_key: signerKey, + base64_salt_separator: saltSeparator, + rounds, + mem_cost: memCost, + }; +} + +function dimIfSaved(saved: FirebaseHashConfig | undefined): string { + return saved + ? "Saved parameters found — press enter to reuse them." + : "Leave the signer key blank if this export carries no passwords."; +} + +async function askNumber(label: string, defaultValue: number | undefined): Promise { + const answer = await text({ + message: label, + default: defaultValue === undefined ? undefined : String(defaultValue), + validate: (value) => { + const parsed = Number(value?.trim()); + return Number.isInteger(parsed) && parsed > 0 ? undefined : "Enter a positive whole number"; + }, + }); + return Number(answer.trim()); +} + +/** + * Fills in whichever of transformer and file were not passed as flags. + * + * @param provided - Flags the caller already supplied; those are not asked for. + */ +export async function runWizard(provided: { + transformer?: string; + file?: string; + firebaseHashConfig?: FirebaseHashConfig; +}): Promise { + const saved = loadSettings(); + + const transformer = provided.transformer ?? (await pickTransformer(saved.key)); + const file = provided.file ?? (await askFile(saved.file)); + + let firebaseHashConfig = provided.firebaseHashConfig; + if (transformer === "firebase" && !firebaseHashConfig) { + firebaseHashConfig = await askFirebaseHashConfig(saved.firebaseHashConfig); + } + + return { transformer, file, ...(firebaseHashConfig ? { firebaseHashConfig } : {}) }; +} + +/** + * The error an agent gets instead of a prompt. + * + * Names exactly the flags that are missing, so the caller can retry without + * guessing which of the two it forgot. + */ +export function throwAgentFlagsRequired(missing: { transformer: boolean; file: boolean }): never { + const flags = [ + missing.transformer ? "--transformer " : undefined, + missing.file ? "--file " : undefined, + ].filter(Boolean); + + throwUsageError( + `\`clerk migrate\` is interactive and cannot prompt in agent mode. Pass ${flags.join(" and ")}.`, + undefined, + undefined, + [ + { + command: `clerk migrate run -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, + description: "Run non-interactively", + }, + ], + ); +} From f32b73f598f510fa362ece6fdb757f274da19346 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 12:47:58 -0400 Subject: [PATCH 02/34] test(migrate): support multiselect prompt stubs --- packages/cli-core/src/test/integration/lib/harness.ts | 8 +++++++- packages/cli-core/src/test/lib/stubs.ts | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/test/integration/lib/harness.ts b/packages/cli-core/src/test/integration/lib/harness.ts index 6d80b75cf..2ad58c8a7 100644 --- a/packages/cli-core/src/test/integration/lib/harness.ts +++ b/packages/cli-core/src/test/integration/lib/harness.ts @@ -110,7 +110,7 @@ mock.module( // ── Prompt queue (drives lib/prompts.ts and lib/listage.ts mocks) ──────────── -type PromptType = "select" | "search" | "input" | "confirm" | "password" | "editor"; +type PromptType = "select" | "search" | "input" | "confirm" | "password" | "editor" | "multiselect"; const promptQueues: Record = { select: [], @@ -119,6 +119,7 @@ const promptQueues: Record = { confirm: [], password: [], editor: [], + multiselect: [], }; function dequeuePrompt(name: PromptType) { @@ -159,6 +160,7 @@ export const mockPrompts = { input: (...responses: string[]) => promptQueues.input.push(...responses), password: (...responses: string[]) => promptQueues.password.push(...responses), editor: (...responses: string[]) => promptQueues.editor.push(...responses), + multiselect: (...responses: unknown[][]) => promptQueues.multiselect.push(...responses), }; function resetPromptQueues() { @@ -198,8 +200,12 @@ mock.module("../../../lib/listage.ts", () => ({ }, })); +// Every export of the real module must appear here: a missing one is a module +// link error at import time, not a failed prompt, so it takes down every test +// in the file the moment any command imports it. mock.module("../../../lib/prompts.ts", () => ({ confirm: dequeuePrompt("confirm"), + multiselect: dequeuePrompt("multiselect"), text: dequeuePrompt("input"), password: dequeuePrompt("password"), editor: dequeuePrompt("editor"), diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index 70598ff99..efc4e5301 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -213,9 +213,14 @@ export const gitStubs = { * Stubs for `lib/prompts.ts` — the @clack/prompts-backed wrapper. Default * responses return benign values so tests can mock the module without * configuring each prompt explicitly. + * + * Must cover every export of the real module: an omission is a module link + * error at import time, which takes down the whole test file rather than + * failing one prompt. */ export const libPromptsStubs = { confirm: async () => true, + multiselect: async () => [], text: async () => "", password: async () => "", editor: async () => "{}", From 9a56d9fb4e7d251508428bfce2c7cf56f2d3a86d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 12:52:05 -0400 Subject: [PATCH 03/34] docs(migrate): mention migration command --- CLAUDE.md | 4 +++- README.md | 1 + scripts/check-bun-version.ts | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index bd4118cba..49f96e04a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,7 @@ Default to using Bun instead of Node.js. - `Bun.serve()` supports WebSockets, HTTPS, and routes. Don't use `express`. - `bun:sqlite` for SQLite. Don't use `better-sqlite3`. - `Bun.redis` for Redis. Don't use `ioredis`. -- `Bun.sql` for Postgres. Don't use `pg` or `postgres.js`. +- `Bun.sql` for Postgres and MySQL. Don't use `pg`, `postgres.js`, or `mysql2`. - `WebSocket` is built-in. Don't use `ws`. - Prefer `Bun.file` over `node:fs`'s readFile/writeFile - Bun.$`ls` instead of execa. @@ -56,6 +56,8 @@ When running multiple test files directly with `bun test`, always pass `--isolat These flags require Bun >= 1.3.13 — older versions silently ignore them and lose isolation. `bun run test` and `bun run test:e2e` run `scripts/check-bun-version.ts` first, which fails fast when the installed Bun is older than the `engines.bun` floor in package.json. +The same floor also covers `Bun.sql`'s MySQL adapter used by the DB-backed export commands: MySQL support landed in Bun 1.2.21, but binary columns (password hashes) only decoded correctly from 1.3.6. See the header of `scripts/check-bun-version.ts`. + ## Versioning The `CLI_VERSION` global is injected at compile time via `bun build --compile --define "CLI_VERSION=..."`. Local `build:compile` omits it, so the binary reports `0.0.0-dev`. The CI release workflow injects the real version. diff --git a/README.md b/README.md index ab88d248d..0454faada 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Commands: update [options] Update the Clerk CLI to the latest version deploy Deploy a Clerk application to production webhooks Stream webhook events to a local handler and verify their signatures + migrate Migrate users into Clerk from another auth provider help [command] Display help for command bird Play Clerk Bird, a Flappy Bird game in your terminal ``` diff --git a/scripts/check-bun-version.ts b/scripts/check-bun-version.ts index a05b2d4b7..74f576892 100644 --- a/scripts/check-bun-version.ts +++ b/scripts/check-bun-version.ts @@ -9,6 +9,13 @@ * producing hundreds of order-dependent failures. Bun does not enforce * `engines.bun` at install time, so this preflight fails loudly instead. * + * A second, lower constraint rides along: the DB-backed `clerk migrate export` + * commands read MySQL through `Bun.sql` rather than `mysql2`. Verified against + * MySQL 8.4 -- the adapter landed in Bun 1.2.21, but VARBINARY/BLOB columns + * came back as lossily decoded strings until 1.3.6, which would silently + * corrupt exported password hashes. The 1.3.13 floor above already covers it; + * do not drop below 1.3.6 if the `--parallel` requirement ever goes away. + * * Usage: * bun run scripts/check-bun-version.ts */ From 080eb66cda435731c2178b3033523e48ad199907 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 12:52:53 -0400 Subject: [PATCH 04/34] docs(changeset): add migrate changeset --- .changeset/migrate-cli.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/migrate-cli.md diff --git a/.changeset/migrate-cli.md b/.changeset/migrate-cli.md new file mode 100644 index 000000000..42864a475 --- /dev/null +++ b/.changeset/migrate-cli.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Add `clerk migrate` for importing users, exporting from supported auth providers, reviewing migration logs, undoing a migration, and extending imports with custom transformers. From d94307e2dd0853886c4347b92a524c08c58a188f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 15:52:29 -0400 Subject: [PATCH 05/34] refactor(migrate): keep migration state in the CLI config, not a cwd `.settings` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clerk migrate run` wrote a `.settings` file into the working directory to remember what it last imported. The CLI cannot gitignore that on the user's behalf, so it lands inside the repository being migrated — and it carried the Firebase signer key, a secret, as plaintext JSON. That state now lives in the `migrations` section of the CLI's own config file, keyed by project through `getProjectKey()` (linked profile, then git remote, then directory). This is the shape `clerk webhooks listen` already uses for its relay token, so `migrations` sits beside `relay` with the same accessor pair. The Firebase hash parameters are dropped from persistence rather than moved: remembering a secret writes it to disk wherever the file lives. They now fall back to `CLERK_FIREBASE_SIGNER_KEY`, `CLERK_FIREBASE_SALT_SEPARATOR`, `CLERK_FIREBASE_ROUNDS` and `CLERK_FIREBASE_MEM_COST`, so a repeat run still need not re-type four flags, and `.env.local` is already gitignored. No migration path for existing `.settings` files: `clerk migrate` is unreleased, so nothing in the wild has one. --- .../cli-core/src/commands/migrate/README.md | 36 ++++++-- .../src/commands/migrate/delete.test.ts | 40 +++++---- .../cli-core/src/commands/migrate/delete.ts | 21 ++--- .../src/commands/migrate/lib/settings.test.ts | 60 ++++++++++--- .../src/commands/migrate/lib/settings.ts | 56 ++++++------ .../commands/migrate/run-interactive.test.ts | 14 ++- .../cli-core/src/commands/migrate/run.test.ts | 87 ++++++++++++++----- packages/cli-core/src/commands/migrate/run.ts | 53 +++++++---- .../commands/migrate/transformers/firebase.ts | 3 +- .../cli-core/src/commands/migrate/types.ts | 14 --- .../src/commands/migrate/wizard.test.ts | 30 +++---- .../cli-core/src/commands/migrate/wizard.ts | 40 ++++----- packages/cli-core/src/lib/config.test.ts | 39 +++++++++ packages/cli-core/src/lib/config.ts | 43 ++++++++- packages/cli-core/src/test/lib/stubs.ts | 3 + 15 files changed, 362 insertions(+), 177 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index f18e38223..17eecf60b 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -35,9 +35,10 @@ clerk migrate ``` It picks the transformer from a list built off the registry, asks for the file, -collects Firebase's hash parameters when they are needed, and pre-fills every -answer from the last run's `.settings` so a repeat migration is mostly pressing -enter. Anything already passed as a flag is not asked for. +collects Firebase's hash parameters when they are needed, and pre-fills the +platform and file from the last run so a repeat migration is mostly pressing +enter. Anything already passed as a flag is not asked for. Firebase's hash +parameters are never pre-filled — see [below](#--firebase--firebase). Then it prints the [Migration Readiness report](#migration-readiness-report) and waits for confirmation. Declining writes nothing to Clerk. @@ -312,8 +313,8 @@ destroys data **in Clerk**, and is worth keeping short and prominent. (Contrast #### What it will and will not touch -`.settings` is the only record of what a run created, so that is what -identifies the migration being undone. Without it the command fails and +The saved migration record is the only account of what a run created, so that +is what identifies the migration being undone. Without it the command fails and explains — deleting nothing silently would look like a successful undo. Users are found with `GET /v1/users?external_id=…`, 100 IDs per request. Only a @@ -525,7 +526,21 @@ clerk migrate run -y -t firebase -f users.json \ All four are **required as a set** — supplying some but not all is a usage error naming what is missing. A partial set produces a well-formed digest that verifies against nothing, so users would import successfully and then be unable -to sign in. They are saved to `.settings` and reused on the next run. +to sign in. + +They are **never saved**: the signer key is a Firebase secret, and remembering +it would mean writing it to disk in plaintext. To avoid re-passing all four on +every run, set them in the environment (`.env.local` is already gitignored): + +| Variable | Flag | +| ------------------------------- | --------------------------- | +| `CLERK_FIREBASE_SIGNER_KEY` | `--firebase-signer-key` | +| `CLERK_FIREBASE_SALT_SEPARATOR` | `--firebase-salt-separator` | +| `CLERK_FIREBASE_ROUNDS` | `--firebase-rounds` | +| `CLERK_FIREBASE_MEM_COST` | `--firebase-mem-cost` | + +Flags win over the environment, and the two can be mixed as long as all four +end up supplied. An export with no password hashes needs no parameters at all. @@ -666,10 +681,13 @@ rather than "which project is linked here". | `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | | `./logs/export-.log` | NDJSON: one line per exported user | | `./exports/-export.json` | The export itself, unless `--output` says otherwise | -| `./.settings` | The transformer key and file path of the last run | -`.settings` is what `migrate delete` reads to know which migration to undo, so -it is load-bearing rather than a convenience. +The transformer and file of the last run are **not** written here. They go to +the `migrations` section of the CLI's own config file (`clerk config --help` +names its location), keyed by project the same way a linked profile is. That is +what `migrate delete` reads to know which migration to undo, so it is +load-bearing rather than a convenience — and it has no business being written +into the repository being migrated. Log writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. Use the last diff --git a/packages/cli-core/src/commands/migrate/delete.test.ts b/packages/cli-core/src/commands/migrate/delete.test.ts index 5336e2312..ef110ed00 100644 --- a/packages/cli-core/src/commands/migrate/delete.test.ts +++ b/packages/cli-core/src/commands/migrate/delete.test.ts @@ -2,6 +2,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } fr import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { _setConfigDir } from "../../lib/config.ts"; import { CliError } from "../../lib/errors.ts"; import { useCaptureLog } from "../../test/lib/stubs.ts"; import { @@ -22,6 +23,7 @@ const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurr const DATE_TIME = "2026-01-01T00:00:00"; let workDir: string; +let configDir: string; let originalCwd: string; let originalFetch: typeof globalThis.fetch; let requests: { method: string; url: string }[]; @@ -35,19 +37,23 @@ beforeAll(() => { originalCwd = process.cwd(); originalFetch = globalThis.fetch; workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-delete-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-delete-config-")); + _setConfigDir(configDir); process.chdir(workDir); }); afterAll(() => { globalThis.fetch = originalFetch; + _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); }); beforeEach(() => { requests = []; fs.rmSync(getLogDir(), { recursive: true, force: true }); - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); }); @@ -91,27 +97,27 @@ const logEntries = () => const deleteCalls = () => requests.filter((r) => r.method === "DELETE").map((r) => r.url); describe("resolveMigrationToUndo", () => { - test("reads the file and transformer from .settings", () => { - saveSettings({ key: "clerk", file: "export.json" }); - expect(resolveMigrationToUndo()).toEqual({ file: "export.json", key: "clerk" }); + test("reads the file and transformer from the saved migration", async () => { + await saveSettings({ transformer: "clerk", file: "export.json" }); + expect(await resolveMigrationToUndo()).toEqual({ file: "export.json", key: "clerk" }); }); // Deleting nothing silently would look like a successful undo. - test("explains when there is no .settings at all", () => { - expect(() => resolveMigrationToUndo()).toThrow(/no `.settings` from a previous/); + test("explains when there is no saved migration at all", async () => { + await expect(resolveMigrationToUndo()).rejects.toThrow(/no record of a previous/); }); test.each([ - ["no file", { key: "clerk" }], + ["no file", { transformer: "clerk" }], ["no transformer", { file: "export.json" }], - ])("explains when .settings has %s", (_label, settings) => { - saveSettings(settings); - expect(() => resolveMigrationToUndo()).toThrow(CliError); + ])("explains when the saved migration has %s", async (_label, settings) => { + await saveSettings(settings); + await expect(resolveMigrationToUndo()).rejects.toThrow(CliError); }); - test("explains when the migration file has since been removed", () => { - saveSettings({ key: "clerk", file: "gone.json" }); - expect(() => resolveMigrationToUndo()).toThrow(/no longer there/); + test("explains when the migration file has since been removed", async () => { + await saveSettings({ transformer: "clerk", file: "gone.json" }); + await expect(resolveMigrationToUndo()).rejects.toThrow(/no longer there/); }); }); @@ -344,8 +350,8 @@ describe("deleteMigratedUsers", () => { describe("deleteMigration", () => { const baseOptions = { yes: true, secretKey: "sk_test_x" }; - beforeEach(() => { - saveSettings({ key: "clerk", file: "export.json" }); + beforeEach(async () => { + await saveSettings({ transformer: "clerk", file: "export.json" }); }); test("deletes the users the last run created", async () => { @@ -398,8 +404,8 @@ describe("deleteMigration", () => { expect(deleteCalls()).toHaveLength(0); }); - test("fails before any API call when there is no .settings", async () => { - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + test("fails before any API call when there is no saved migration", async () => { + fs.rmSync(path.join(configDir, "config.json"), { force: true }); stubBapi({ legacy_a: "user_1" }); await expect(deleteMigration(baseOptions)).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 5cc78dbef..8b14abe55 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -64,28 +64,29 @@ export type MigratedUser = { /** * Resolves which migration is being undone. * - * `.settings` is the only record of that — this command has no independent way - * to know what a previous run created, which is why it is coupled to `run`. + * The saved migration record is the only account of that — this command has no + * independent way to know what a previous run created, which is why it is + * coupled to `run`. */ -export function resolveMigrationToUndo(): { file: string; key: string } { - const settings = loadSettings(); +export async function resolveMigrationToUndo(): Promise<{ file: string; key: string }> { + const settings = await loadSettings(); - if (!settings.file || !settings.key) { + if (!settings.file || !settings.transformer) { throw new CliError( - "No migration to undo: this directory has no `.settings` from a previous `clerk migrate run`.\n" + - "Run `clerk migrate delete` from the directory you migrated from.", + "No migration to undo: this project has no record of a previous `clerk migrate run`.\n" + + "Run `clerk migrate delete` from the project you migrated from.", { code: ERROR_CODE.FILE_NOT_FOUND }, ); } if (!fileExists(settings.file)) { throw new CliError( - `The migration file ${settings.file} named in .settings is no longer there, so the users it created cannot be identified.`, + `The migration file ${settings.file} is no longer there, so the users it created cannot be identified.`, { code: ERROR_CODE.FILE_NOT_FOUND }, ); } - return { file: settings.file, key: settings.key }; + return { file: settings.file, key: settings.transformer }; } /** @@ -261,7 +262,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); diff --git a/packages/cli-core/src/commands/migrate/lib/settings.test.ts b/packages/cli-core/src/commands/migrate/lib/settings.test.ts index 579d508df..7ad3cf9ed 100644 --- a/packages/cli-core/src/commands/migrate/lib/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/settings.test.ts @@ -1,15 +1,19 @@ -import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { afterAll, afterEach, beforeAll, beforeEach, expect, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { _setConfigDir, getMigrationEntry, getProjectKey } from "../../../lib/config.ts"; import { loadSettings, saveSettings } from "./settings.ts"; let workDir: string; +let configDir: string; let originalCwd: string; beforeAll(() => { originalCwd = process.cwd(); - workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-")); + // Realpath'd because the project key is derived from `process.cwd()`, which + // resolves the /var → /private/var symlink macOS puts in front of tmpdir. + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-"))); process.chdir(workDir); }); @@ -19,24 +23,52 @@ afterAll(() => { }); beforeEach(() => { - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-config-")); + _setConfigDir(configDir); }); -test("returns empty settings when the file is absent", () => { - expect(loadSettings()).toEqual({}); +afterEach(() => { + _setConfigDir(undefined); + fs.rmSync(configDir, { recursive: true, force: true }); }); -test("round-trips the transformer key and file path", () => { - saveSettings({ key: "clerk", file: "users.json" }); - expect(loadSettings()).toEqual({ key: "clerk", file: "users.json" }); +test("returns empty settings when nothing was saved", async () => { + expect(await loadSettings()).toEqual({}); }); -test("writes to the current working directory", () => { - saveSettings({ key: "clerk" }); - expect(fs.existsSync(path.join(workDir, ".settings"))).toBe(true); +test("round-trips the transformer key and file path", async () => { + await saveSettings({ transformer: "clerk", file: "users.json" }); + expect(await loadSettings()).toEqual({ transformer: "clerk", file: "users.json" }); }); -test("treats a corrupt settings file as empty rather than failing the run", () => { - fs.writeFileSync(path.join(workDir, ".settings"), "{not json"); - expect(loadSettings()).toEqual({}); +test("writes to the CLI config file, not the working directory", async () => { + await saveSettings({ transformer: "clerk" }); + + expect(fs.existsSync(path.join(workDir, ".settings"))).toBe(false); + const config = JSON.parse(fs.readFileSync(path.join(configDir, "config.json"), "utf-8")); + expect(config.migrations).toEqual({ [await getProjectKey(workDir)]: { transformer: "clerk" } }); +}); + +test("keys the record by project, so another directory does not see it", async () => { + await saveSettings({ transformer: "clerk", file: "users.json" }); + + const elsewhere = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-other-"))); + try { + expect(await getMigrationEntry(await getProjectKey(elsewhere))).toBeUndefined(); + } finally { + fs.rmSync(elsewhere, { recursive: true, force: true }); + } +}); + +test("treats a corrupt config file as empty rather than failing the run", async () => { + fs.writeFileSync(path.join(configDir, "config.json"), "{not json"); + expect(await loadSettings()).toEqual({}); +}); + +test("leaves the run standing when the config cannot be written", async () => { + fs.rmSync(configDir, { recursive: true, force: true }); + fs.writeFileSync(configDir, "not a directory"); + + await saveSettings({ transformer: "clerk" }); + expect(await loadSettings()).toEqual({}); }); diff --git a/packages/cli-core/src/commands/migrate/lib/settings.ts b/packages/cli-core/src/commands/migrate/lib/settings.ts index fc9dfcd7e..958bf1cd2 100644 --- a/packages/cli-core/src/commands/migrate/lib/settings.ts +++ b/packages/cli-core/src/commands/migrate/lib/settings.ts @@ -1,43 +1,39 @@ /** - * The cwd-relative `.settings` file: what this directory last migrated, and - * with which transformer. + * What this project last migrated, and with which transformer. * - * Ported from the standalone migration-tool's `src/lib/settings.ts`. Kept out - * of `~/.config/clerk/config.json` on purpose — that file is keyed by linked - * project identity, not by "which export file am I working through". + * Kept in the CLI's own config file under `migrations`, keyed by project — the + * same shape `clerk webhooks listen` files its relay token under. An earlier + * version wrote a `.settings` file into the user's cwd instead, which the CLI + * cannot gitignore on the user's behalf and which put migration state inside + * the repository being migrated. * - * Both halves fail silently: a missing, unreadable or unwritable `.settings` - * only costs the user a remembered default. + * Both halves fail silently: an unreadable or unwritable config only costs the + * user a remembered default, so it must not take the run down with it. */ -import fs from "node:fs"; -import path from "node:path"; -import type { Settings } from "../types.ts"; +import { + getMigrationEntry, + getProjectKey, + setMigrationEntry, + type MigrationEntry, +} from "../../../lib/config.ts"; +import { log } from "../../../lib/log.ts"; -const SETTINGS_FILE = ".settings"; - -function settingsPath(): string { - return path.join(process.cwd(), SETTINGS_FILE); -} - -/** Reads saved settings, or `{}` when absent or corrupt. */ -export function loadSettings(): Settings { +/** Reads saved settings, or `{}` when absent or unreadable. */ +export async function loadSettings(): Promise { try { - const file = settingsPath(); - if (fs.existsSync(file)) { - return JSON.parse(fs.readFileSync(file, "utf-8")) as Settings; - } - } catch { - // Corrupt or unreadable settings are indistinguishable from none. + return (await getMigrationEntry(await getProjectKey(process.cwd()))) ?? {}; + } catch (error) { + log.debug(`config: could not read migration settings — ${error}`); + return {}; } - return {}; } -/** Persists settings for the next run in this directory. */ -export function saveSettings(settings: Settings): void { +/** Persists settings for the next run in this project. */ +export async function saveSettings(settings: MigrationEntry): Promise { try { - fs.writeFileSync(settingsPath(), JSON.stringify(settings, null, 2)); - } catch { - // Read-only cwd; the run itself is unaffected. + await setMigrationEntry(await getProjectKey(process.cwd()), settings); + } catch (error) { + log.debug(`config: could not save migration settings — ${error}`); } } diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index ef518506c..c5355da20 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -40,10 +40,12 @@ const { run } = await import("./run.ts"); const { deleteMigration } = await import("./delete.ts"); const { UserAbortError } = await import("../../lib/errors.ts"); const { loadSettings, saveSettings } = await import("./lib/settings.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); const captured = useCaptureLog(); let workDir: string; +let configDir: string; let originalCwd: string; let originalFetch: typeof globalThis.fetch; let requests: { method: string; url: string; body: unknown }[]; @@ -61,14 +63,18 @@ beforeAll(() => { originalCwd = process.cwd(); originalFetch = globalThis.fetch; workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-config-")); + _setConfigDir(configDir); process.chdir(workDir); }); afterAll(() => { setMode(originalMode); globalThis.fetch = originalFetch; + _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); }); beforeEach(() => { @@ -79,7 +85,7 @@ beforeEach(() => { mockSelect.mockResolvedValue("clerk"); mockText.mockResolvedValue("export.json"); fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); }); @@ -137,7 +143,7 @@ describe("the wizard fills in missing flags", () => { test("records the wizard's answers for the next run", async () => { await run({ secretKey: "sk_test_x" }); - expect(loadSettings()).toMatchObject({ key: "clerk", file: "export.json" }); + expect(await loadSettings()).toMatchObject({ transformer: "clerk", file: "export.json" }); }); }); @@ -261,8 +267,8 @@ describe("migrate delete confirmation", () => { const deleted = () => requests.filter((r) => r.method === "DELETE"); - beforeEach(() => { - saveSettings({ key: "clerk", file: "export.json" }); + beforeEach(async () => { + await saveSettings({ transformer: "clerk", file: "export.json" }); stubDeleteTargets({ legacy_a: "user_1", legacy_b: "user_2" }); fs.writeFileSync( path.join(workDir, "export.json"), diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index e5408e422..ff9ffbbae 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2,15 +2,17 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } fr import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { _setConfigDir } from "../../lib/config.ts"; import { CliError } from "../../lib/errors.ts"; import { useCaptureLog } from "../../test/lib/stubs.ts"; import { getLogDir } from "./lib/logger.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; import { loadSettings } from "./lib/settings.ts"; import { applyResumeAfter, resolveFirebaseHashConfig, run, validateRunOptions } from "./run.ts"; -import type { FirebaseHashConfig, User } from "./types.ts"; +import type { User } from "./types.ts"; let workDir: string; +let configDir: string; let originalCwd: string; const users = (...ids: string[]): User[] => ids.map((userId) => ({ userId }) as User); @@ -18,14 +20,18 @@ const users = (...ids: string[]): User[] => ids.map((userId) => ({ userId }) as beforeAll(() => { originalCwd = process.cwd(); workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-config-")); + _setConfigDir(configDir); process.chdir(workDir); fs.writeFileSync(path.join(workDir, "users.json"), "[]"); fs.writeFileSync(path.join(workDir, "users.txt"), ""); }); afterAll(() => { + _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); }); describe("validateRunOptions", () => { @@ -91,27 +97,62 @@ describe("resolveFirebaseHashConfig", () => { ); }); - test("falls back to saved settings when no flag is passed", () => { - const saved: FirebaseHashConfig = { - base64_signer_key: "S", - base64_salt_separator: "B", - rounds: 8, - mem_cost: 14, + describe("environment fallback", () => { + const ENV = { + CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER", + CLERK_FIREBASE_SALT_SEPARATOR: "Bw==", + CLERK_FIREBASE_ROUNDS: "8", + CLERK_FIREBASE_MEM_COST: "14", }; - expect(resolveFirebaseHashConfig({}, saved)).toEqual(saved); - }); - test("prefers flags over saved settings", () => { - const saved: FirebaseHashConfig = { - base64_signer_key: "OLD", - base64_salt_separator: "B", - rounds: 1, - mem_cost: 1, - }; - expect(resolveFirebaseHashConfig(ALL, saved)?.base64_signer_key).toBe("SIGNER"); + afterEach(() => { + for (const name of Object.keys(ENV)) delete process.env[name]; + }); + + const setEnv = (vars: Partial) => Object.assign(process.env, vars); + + test("builds the config when no flag is passed", () => { + setEnv(ENV); + expect(resolveFirebaseHashConfig({})).toEqual({ + base64_signer_key: "ENV_SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + test("prefers a flag over the environment", () => { + setEnv(ENV); + expect(resolveFirebaseHashConfig(ALL)?.base64_signer_key).toBe("SIGNER"); + }); + + // Half from the environment and half from flags is still a complete set. + test("fills only the gaps the flags left", () => { + setEnv({ CLERK_FIREBASE_ROUNDS: "8", CLERK_FIREBASE_MEM_COST: "14" }); + expect( + resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER", firebaseSaltSeparator: "Bw==" }), + ).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + test("still demands the full set when the environment supplies only part", () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); + expect(() => resolveFirebaseHashConfig({})).toThrow(/--firebase-salt-separator/); + }); + + // An empty var is how a shell spells "unset", and treating it as set would + // demand the other three for a config nobody asked for. + test("ignores an empty variable", () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "" }); + expect(resolveFirebaseHashConfig({})).toBeUndefined(); + }); }); - test("returns nothing when neither flags nor settings supply a config", () => { + test("returns nothing when neither flags nor the environment supply a config", () => { expect(resolveFirebaseHashConfig({})).toBeUndefined(); }); }); @@ -157,7 +198,7 @@ describe("run", () => { requests = []; delete process.env.CLERK_MIGRATE_RATE_LIMIT; fs.rmSync(getLogDir(), { recursive: true, force: true }); - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { requests.push({ @@ -209,9 +250,9 @@ describe("run", () => { expect(entries.filter((e) => e.status === "success")).toHaveLength(2); }); - test("records the run's key and file in .settings", async () => { + test("records the run's transformer and file for the next run", async () => { await run(baseOptions); - expect(loadSettings()).toEqual({ key: "clerk", file: "export.json" }); + expect(await loadSettings()).toEqual({ transformer: "clerk", file: "export.json" }); }); test("--require-password imports only the users that have one", async () => { @@ -738,12 +779,12 @@ describe("run", () => { expect(captured.err).toContain("only applies to supabase"); }); - test("records the flag in .settings", async () => { + test("records the flag for the next run", async () => { stubInstance({ oauth_discord: { enabled: true } }); await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); - expect(loadSettings().skipUnsupportedProviders).toBe(true); + expect((await loadSettings()).skipUnsupportedProviders).toBe(true); }); }); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 0d518fb3c..cd1e1934f 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -27,7 +27,7 @@ import { import { buildReadinessReport, formatReadinessReport } from "./lib/readiness.ts"; import { DEV_USER_LIMIT, resolveLimits } from "./lib/instance.ts"; import { getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; -import { loadSettings, saveSettings } from "./lib/settings.ts"; +import { saveSettings } from "./lib/settings.ts"; import { countSocialProviders, findDisabledProviders, @@ -62,15 +62,38 @@ export type MigrateRunOptions = { }; const FIREBASE_FLAGS = [ - ["firebaseSignerKey", "--firebase-signer-key"], - ["firebaseSaltSeparator", "--firebase-salt-separator"], - ["firebaseRounds", "--firebase-rounds"], - ["firebaseMemCost", "--firebase-mem-cost"], + ["firebaseSignerKey", "--firebase-signer-key", "CLERK_FIREBASE_SIGNER_KEY"], + ["firebaseSaltSeparator", "--firebase-salt-separator", "CLERK_FIREBASE_SALT_SEPARATOR"], + ["firebaseRounds", "--firebase-rounds", "CLERK_FIREBASE_ROUNDS"], + ["firebaseMemCost", "--firebase-mem-cost", "CLERK_FIREBASE_MEM_COST"], ] as const; +const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); + /** - * Resolves Firebase's four hash parameters from flags, falling back to - * `.settings` when none were passed. + * Overlays the `CLERK_FIREBASE_*` environment variables onto whichever flags + * were not passed. + * + * The signer key is a Firebase secret, so it is read rather than stored: the + * CLI never persists these, and `.env.local` is already gitignored and already + * where the CLI keeps a project's local secrets. + */ +function withFirebaseEnv(options: MigrateRunOptions): MigrateRunOptions { + const merged = { ...options }; + for (const [key, , envVar] of FIREBASE_FLAGS) { + if (merged[key] !== undefined) continue; + const value = process.env[envVar]; + if (value === undefined || value.trim() === "") continue; + // A non-numeric round count is left to fail the flag's own validation + // rather than silently becoming NaN. + (merged as Record)[key] = FIREBASE_NUMERIC.has(key) ? Number(value) : value; + } + return merged; +} + +/** + * Resolves Firebase's four hash parameters from flags, falling back to the + * `CLERK_FIREBASE_*` environment variables. * * The four are required as a set: a digest built from a partial set is * well-formed but verifies against nothing, so every migrated user would fail @@ -80,12 +103,12 @@ const FIREBASE_FLAGS = [ * for an export that carries no password hashes. */ export function resolveFirebaseHashConfig( - options: MigrateRunOptions, - saved?: FirebaseHashConfig, + rawOptions: MigrateRunOptions, ): FirebaseHashConfig | undefined { + const options = withFirebaseEnv(rawOptions); const provided = FIREBASE_FLAGS.filter(([key]) => options[key] !== undefined); - if (provided.length === 0) return saved; + if (provided.length === 0) return undefined; if (provided.length < FIREBASE_FLAGS.length) { const missing = FIREBASE_FLAGS.filter(([key]) => options[key] === undefined).map( @@ -415,8 +438,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const secretKeyOption = options.secretKey ?? options.clerkSecretKey; const { transformer, file } = validateRunOptions(options); - const saved = loadSettings(); - const firebaseHashConfig = resolveFirebaseHashConfig(options, saved.firebaseHashConfig); + const firebaseHashConfig = resolveFirebaseHashConfig(options); await withGutter("Migrating users to Clerk", async () => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); @@ -490,11 +512,12 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (!proceed) throwUserAbort(); } - saveSettings({ - key: transformer, + // The Firebase hash parameters are deliberately not among these: the signer + // key is a secret, and remembering it would write it to disk in plaintext. + await saveSettings({ + transformer, file, ...(options.skipUnsupportedProviders ? { skipUnsupportedProviders: true } : {}), - ...(firebaseHashConfig ? { firebaseHashConfig } : {}), }); const summary = await withSpinner( diff --git a/packages/cli-core/src/commands/migrate/transformers/firebase.ts b/packages/cli-core/src/commands/migrate/transformers/firebase.ts index 73966c1b2..9c9d09c6e 100644 --- a/packages/cli-core/src/commands/migrate/transformers/firebase.ts +++ b/packages/cli-core/src/commands/migrate/transformers/firebase.ts @@ -25,7 +25,8 @@ const FIREBASE_CSV_HEADERS = * * Firebase's scrypt is a modified variant, so Clerk needs the project's four * hash parameters alongside each digest. They arrive on the run's - * {@link TransformContext} from `--firebase-*` flags or saved `.settings`. + * {@link TransformContext} from `--firebase-*` flags or the matching + * `CLERK_FIREBASE_*` environment variables; they are never persisted. * * See https://clerk.com/docs/guides/development/migrating/firebase */ diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts index 40ba3a5ec..904267037 100644 --- a/packages/cli-core/src/commands/migrate/types.ts +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -117,20 +117,6 @@ export type ImportSummary = { errorBreakdown: Map; }; -/** - * Per-directory migration state, persisted to a cwd-relative `.settings` file. - * - * Deliberately not routed through `~/.config/clerk/config.json`: that file is - * keyed by linked-project identity, which is a different concept from "which - * file did I last migrate with". - */ -export type Settings = { - key?: string; - file?: string; - skipUnsupportedProviders?: boolean; - firebaseHashConfig?: FirebaseHashConfig; -}; - /** * Firebase's scrypt parameters, needed to rebuild a password hash Clerk can * verify. diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts index f3633665d..e913a0064 100644 --- a/packages/cli-core/src/commands/migrate/wizard.test.ts +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -28,27 +28,33 @@ mock.module("../../lib/prompts.ts", () => ({ const { runWizard, throwAgentFlagsRequired } = await import("./wizard.ts"); const { saveSettings } = await import("./lib/settings.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); let workDir: string; +let configDir: string; let originalCwd: string; beforeAll(() => { originalCwd = process.cwd(); workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-config-")); + _setConfigDir(configDir); process.chdir(workDir); fs.writeFileSync(path.join(workDir, "users.json"), "[]"); fs.writeFileSync(path.join(workDir, "other.csv"), ""); }); afterAll(() => { + _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); }); beforeEach(() => { mockSelect.mockReset(); mockText.mockReset(); - fs.rmSync(path.join(workDir, ".settings"), { force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); }); /** The config object the wizard passed to its Nth `text`/`select` prompt. */ @@ -93,7 +99,7 @@ describe("transformer picker", () => { describe("defaults from the previous run", () => { test("pre-selects the last transformer and pre-fills the last file", async () => { - saveSettings({ key: "supabase", file: "other.csv" }); + await saveSettings({ transformer: "supabase", file: "other.csv" }); mockSelect.mockResolvedValue("supabase"); mockText.mockResolvedValue("other.csv"); @@ -116,7 +122,7 @@ describe("defaults from the previous run", () => { // A saved key from a build that has since dropped that transformer would // otherwise pre-select a value the picker cannot offer. test("ignores a saved transformer that is no longer registered", async () => { - saveSettings({ key: "okta" }); + await saveSettings({ transformer: "okta" }); mockSelect.mockResolvedValue("clerk"); mockText.mockResolvedValue("users.json"); @@ -187,27 +193,21 @@ describe("firebase hash parameters", () => { expect(mockText).toHaveBeenCalledTimes(2); }); - test("are pre-filled from the previous run", async () => { - saveSettings({ - firebaseHashConfig: { - base64_signer_key: "SAVED", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }, - }); + // The signer key is a Firebase secret, so it is never written to disk and so + // there is nothing to offer back. A repeat run passes it as a flag or env var. + test("are never pre-filled, because they are not saved", async () => { mockSelect.mockResolvedValue("firebase"); mockText .mockResolvedValueOnce("users.json") - .mockResolvedValueOnce("SAVED") + .mockResolvedValueOnce("SIGNER") .mockResolvedValueOnce("Bw==") .mockResolvedValueOnce("8") .mockResolvedValueOnce("14"); await runWizard({}); - expect(textCall(1)?.default).toBe("SAVED"); - expect(textCall(3)?.default).toBe("8"); + expect(textCall(1)?.default).toBeUndefined(); + expect(textCall(3)?.default).toBeUndefined(); }); test("are not asked for on a non-firebase transformer", async () => { diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index 3082a1b63..d5b556edb 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -2,8 +2,9 @@ * The interactive path behind a bare `clerk migrate`. * * Ported from the standalone migration-tool's `src/migrate/cli.ts` interactive - * flow. Every answer is pre-filled from the previous run's `.settings`, so a - * repeat migration is mostly pressing enter. + * flow. The platform and file are pre-filled from the previous run, so a repeat + * migration is mostly pressing enter. Firebase's hash parameters are not: the + * signer key is a secret, and the CLI does not keep those. * * Agent mode never reaches here — `run` raises a usage error naming the flags * instead, because an agent cannot answer a prompt. @@ -65,18 +66,17 @@ async function askFile(defaultFile: string | undefined): Promise { * nothing. Pressing enter through all four leaves the config unset, which is * correct for an export with no password hashes. */ -async function askFirebaseHashConfig( - saved: FirebaseHashConfig | undefined, -): Promise { +async function askFirebaseHashConfig(): Promise { log.info( "Firebase password hashes need the project's hash parameters. Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", ); - log.info(dimIfSaved(saved)); + log.info( + "Set CLERK_FIREBASE_SIGNER_KEY, CLERK_FIREBASE_SALT_SEPARATOR, CLERK_FIREBASE_ROUNDS and CLERK_FIREBASE_MEM_COST to skip these prompts on the next run.", + ); const signerKey = ( await text({ message: "base64 signer key (leave blank if this export has no passwords)", - default: saved?.base64_signer_key, }) ).trim(); if (!signerKey) return undefined; @@ -84,32 +84,21 @@ async function askFirebaseHashConfig( const saltSeparator = ( await text({ message: "base64 salt separator", - default: saved?.base64_salt_separator, validate: (value) => (value?.trim() ? undefined : "Required alongside the signer key"), }) ).trim(); - const rounds = await askNumber("rounds", saved?.rounds); - const memCost = await askNumber("mem cost", saved?.mem_cost); - return { base64_signer_key: signerKey, base64_salt_separator: saltSeparator, - rounds, - mem_cost: memCost, + rounds: await askNumber("rounds"), + mem_cost: await askNumber("mem cost"), }; } -function dimIfSaved(saved: FirebaseHashConfig | undefined): string { - return saved - ? "Saved parameters found — press enter to reuse them." - : "Leave the signer key blank if this export carries no passwords."; -} - -async function askNumber(label: string, defaultValue: number | undefined): Promise { +async function askNumber(label: string): Promise { const answer = await text({ message: label, - default: defaultValue === undefined ? undefined : String(defaultValue), validate: (value) => { const parsed = Number(value?.trim()); return Number.isInteger(parsed) && parsed > 0 ? undefined : "Enter a positive whole number"; @@ -128,14 +117,17 @@ export async function runWizard(provided: { file?: string; firebaseHashConfig?: FirebaseHashConfig; }): Promise { - const saved = loadSettings(); + const saved = await loadSettings(); - const transformer = provided.transformer ?? (await pickTransformer(saved.key)); + const transformer = provided.transformer ?? (await pickTransformer(saved.transformer)); const file = provided.file ?? (await askFile(saved.file)); let firebaseHashConfig = provided.firebaseHashConfig; if (transformer === "firebase" && !firebaseHashConfig) { - firebaseHashConfig = await askFirebaseHashConfig(saved.firebaseHashConfig); + // Never prefilled: the signer key is a secret the CLI does not keep. A + // repeat run supplies it through `--firebase-*` or `CLERK_FIREBASE_*`, + // which short-circuits this prompt entirely. + firebaseHashConfig = await askFirebaseHashConfig(); } return { transformer, file, ...(firebaseHashConfig ? { firebaseHashConfig } : {}) }; diff --git a/packages/cli-core/src/lib/config.test.ts b/packages/cli-core/src/lib/config.test.ts index c8deae64a..d69ed52fe 100644 --- a/packages/cli-core/src/lib/config.test.ts +++ b/packages/cli-core/src/lib/config.test.ts @@ -11,6 +11,9 @@ const { clearAuth, getProfile, setProfile, + getMigrationEntry, + setMigrationEntry, + getProjectKey, listProfiles, resolveProfile, resolveInstanceId, @@ -83,6 +86,42 @@ describe("config", () => { expect(await getAuth()).toBeUndefined(); }); + test("setMigrationEntry and getMigrationEntry", async () => { + expect(await getMigrationEntry("/projects/my-app")).toBeUndefined(); + await setMigrationEntry("/projects/my-app", { transformer: "clerk", file: "users.json" }); + expect(await getMigrationEntry("/projects/my-app")).toEqual({ + transformer: "clerk", + file: "users.json", + }); + expect(await getMigrationEntry("/projects/other")).toBeUndefined(); + }); + + // readConfig rebuilds the document field by field, so a key it does not know + // about is dropped on the next write rather than merely ignored. + // readConfig rebuilds the document field by field, so a section it does not + // know about is dropped on the next write rather than merely ignored. + test("migrations survive a write to another section", async () => { + await setMigrationEntry("/projects/my-app", { transformer: "clerk" }); + await setProfile("/projects/my-app", { + workspaceId: "org_abc", + appId: "app_def", + instances: { development: "ins_ghi" }, + }); + + expect(await getMigrationEntry("/projects/my-app")).toEqual({ transformer: "clerk" }); + }); + + test("getProjectKey prefers the linked profile's key over the directory", async () => { + expect(await getProjectKey("/projects/unlinked")).toBe("/projects/unlinked"); + + await setProfile("/projects/linked", { + workspaceId: "org_abc", + appId: "app_def", + instances: { development: "ins_ghi" }, + }); + expect(await getProjectKey("/projects/linked/src")).toBe("/projects/linked"); + }); + test("setProfile and getProfile", async () => { const profile = { workspaceId: "org_abc", diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index c0bccb29f..8139d9ca0 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -50,11 +50,19 @@ interface RelayEntry { token: string; } +/** What `clerk migrate run` last imported for a project, and how. */ +interface MigrationEntry { + transformer?: string; + file?: string; + skipUnsupportedProviders?: boolean; +} + interface ClerkConfig { environment?: string; auth?: Record; profiles: Record; relay?: Record; + migrations?: Record; } function defaultConfig(): ClerkConfig { @@ -86,6 +94,13 @@ function migrateRawConfig(raw: Record): ClerkConfig { config.relay = relay; } + // Not validated per entry the way `relay` is: every field is optional, so + // there is no key whose absence marks an entry as junk. A malformed one costs + // a remembered default, not a failed run. + if (raw.migrations && typeof raw.migrations === "object" && !Array.isArray(raw.migrations)) { + config.migrations = raw.migrations as Record; + } + if (raw.auth && typeof raw.auth === "object") { const auth = raw.auth as Record; if (typeof auth.userId === "string") { @@ -207,6 +222,18 @@ export async function setRelayEntry(key: string, entry: RelayEntry): Promise { + const config = await readConfig(); + return config.migrations?.[key]; +} + +export async function setMigrationEntry(key: string, entry: MigrationEntry): Promise { + const config = await readConfig(); + if (!config.migrations) config.migrations = {}; + config.migrations[key] = entry; + await writeConfig(config); +} + type ResolvedVia = "remote" | "git-common-dir" | "directory"; export async function resolveProfile(cwd: string): Promise< @@ -258,6 +285,20 @@ export async function resolveProfile(cwd: string): Promise< return undefined; } +/** + * The key a per-project record (e.g. `migrations`) is filed under. + * + * Prefers the linked profile's own key so the record sits beside the profile it + * belongs to, and so it survives `clerk link` being re-run from a subdirectory. + * Falls back to the git remote and then the directory, because a project that + * has never been linked still deserves to be remembered. + */ +export async function getProjectKey(cwd: string): Promise { + const resolved = await resolveProfile(cwd); + if (resolved) return resolved.path; + return (await getGitNormalizedRemote(cwd)) ?? cwd; +} + const INSTANCE_ALIASES: Record = { dev: "development", development: "development", @@ -397,4 +438,4 @@ export async function resolveAppContext( }; } -export type { Auth, Profile, ClerkConfig, AppContextOptions }; +export type { Auth, Profile, ClerkConfig, MigrationEntry, AppContextOptions }; diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index efc4e5301..89edb89d1 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -146,6 +146,9 @@ export const configStubs = { listProfiles: noop, getRelayEntry: noop, setRelayEntry: noop, + getMigrationEntry: noop, + setMigrationEntry: noop, + getProjectKey: async () => "", resolveProfile: noop, resolveProfileOrAutolink: noop, resolveInstanceId: () => ({ id: "", label: "" }), From 2f22633d887473ca56a239ee2964410638050f86 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 16:40:04 -0400 Subject: [PATCH 06/34] feat(migrate): add `clerk migrate settings`, and resolve migration env vars the way the secret key does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related gaps. **The env vars did not work from a file.** `CLERK_FIREBASE_*`, `AUTH0_*` and the `*_DB_URL` vars were read straight off `process.env`, which the shipped binary never populates from a `.env` file — it is compiled with `--no-compile-autoload-dotenv`. Only an exported shell variable reached them. The secret key avoided this by parsing the project's env files itself. That lookup is now shared: `findKeyInProject` moves out of `keyless-target.ts` into `lib/dotenv.ts` as `findEnvValue`, and every migration value goes through it. Each resolution reports its source, so `--verbose` names the file a value came from. Left on `process.env`: `CLERK_MIGRATE_RATE_LIMIT`, `CLERK_MIGRATE_CONCURRENCY_LIMIT` and `FIREBASE_AUTH_EMULATOR_HOST` — runtime knobs rather than project config, and `resolveLimits` is sync on a hot path. **There was no way to see or change what a run would pick up.** `clerk migrate settings` lists every setting with its value and its source, `set` changes one, `clear` forgets them. Values are split by what they are, not by which command wrote them: project state to the CLI config, credentials to `.env.clerk-migrate`. That file is the migration's own rather than the app's `.env.local`, because a Firebase signer key is of no use to the application being migrated and does not belong in the file its developers read daily. It is added to `.gitignore` on creation — reusing `ensureGitignoreEntry`, promoted from `keyless.ts` to `lib/git.ts` for the second caller — and deleted when `clear` removes its last value. Credentials are redacted everywhere they are shown, `--json` included. --- .../cli-core/src/commands/migrate/README.md | 70 ++++++- .../src/commands/migrate/export/auth0.test.ts | 18 +- .../src/commands/migrate/export/auth0.ts | 11 +- .../migrate/export/db-exports.test.ts | 34 +++- .../src/commands/migrate/export/db-options.ts | 5 +- .../cli-core/src/commands/migrate/index.ts | 2 + .../src/commands/migrate/lib/env-file.test.ts | 121 ++++++++++++ .../src/commands/migrate/lib/env-file.ts | 123 +++++++++++++ .../cli-core/src/commands/migrate/run.test.ts | 39 ++-- packages/cli-core/src/commands/migrate/run.ts | 33 ++-- .../src/commands/migrate/settings/clear.ts | 58 ++++++ .../src/commands/migrate/settings/index.ts | 75 ++++++++ .../src/commands/migrate/settings/list.ts | 87 +++++++++ .../src/commands/migrate/settings/registry.ts | 111 +++++++++++ .../src/commands/migrate/settings/set.ts | 48 +++++ .../migrate/settings/settings.test.ts | 172 ++++++++++++++++++ packages/cli-core/src/lib/dotenv.ts | 68 +++++++ packages/cli-core/src/lib/git.ts | 21 ++- packages/cli-core/src/lib/keyless-target.ts | 47 +---- packages/cli-core/src/lib/keyless.ts | 13 +- .../src/test/integration/lib/harness.ts | 1 + 21 files changed, 1038 insertions(+), 119 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/env-file.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/env-file.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/clear.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/index.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/list.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/registry.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/set.ts create mode 100644 packages/cli-core/src/commands/migrate/settings/settings.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 17eecf60b..1e479a8f8 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -441,6 +441,54 @@ clerk migrate transformers list --transformer-file ./my-transformer.ts `--json` gives an agent the same data, including which source field each transformer maps to `userId`. +### `clerk migrate settings` + +What a run in this directory would pick up, and where each value comes from. + +```sh +clerk migrate settings # list +clerk migrate settings set transformer firebase +clerk migrate settings set firebase-signer-key abc123… +clerk migrate settings clear -y +``` + +``` +SETTING VALUE SOURCE +transformer firebase clerk config +file ./users.json clerk config +firebase-signer-key aVer…3456 .env.clerk-migrate +firebase-rounds — not set +``` + +The source column is the point. A migration reads from flags, the environment, +two of the app's env files and the CLI's config, so when a run picks up a stale +value the question is never "what is it" but "which of those won". + +| Command | Description | +| ----------------------------- | ----------------------------------------------------- | +| `settings` / `settings list` | Show every setting, its value and its source | +| `settings list --json` | The same, machine-readable | +| `settings set ` | Change one setting | +| `settings clear [-y]` | Forget this project's settings and delete its secrets | + +#### Where each setting is kept + +Two stores, split by what the value **is** rather than by which command wrote it: + +| Store | Holds | Why | +| -------------------- | --------------------------------------------------- | ---------------------------------------------------------------- | +| CLI config | `transformer`, `file`, `skip-unsupported-providers` | Project state, not secret, useless outside the CLI | +| `.env.clerk-migrate` | `firebase-*` | Credentials: gitignored on write, and hand-editable for rotation | + +`.env.clerk-migrate` is the migration's own file rather than the app's +`.env.local`, because a Firebase signer key is of no use to the application +being migrated and does not belong in the file its developers read daily. The +CLI adds it to `.gitignore` the first time it writes it, and deletes it when +`settings clear` removes the last value. + +Credentials are redacted wherever they are displayed, including under `--json`, +so the output is safe to paste into an issue. + ### Custom transformers (`--transformer-file`) Migrating from a platform with no built-in, without recompiling the CLI: @@ -528,9 +576,10 @@ naming what is missing. A partial set produces a well-formed digest that verifies against nothing, so users would import successfully and then be unable to sign in. -They are **never saved**: the signer key is a Firebase secret, and remembering -it would mean writing it to disk in plaintext. To avoid re-passing all four on -every run, set them in the environment (`.env.local` is already gitignored): +They never go into the CLI's config: the signer key is a Firebase secret, and +that file is not a secret store. To avoid re-passing all four on every run, set +them once with [`clerk migrate settings`](#clerk-migrate-settings), or export +them yourself: | Variable | Flag | | ------------------------------- | --------------------------- | @@ -539,8 +588,9 @@ every run, set them in the environment (`.env.local` is already gitignored): | `CLERK_FIREBASE_ROUNDS` | `--firebase-rounds` | | `CLERK_FIREBASE_MEM_COST` | `--firebase-mem-cost` | -Flags win over the environment, and the two can be mixed as long as all four -end up supplied. +Resolution order is flag, then exported variable, then `.env.clerk-migrate`, +then the app's `.env.local`/`.env`. The sources can be mixed as long as all four +end up supplied. Run with `--verbose` to see which one each came from. An export with no password hashes needs no parameters at all. @@ -681,13 +731,13 @@ rather than "which project is linked here". | `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | | `./logs/export-.log` | NDJSON: one line per exported user | | `./exports/-export.json` | The export itself, unless `--output` says otherwise | +| `./.env.clerk-migrate` | Migration credentials, written by `settings set` and gitignored | The transformer and file of the last run are **not** written here. They go to -the `migrations` section of the CLI's own config file (`clerk config --help` -names its location), keyed by project the same way a linked profile is. That is -what `migrate delete` reads to know which migration to undo, so it is -load-bearing rather than a convenience — and it has no business being written -into the repository being migrated. +the `migrations` section of the CLI's own config file, keyed by project the +same way a linked profile is. That is what `migrate delete` reads to know which +migration to undo, so it is load-bearing rather than a convenience — and it has +no business being written into the repository being migrated. Log writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. Use the last diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index b48b89819..58ce5add2 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -15,6 +15,9 @@ import { resolveAuth0Credentials, } from "./auth0.ts"; +/** A cwd with no `.env` files, so these tests exercise only the injected env. */ +const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); + const captured = useCaptureLog(); const CREDENTIALS = { domain: "t.auth0.com", clientId: "cid", clientSecret: "csec" }; @@ -88,22 +91,25 @@ describe("resolveAuth0Credentials", () => { test("prefers flags", async () => { const resolved = await resolveAuth0Credentials( { domain: "flag.auth0.com", clientId: "f", clientSecret: "s" }, + NO_ENV_FILES, { AUTH0_DOMAIN: "env.auth0.com" }, ); expect(resolved.domain).toBe("flag.auth0.com"); }); test("falls back to the environment", async () => { - const resolved = await resolveAuth0Credentials( - {}, - { AUTH0_DOMAIN: "env.auth0.com", AUTH0_CLIENT_ID: "e", AUTH0_CLIENT_SECRET: "s" }, - ); + const resolved = await resolveAuth0Credentials({}, NO_ENV_FILES, { + AUTH0_DOMAIN: "env.auth0.com", + AUTH0_CLIENT_ID: "e", + AUTH0_CLIENT_SECRET: "s", + }); expect(resolved).toEqual({ domain: "env.auth0.com", clientId: "e", clientSecret: "s" }); }); test("normalizes a domain that came with a scheme", async () => { const resolved = await resolveAuth0Credentials( { domain: "https://t.auth0.com/", clientId: "c", clientSecret: "s" }, + NO_ENV_FILES, {}, ); expect(resolved.domain).toBe("t.auth0.com"); @@ -111,14 +117,14 @@ describe("resolveAuth0Credentials", () => { // Tests run non-TTY, the same signal an agent gives. test("names every missing credential at once rather than one at a time", async () => { - await expect(resolveAuth0Credentials({}, {})).rejects.toThrow( + await expect(resolveAuth0Credentials({}, NO_ENV_FILES, {})).rejects.toThrow( /--domain \(or AUTH0_DOMAIN\), --client-id \(or AUTH0_CLIENT_ID\), --client-secret \(or AUTH0_CLIENT_SECRET\)/, ); }); test("names only what is actually missing", async () => { await expect( - resolveAuth0Credentials({ domain: "t.auth0.com", clientId: "c" }, {}), + resolveAuth0Credentials({ domain: "t.auth0.com", clientId: "c" }, NO_ENV_FILES, {}), ).rejects.toThrow(/Missing: --client-secret \(or AUTH0_CLIENT_SECRET\)\./); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 0e94933b5..3d90d661c 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -21,6 +21,7 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt, text } from "../../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; @@ -64,12 +65,16 @@ export function normalizeAuth0Domain(domain: string): string { */ export async function resolveAuth0Credentials( options: ExportAuth0Options, + cwd: string = process.cwd(), env: Record = process.env, ): Promise { + const fromEnv = async (name: string): Promise => + (await findMigrateEnvValue([name], cwd, env))?.value; + const resolved = { - domain: options.domain ?? env.AUTH0_DOMAIN, - clientId: options.clientId ?? env.AUTH0_CLIENT_ID, - clientSecret: options.clientSecret ?? env.AUTH0_CLIENT_SECRET, + domain: options.domain ?? (await fromEnv("AUTH0_DOMAIN")), + clientId: options.clientId ?? (await fromEnv("AUTH0_CLIENT_ID")), + clientSecret: options.clientSecret ?? (await fromEnv("AUTH0_CLIENT_SECRET")), }; const missing = ( diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index e7601df39..15116d255 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -27,6 +27,9 @@ import { import { buildSupabaseExport } from "./supabase.ts"; import { looksLikeConnectionString, resolveDbUrl } from "./db-options.ts"; +/** A cwd with no `.env` files, so these tests exercise only the injected env. */ +const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); + const captured = useCaptureLog(); let workDir: string; @@ -105,32 +108,45 @@ describe("resolveDbUrl", () => { const config = { platform: "authjs" as const, envVar: "AUTHJS_DB_URL", prompt: "url" }; test("prefers the flag", async () => { - const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, { + const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, NO_ENV_FILES, { AUTHJS_DB_URL: "mysql://u:p@h/db", }); expect(url).toBe("postgres://u:p@h/db"); }); test("falls back to the environment variable", async () => { - expect(await resolveDbUrl({}, config, { AUTHJS_DB_URL: "mysql://u:p@h/db" })).toBe( - "mysql://u:p@h/db", - ); + expect( + await resolveDbUrl({}, config, NO_ENV_FILES, { AUTHJS_DB_URL: "mysql://u:p@h/db" }), + ).toBe("mysql://u:p@h/db"); }); test("rejects a flag that is not a connection string, naming the encoding trap", async () => { - await expect(resolveDbUrl({ dbUrl: "not a url" }, config, {})).rejects.toThrow(/URL-encode it/); + await expect(resolveDbUrl({ dbUrl: "not a url" }, config, NO_ENV_FILES, {})).rejects.toThrow( + /URL-encode it/, + ); }); test("warns and moves on when the environment variable is unusable", async () => { // Tests run non-TTY, so it then hits the agent-mode branch. - await expect(resolveDbUrl({}, config, { AUTHJS_DB_URL: "garbage" })).rejects.toThrow( - /cannot prompt here/, - ); + await expect( + resolveDbUrl({}, config, NO_ENV_FILES, { AUTHJS_DB_URL: "garbage" }), + ).rejects.toThrow(/cannot prompt here/); expect(captured.err).toContain("AUTHJS_DB_URL is not a valid connection string"); }); + // The env var reaching process.env is the runtime's job; this is the fallback + // for when it did not, and is the rung the secret key has always had. + test("falls back to a .env file when the variable is not in the environment", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-dburl-env-")); + fs.writeFileSync(path.join(dir, ".env.local"), "AUTHJS_DB_URL=postgres://u:p@h/db\n"); + + expect(await resolveDbUrl({}, config, dir, {})).toBe("postgres://u:p@h/db"); + }); + test("names both the flag and the variable when it cannot prompt", async () => { - await expect(resolveDbUrl({}, config, {})).rejects.toThrow(/--db-url.*AUTHJS_DB_URL/s); + await expect(resolveDbUrl({}, config, NO_ENV_FILES, {})).rejects.toThrow( + /--db-url.*AUTHJS_DB_URL/s, + ); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 880e177da..999e6d5be 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -11,6 +11,7 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { detectDbType, redactConnectionString, type DbPlatform } from "../lib/db.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; export type DbExportOptions = { dbUrl?: string; @@ -57,6 +58,7 @@ export function looksLikeConnectionString(value: string): boolean { export async function resolveDbUrl( options: DbExportOptions, config: ResolveConfig, + cwd: string = process.cwd(), env: Record = process.env, ): Promise { const fromFlag = options.dbUrl?.trim(); @@ -70,7 +72,8 @@ export async function resolveDbUrl( return fromFlag; } - const fromEnv = env[config.envVar]?.trim(); + const located = await findMigrateEnvValue([config.envVar], cwd, env); + const fromEnv = located?.value.trim(); if (fromEnv) { if (looksLikeConnectionString(fromEnv)) return fromEnv; // Falling through silently would make the prompt look unexplained. diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index 3d34ca55c..a1a56e1c7 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -4,6 +4,7 @@ import { parseIntegerOption } from "../../lib/option-parsers.ts"; import { deleteMigration } from "./delete.ts"; import { registerMigrateExport } from "./export/index.ts"; import { registerMigrateLogs } from "./logs/index.ts"; +import { registerMigrateSettings } from "./settings/index.ts"; import { run } from "./run.ts"; import { list as transformersList } from "./transformers/list.ts"; import { transformerKeys } from "./transformers/registry.ts"; @@ -131,4 +132,5 @@ export function registerMigrate(program: Program): void { ); registerMigrateLogs(migrateCommand); + registerMigrateSettings(migrateCommand); } diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.test.ts b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts new file mode 100644 index 000000000..7c190a9d7 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts @@ -0,0 +1,121 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + clearMigrateEnvValues, + findMigrateEnvValue, + MIGRATE_ENV_FILE, + writeMigrateEnvValues, +} from "./env-file.ts"; + +let workDir: string; + +const envFile = () => path.join(workDir, MIGRATE_ENV_FILE); +const read = (file: string) => fs.readFileSync(path.join(workDir, file), "utf-8"); + +beforeEach(() => { + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-envfile-"))); +}); + +afterEach(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("writeMigrateEnvValues", () => { + test("creates the file and gitignores it", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).toBe("CLERK_FIREBASE_ROUNDS=8\n"); + expect(read(".gitignore")).toContain(MIGRATE_ENV_FILE); + }); + + test("appends to an existing .gitignore without duplicating the entry", async () => { + fs.writeFileSync(path.join(workDir, ".gitignore"), "node_modules\n"); + + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_MEM_COST: "14" }, workDir); + + expect(read(".gitignore")).toBe(`node_modules\n${MIGRATE_ENV_FILE}\n`); + }); + + // The header `mergeEnvVars` adds is right for an app's shared .env and wrong + // here — one `settings set` per key would stack one header per call. + test("adds no section header, however many times it is called", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_MEM_COST: "14" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_SIGNER_KEY: "k" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).not.toContain("#"); + }); + + test("updates a key in place rather than appending a second copy", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "10" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).toBe("CLERK_FIREBASE_ROUNDS=10\n"); + }); + + // The file is meant to be hand-editable, so a write must not flatten it. + test("preserves hand-written comments and unrelated keys", async () => { + fs.writeFileSync(envFile(), "# my note\nOTHER=keep\n"); + + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).toBe("# my note\nOTHER=keep\nCLERK_FIREBASE_ROUNDS=8\n"); + }); +}); + +describe("findMigrateEnvValue", () => { + test("reads a value out of the file", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_SIGNER_KEY: "from-file" }, workDir); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_SIGNER_KEY"], workDir, {}); + expect(located).toEqual({ value: "from-file", source: MIGRATE_ENV_FILE }); + }); + + test("beats the app's own .env.local", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=1\n"); + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, {}); + expect(located?.value).toBe("8"); + }); + + // An exported variable is the one thing an operator can change per-invocation. + test("loses to an exported environment variable", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "99", + }); + expect(located).toEqual({ value: "99", source: "CLERK_FIREBASE_ROUNDS env var" }); + }); + + test("returns nothing when the setting is absent everywhere", async () => { + expect(await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, {})).toBeUndefined(); + }); +}); + +describe("clearMigrateEnvValues", () => { + test("removes only the named settings", async () => { + fs.writeFileSync(envFile(), "OTHER=keep\nCLERK_FIREBASE_ROUNDS=8\n"); + + expect(await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir)).toEqual([ + "CLERK_FIREBASE_ROUNDS", + ]); + expect(read(MIGRATE_ENV_FILE)).toBe("OTHER=keep\n"); + }); + + // Left behind, it reads as "there is config here" when there is not. + test("deletes the file when nothing but comments would remain", async () => { + fs.writeFileSync(envFile(), "# a note\nCLERK_FIREBASE_ROUNDS=8\n"); + + await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir); + expect(fs.existsSync(envFile())).toBe(false); + }); + + test("reports nothing dropped when there is no file", async () => { + expect(await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir)).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.ts b/packages/cli-core/src/commands/migrate/lib/env-file.ts new file mode 100644 index 000000000..4d7a61f0a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/env-file.ts @@ -0,0 +1,123 @@ +/** + * `.env.clerk-migrate` — the migration's own env file. + * + * Migration credentials are a Firebase signer key, an Auth0 client secret, a + * database URL: things the app being migrated has no use for. Writing them into + * the app's `.env.local` mixes two unrelated sets of config in the file a + * developer reads every day, so they get their own. + * + * Read ahead of `.env`/`.env.local`, so a value set here wins over a stale one + * left in the app's file. An exported shell variable still beats both — that is + * {@link findEnvValue}'s contract for every value the CLI resolves. + * + * Always added to `.gitignore` on write. The CLI creating a credential-bearing + * file in someone's repository without that is how one ends up committed. + */ + +import { unlink } from "node:fs/promises"; +import { join } from "node:path"; +import { + findEnvValue, + parseEnvFile, + serializeEnvFile, + type EnvLine, + type LocatedEnvValue, +} from "../../../lib/dotenv.ts"; +import { ensureGitignoreEntry } from "../../../lib/git.ts"; +import { log } from "../../../lib/log.ts"; + +export const MIGRATE_ENV_FILE = ".env.clerk-migrate"; + +/** Lowest priority first: the migration's own file overrides the app's. */ +const MIGRATE_ENV_FILES = [".env", ".env.local", MIGRATE_ENV_FILE] as const; + +/** Resolves a migration setting: environment first, then the project's env files. */ +export async function findMigrateEnvValue( + names: string[], + cwd: string = process.cwd(), + env: Record = process.env, +): Promise { + const located = await findEnvValue(cwd, names, { env, files: MIGRATE_ENV_FILES }); + if (located) log.debug(`migrate: ${names[0]} from ${located.source}`); + return located; +} + +/** + * Merges `values` into the parsed file: existing keys update in place, new ones + * append. + * + * Deliberately not `mergeEnvVars` from `lib/dotenv.ts`. That one prepends a + * `# Clerk` section header when the file holds none of the keys being written, + * which is right for `env pull` dropping Clerk keys into an app's shared `.env` + * — and wrong here twice over: every key in this file is already Clerk's, and + * writing one setting at a time means the check fires again on every call, + * stacking a fresh header per `settings set`. + */ +function mergeMigrateEnv(lines: EnvLine[], values: Record): EnvLine[] { + const remaining = { ...values }; + + const merged = lines.map((line): EnvLine => { + if (line.type !== "entry" || !(line.key in remaining)) return line; + const value = remaining[line.key]!; + delete remaining[line.key]; + return { type: "entry", key: line.key, value, raw: `${line.key}=${value}` }; + }); + + for (const [key, value] of Object.entries(remaining)) { + merged.push({ type: "entry", key, value, raw: `${key}=${value}` }); + } + return merged; +} + +/** + * Writes settings into `.env.clerk-migrate`, creating and gitignoring it first. + * + * Existing comments, blank lines and key order survive — the file is meant to + * be hand-edited, so rewriting it wholesale would discard the user's notes. + */ +export async function writeMigrateEnvValues( + values: Record, + cwd: string = process.cwd(), +): Promise { + const target = join(cwd, MIGRATE_ENV_FILE); + const existing = await Bun.file(target) + .text() + .catch(() => ""); + + await Bun.write(target, serializeEnvFile(mergeMigrateEnv(parseEnvFile(existing), values))); + await ensureGitignoreEntry(cwd, MIGRATE_ENV_FILE); + + return MIGRATE_ENV_FILE; +} + +/** Removes the named settings from `.env.clerk-migrate`, leaving the rest. */ +export async function clearMigrateEnvValues( + names: string[], + cwd: string = process.cwd(), +): Promise { + const target = join(cwd, MIGRATE_ENV_FILE); + const existing = await Bun.file(target) + .text() + .catch(() => ""); + if (!existing) return []; + + const dropped: string[] = []; + const kept = parseEnvFile(existing).filter((line) => { + if (line.type !== "entry" || !names.includes(line.key)) return true; + dropped.push(line.key); + return false; + }); + + if (dropped.length === 0) return dropped; + + // A file holding nothing but the comments that described the settings it no + // longer has is worse than no file: it reads as "there is config here". + if (kept.some((line) => line.type === "entry")) { + await Bun.write(target, serializeEnvFile(kept)); + } else { + await unlink(target).catch(() => {}); + log.debug(`migrate: removed empty ${MIGRATE_ENV_FILE}`); + } + + return dropped; +} diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index ff9ffbbae..330d70645 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -69,8 +69,8 @@ describe("resolveFirebaseHashConfig", () => { firebaseMemCost: 14, }; - test("builds the config when all four flags are present", () => { - expect(resolveFirebaseHashConfig(ALL)).toEqual({ + test("builds the config when all four flags are present", async () => { + expect(await resolveFirebaseHashConfig(ALL)).toEqual({ base64_signer_key: "SIGNER", base64_salt_separator: "Bw==", rounds: 8, @@ -85,14 +85,14 @@ describe("resolveFirebaseHashConfig", () => { ["firebaseSaltSeparator", "--firebase-salt-separator"], ["firebaseRounds", "--firebase-rounds"], ["firebaseMemCost", "--firebase-mem-cost"], - ] as const)("rejects a set missing %s, naming the flag", (omit, flag) => { + ] as const)("rejects a set missing %s, naming the flag", async (omit, flag) => { const partial = { ...ALL }; delete (partial as Record)[omit]; - expect(() => resolveFirebaseHashConfig(partial)).toThrow(new RegExp(flag)); + await expect(resolveFirebaseHashConfig(partial)).rejects.toThrow(new RegExp(flag)); }); - test("names every missing flag at once", () => { - expect(() => resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" })).toThrow( + test("names every missing flag at once", async () => { + await expect(resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" })).rejects.toThrow( /--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/, ); }); @@ -111,9 +111,9 @@ describe("resolveFirebaseHashConfig", () => { const setEnv = (vars: Partial) => Object.assign(process.env, vars); - test("builds the config when no flag is passed", () => { + test("builds the config when no flag is passed", async () => { setEnv(ENV); - expect(resolveFirebaseHashConfig({})).toEqual({ + expect(await resolveFirebaseHashConfig({})).toEqual({ base64_signer_key: "ENV_SIGNER", base64_salt_separator: "Bw==", rounds: 8, @@ -121,16 +121,19 @@ describe("resolveFirebaseHashConfig", () => { }); }); - test("prefers a flag over the environment", () => { + test("prefers a flag over the environment", async () => { setEnv(ENV); - expect(resolveFirebaseHashConfig(ALL)?.base64_signer_key).toBe("SIGNER"); + expect((await resolveFirebaseHashConfig(ALL))?.base64_signer_key).toBe("SIGNER"); }); // Half from the environment and half from flags is still a complete set. - test("fills only the gaps the flags left", () => { + test("fills only the gaps the flags left", async () => { setEnv({ CLERK_FIREBASE_ROUNDS: "8", CLERK_FIREBASE_MEM_COST: "14" }); expect( - resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER", firebaseSaltSeparator: "Bw==" }), + await resolveFirebaseHashConfig({ + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + }), ).toEqual({ base64_signer_key: "SIGNER", base64_salt_separator: "Bw==", @@ -139,21 +142,21 @@ describe("resolveFirebaseHashConfig", () => { }); }); - test("still demands the full set when the environment supplies only part", () => { + test("still demands the full set when the environment supplies only part", async () => { setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); - expect(() => resolveFirebaseHashConfig({})).toThrow(/--firebase-salt-separator/); + await expect(resolveFirebaseHashConfig({})).rejects.toThrow(/--firebase-salt-separator/); }); // An empty var is how a shell spells "unset", and treating it as set would // demand the other three for a config nobody asked for. - test("ignores an empty variable", () => { + test("ignores an empty variable", async () => { setEnv({ CLERK_FIREBASE_SIGNER_KEY: "" }); - expect(resolveFirebaseHashConfig({})).toBeUndefined(); + expect(await resolveFirebaseHashConfig({})).toBeUndefined(); }); }); - test("returns nothing when neither flags nor the environment supply a config", () => { - expect(resolveFirebaseHashConfig({})).toBeUndefined(); + test("returns nothing when neither flags nor the environment supply a config", async () => { + expect(await resolveFirebaseHashConfig({})).toBeUndefined(); }); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index cd1e1934f..9b6e9bdf0 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -19,6 +19,7 @@ import { withGutter, withSpinner } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; import { importUsers } from "./import-users.ts"; import { analyzeFields } from "./lib/analysis.ts"; +import { findMigrateEnvValue } from "./lib/env-file.ts"; import { enabledSocialProviders, fetchInstanceSettings, @@ -71,29 +72,31 @@ const FIREBASE_FLAGS = [ const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); /** - * Overlays the `CLERK_FIREBASE_*` environment variables onto whichever flags - * were not passed. + * Overlays the `CLERK_FIREBASE_*` values onto whichever flags were not passed. * - * The signer key is a Firebase secret, so it is read rather than stored: the - * CLI never persists these, and `.env.local` is already gitignored and already - * where the CLI keeps a project's local secrets. + * Resolved through {@link findMigrateEnvValue}: the environment first, then + * `.env.clerk-migrate`, then the app's own `.env` files. The signer key is a + * Firebase secret, so it is never written to the CLI's config — + * `.env.clerk-migrate` is gitignored on creation. */ -function withFirebaseEnv(options: MigrateRunOptions): MigrateRunOptions { +async function withFirebaseEnv(options: MigrateRunOptions): Promise { const merged = { ...options }; for (const [key, , envVar] of FIREBASE_FLAGS) { if (merged[key] !== undefined) continue; - const value = process.env[envVar]; - if (value === undefined || value.trim() === "") continue; + const located = await findMigrateEnvValue([envVar]); + if (!located || located.value.trim() === "") continue; // A non-numeric round count is left to fail the flag's own validation // rather than silently becoming NaN. - (merged as Record)[key] = FIREBASE_NUMERIC.has(key) ? Number(value) : value; + (merged as Record)[key] = FIREBASE_NUMERIC.has(key) + ? Number(located.value) + : located.value; } return merged; } /** * Resolves Firebase's four hash parameters from flags, falling back to the - * `CLERK_FIREBASE_*` environment variables. + * `CLERK_FIREBASE_*` environment variables and the project's `.env` files. * * The four are required as a set: a digest built from a partial set is * well-formed but verifies against nothing, so every migrated user would fail @@ -102,10 +105,10 @@ function withFirebaseEnv(options: MigrateRunOptions): MigrateRunOptions { * @returns The config, or `undefined` when none was supplied — which is fine * for an export that carries no password hashes. */ -export function resolveFirebaseHashConfig( +export async function resolveFirebaseHashConfig( rawOptions: MigrateRunOptions, -): FirebaseHashConfig | undefined { - const options = withFirebaseEnv(rawOptions); +): Promise { + const options = await withFirebaseEnv(rawOptions); const provided = FIREBASE_FLAGS.filter(([key]) => options[key] !== undefined); if (provided.length === 0) return undefined; @@ -370,7 +373,7 @@ async function resolveMissingOptions(options: MigrateRunOptions): Promise { const secretKeyOption = options.secretKey ?? options.clerkSecretKey; const { transformer, file } = validateRunOptions(options); - const firebaseHashConfig = resolveFirebaseHashConfig(options); + const firebaseHashConfig = await resolveFirebaseHashConfig(options); await withGutter("Migrating users to Clerk", async () => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); diff --git a/packages/cli-core/src/commands/migrate/settings/clear.ts b/packages/cli-core/src/commands/migrate/settings/clear.ts new file mode 100644 index 000000000..933d1e540 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/clear.ts @@ -0,0 +1,58 @@ +/** + * `clerk migrate settings clear` — forget this project's migration settings. + * + * Clears both stores by default. The credentials half is the reason this + * command exists: after a migration finishes, a Firebase signer key sitting in + * the repo has no further use, and "delete the file yourself" is a step people + * skip. + * + * `migrate delete` reads the saved transformer and file to know what to undo, + * so clearing is confirmed unless `-y` — an operator who clears and then wants + * to undo has no record left to undo from. + */ + +import { throwUserAbort } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { confirm } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { clearMigrateEnvValues, MIGRATE_ENV_FILE } from "../lib/env-file.ts"; +import { loadSettings, saveSettings } from "../lib/settings.ts"; +import { SETTINGS } from "./registry.ts"; + +export type SettingsClearOptions = { + yes?: boolean; +}; + +const ENV_VARS = SETTINGS.filter((s) => s.store === "env").map((s) => s.envVar as string); + +export async function clear(options: SettingsClearOptions = {}): Promise { + const saved = await loadSettings(); + const hadConfig = Object.keys(saved).length > 0; + + if (!options.yes && isHuman() && !isAgent()) { + if (hadConfig && saved.file) { + log.warn( + `\`clerk migrate delete\` uses the saved file (${saved.file}) to identify the users the last run created. ` + + "Clearing it leaves nothing to undo from.", + ); + } + const proceed = await confirm({ + message: "Clear this project's migration settings?", + default: false, + }); + if (!proceed) throwUserAbort(); + } + + if (hadConfig) await saveSettings({}); + const dropped = await clearMigrateEnvValues(ENV_VARS); + + if (!hadConfig && dropped.length === 0) { + log.info("No migration settings to clear for this project."); + return; + } + + if (hadConfig) log.success("Cleared the saved transformer and file."); + if (dropped.length > 0) { + log.success(`Removed ${dropped.length} credential(s) from ${MIGRATE_ENV_FILE}.`); + } +} diff --git a/packages/cli-core/src/commands/migrate/settings/index.ts b/packages/cli-core/src/commands/migrate/settings/index.ts new file mode 100644 index 000000000..7ce5b6f44 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/index.ts @@ -0,0 +1,75 @@ +import { createArgument } from "@commander-js/extra-typings"; +import type { Command } from "@commander-js/extra-typings"; +import { clear } from "./clear.ts"; +import { list } from "./list.ts"; +import { SETTING_NAMES } from "./registry.ts"; +import { set } from "./set.ts"; + +const settings = { clear, list, set }; + +/** + * Registers `settings list|set|clear` under the `migrate` group. + * + * Noun-verb like every other group in the tree, and listing is the default + * because it is the read-only one — a bare `clerk migrate settings` should show, + * never change. + */ +export function registerMigrateSettings( + migrateCommand: Command<[], Record>, +): void { + const settingsCommand = migrateCommand + .command("settings") + .description("Inspect and change this project's saved migration settings") + .setExamples([ + { + command: "clerk migrate settings", + description: "Show every setting and where it resolves from", + }, + { + command: "clerk migrate settings set firebase-signer-key abc123", + description: "Save a credential to the gitignored .env.clerk-migrate", + }, + { command: "clerk migrate settings clear -y", description: "Forget this project's settings" }, + ]); + + settingsCommand + .command("list", { isDefault: true }) + .description("Show each setting, its value and which source supplied it") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate settings list", description: "Credentials shown redacted" }, + { command: "clerk migrate settings list --json", description: "Machine-readable listing" }, + ]) + .action((_opts, cmd) => + settings.list(cmd.optsWithGlobals() as Parameters[0]), + ); + + settingsCommand + .command("set") + .description("Set one setting for this project") + .addArgument(createArgument("", "Setting to change").choices(SETTING_NAMES)) + .addArgument(createArgument("", "New value")) + .setExamples([ + { + command: "clerk migrate settings set transformer firebase", + description: "Remember the source platform", + }, + { + command: "clerk migrate settings set firebase-signer-key abc123", + description: "Write a credential to .env.clerk-migrate", + }, + ]) + .action((name, value) => settings.set(name, value)); + + settingsCommand + .command("clear") + .description("Forget the saved settings and remove the saved credentials") + .option("-y, --yes", "Skip the confirmation prompt") + .setExamples([ + { command: "clerk migrate settings clear", description: "Clear after confirming" }, + { command: "clerk migrate settings clear -y", description: "Clear without prompting" }, + ]) + .action((_opts, cmd) => + settings.clear(cmd.optsWithGlobals() as Parameters[0]), + ); +} diff --git a/packages/cli-core/src/commands/migrate/settings/list.ts b/packages/cli-core/src/commands/migrate/settings/list.ts new file mode 100644 index 000000000..9fb1683d0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/list.ts @@ -0,0 +1,87 @@ +/** + * `clerk migrate settings` — what a run in this project would pick up, and + * where each value is coming from. + * + * The source column is the point. A migration reads from flags, the + * environment, two of the app's env files and the CLI's config; when a run uses + * a stale value, the question is never "what is it" but "which of those is + * winning". Credentials are redacted, so this is safe to paste into an issue. + */ + +import { cyan, dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; +import { loadSettings } from "../lib/settings.ts"; +import { displayValue, SETTINGS, type SettingDef } from "./registry.ts"; + +export type SettingsListOptions = { + json?: boolean; +}; + +interface ResolvedSetting { + setting: SettingDef; + value?: string; + source?: string; +} + +async function resolveAll(): Promise { + const saved = await loadSettings(); + + return Promise.all( + SETTINGS.map(async (setting): Promise => { + if (setting.store === "config") { + const value = saved[setting.configKey as keyof typeof saved]; + return value === undefined + ? { setting } + : { setting, value: String(value), source: "clerk config" }; + } + + const located = await findMigrateEnvValue([setting.envVar as string]); + return located ? { setting, value: located.value, source: located.source } : { setting }; + }), + ); +} + +function toJson(resolved: ResolvedSetting[]) { + return resolved.map(({ setting, value, source }) => ({ + name: setting.name, + store: setting.store, + // Redacted here too: `--json` is what gets piped into a log or a ticket. + value: value === undefined ? null : displayValue(setting, value), + set: value !== undefined, + secret: Boolean(setting.secret), + source: source ?? null, + })); +} + +export async function list(options: SettingsListOptions = {}): Promise { + const resolved = await resolveAll(); + + if (options.json) { + log.data(JSON.stringify(toJson(resolved), null, 2)); + return; + } + + const nameWidth = Math.max(...SETTINGS.map((s) => s.name.length), "SETTING".length) + 2; + const valueWidth = + Math.max( + ...resolved.map(({ setting, value }) => + value === undefined ? 1 : displayValue(setting, value).length, + ), + "VALUE".length, + ) + 2; + + log.info(dim("SETTING".padEnd(nameWidth)) + dim("VALUE".padEnd(valueWidth)) + dim("SOURCE")); + + for (const { setting, value, source } of resolved) { + const shown = value === undefined ? dim("—") : displayValue(setting, value); + log.info( + cyan(setting.name.padEnd(nameWidth)) + + shown.padEnd(valueWidth + (value === undefined ? dim("—").length - 1 : 0)) + + dim(source ?? "not set"), + ); + } + + log.blank(); + log.info(dim("Credentials are shown redacted. `clerk migrate settings set `.")); +} diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts new file mode 100644 index 000000000..8063607a2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -0,0 +1,111 @@ +/** + * What `clerk migrate settings` can show and change. + * + * Two stores, split by what the value is rather than by which command wrote it: + * + * - **config** — what this project last migrated. Not secret, not per-machine + * secret material, and useless to anyone but the CLI, so it lives in the + * CLI's own config file keyed by project. + * - **env** — credentials. They go to `.env.clerk-migrate`, which is + * gitignored on write and hand-editable, because a credential belongs + * somewhere the user can rotate it without the CLI's help. + * + * A setting is listed here exactly once; `list`, `set` and `clear` all read + * this table rather than each keeping their own idea of what exists. + */ + +export type SettingStore = "config" | "env"; + +export interface SettingDef { + /** What the user types: `clerk migrate settings set `. */ + name: string; + store: SettingStore; + description: string; + /** For `env` settings, the variable read at run time. */ + envVar?: string; + /** For `config` settings, the key on the saved migration entry. */ + configKey?: "transformer" | "file" | "skipUnsupportedProviders"; + /** Redact when displaying — the value is a credential. */ + secret?: boolean; + /** Reject a value the run would only fail on later. */ + validate?: (value: string) => string | undefined; +} + +const positiveInteger = (value: string): string | undefined => { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? undefined : "Expected a positive whole number"; +}; + +const boolean = (value: string): string | undefined => + ["true", "false"].includes(value) ? undefined : "Expected true or false"; + +export const SETTINGS: SettingDef[] = [ + { + name: "transformer", + store: "config", + configKey: "transformer", + description: "Source platform the last run imported from", + }, + { + name: "file", + store: "config", + configKey: "file", + description: "Export file the last run imported", + }, + { + name: "skip-unsupported-providers", + store: "config", + configKey: "skipUnsupportedProviders", + description: "Supabase: skip users whose only social provider is disabled in Clerk", + validate: boolean, + }, + { + name: "firebase-signer-key", + store: "env", + envVar: "CLERK_FIREBASE_SIGNER_KEY", + description: "Firebase base64 signer key", + secret: true, + }, + { + name: "firebase-salt-separator", + store: "env", + envVar: "CLERK_FIREBASE_SALT_SEPARATOR", + description: "Firebase base64 salt separator", + }, + { + name: "firebase-rounds", + store: "env", + envVar: "CLERK_FIREBASE_ROUNDS", + description: "Firebase scrypt rounds", + validate: positiveInteger, + }, + { + name: "firebase-mem-cost", + store: "env", + envVar: "CLERK_FIREBASE_MEM_COST", + description: "Firebase scrypt memory cost", + validate: positiveInteger, + }, +]; + +export const SETTING_NAMES = SETTINGS.map((setting) => setting.name); + +export function findSetting(name: string): SettingDef | undefined { + return SETTINGS.find((setting) => setting.name === name); +} + +/** + * Shows enough of a credential to recognise it, never enough to use it. + * + * Anything short enough that head-and-tail would leak most of it is masked + * whole: a 10-character key shown as `abcd…wxyz` has given away 8 of them. + */ +export function redact(value: string): string { + if (value.length < 16) return "•".repeat(8); + return `${value.slice(0, 4)}…${value.slice(-4)}`; +} + +/** The display value for a setting: redacted when it is a credential. */ +export function displayValue(setting: SettingDef, value: string): string { + return setting.secret ? redact(value) : value; +} diff --git a/packages/cli-core/src/commands/migrate/settings/set.ts b/packages/cli-core/src/commands/migrate/settings/set.ts new file mode 100644 index 000000000..1b0e1d997 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/set.ts @@ -0,0 +1,48 @@ +/** + * `clerk migrate settings set ` — change one setting. + * + * Which store it lands in is a property of the setting, not a flag: a + * credential always goes to `.env.clerk-migrate`, project state always goes to + * the CLI config. Letting the caller choose would mean a signer key could be + * put somewhere that is not gitignored. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { writeMigrateEnvValues } from "../lib/env-file.ts"; +import { loadSettings, saveSettings } from "../lib/settings.ts"; +import { displayValue, findSetting, SETTING_NAMES } from "./registry.ts"; + +export async function set(name: string, value: string): Promise { + const setting = findSetting(name); + if (!setting) { + throwUsageError( + `Unknown setting "${name}". Valid names: ${SETTING_NAMES.join(", ")}.`, + undefined, + undefined, + [ + { + command: "clerk migrate settings", + description: "List the settings and their current values", + }, + ], + ); + } + + const invalid = setting.validate?.(value); + if (invalid) throwUsageError(`Invalid value for ${name}: ${invalid}.`); + + if (setting.store === "env") { + const file = await writeMigrateEnvValues({ [setting.envVar as string]: value }); + log.success(`Set \`${name}\` in ${file} (gitignored).`); + return; + } + + const saved = await loadSettings(); + await saveSettings({ + ...saved, + [setting.configKey as string]: + setting.configKey === "skipUnsupportedProviders" ? value === "true" : value, + }); + log.success(`Set \`${name}\` to ${displayValue(setting, value)} for this project.`); +} diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts new file mode 100644 index 000000000..9bee68233 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -0,0 +1,172 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { MIGRATE_ENV_FILE } from "../lib/env-file.ts"; +import { loadSettings, saveSettings } from "../lib/settings.ts"; +import { clear } from "./clear.ts"; +import { list } from "./list.ts"; +import { redact } from "./registry.ts"; +import { set } from "./set.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let configDir: string; +let originalCwd: string; + +const envFileContent = () => fs.readFileSync(path.join(workDir, MIGRATE_ENV_FILE), "utf-8"); + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-cmd-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-cfg-")); + _setConfigDir(configDir); + process.chdir(workDir); +}); + +afterAll(() => { + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.rmSync(path.join(workDir, MIGRATE_ENV_FILE), { force: true }); + fs.rmSync(path.join(workDir, ".gitignore"), { force: true }); +}); + +afterEach(() => { + process.exitCode = 0; +}); + +describe("redact", () => { + test("shows head and tail of a long value", () => { + expect(redact("aVeryLongSignerKeyValue123456")).toBe("aVer…3456"); + }); + + // Head-and-tail on a short value gives away most of it. + test.each([["short"], ["0123456789"], ["123456789012345"]])("masks %p whole", (value) => { + expect(redact(value)).toBe("••••••••"); + }); +}); + +describe("set", () => { + test("writes a credential to the gitignored env file, not the CLI config", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + + expect(envFileContent()).toContain("CLERK_FIREBASE_SIGNER_KEY=aVeryLongSignerKeyValue123456"); + expect(await loadSettings()).toEqual({}); + expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toContain(MIGRATE_ENV_FILE); + }); + + test("writes project state to the CLI config, not the env file", async () => { + await set("transformer", "firebase"); + + expect(await loadSettings()).toEqual({ transformer: "firebase" }); + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + }); + + test("keeps the settings it is not changing", async () => { + await saveSettings({ transformer: "clerk", file: "users.json" }); + await set("file", "other.json"); + + expect(await loadSettings()).toEqual({ transformer: "clerk", file: "other.json" }); + }); + + test("stores a boolean setting as a boolean", async () => { + await set("skip-unsupported-providers", "true"); + expect(await loadSettings()).toEqual({ skipUnsupportedProviders: true }); + }); + + test.each([ + ["firebase-rounds", "zero", /positive whole number/], + ["skip-unsupported-providers", "yes", /true or false/], + ])("rejects an invalid value for %s", async (name, value, message) => { + await expect(set(name, value)).rejects.toThrow(message); + }); + + test("names the valid settings when given an unknown one", async () => { + await expect(set("nope", "x")).rejects.toThrow(/firebase-signer-key/); + }); + + // A run would fail on it later; failing at write time keeps the bad value out + // of the file entirely. + test("writes nothing when the value is rejected", async () => { + await expect(set("firebase-rounds", "-1")).rejects.toThrow(); + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + }); +}); + +describe("list", () => { + test("names the source each value resolved from", async () => { + await set("transformer", "firebase"); + await set("firebase-salt-separator", "Bw=="); + captured.clear(); + + await list(); + + expect(captured.err).toContain("clerk config"); + expect(captured.err).toContain(MIGRATE_ENV_FILE); + }); + + test("redacts a credential but not the rest", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + await set("transformer", "firebase"); + captured.clear(); + + await list(); + + expect(captured.err).toContain("aVer…3456"); + expect(captured.err).not.toContain("aVeryLongSignerKeyValue123456"); + expect(captured.err).toContain("firebase"); + }); + + // --json is what gets piped into a ticket or a CI log. + test("redacts in JSON output too", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + captured.clear(); + + await list({ json: true }); + + expect(captured.out).not.toContain("aVeryLongSignerKeyValue123456"); + expect(JSON.parse(captured.out)).toContainEqual( + expect.objectContaining({ name: "firebase-signer-key", value: "aVer…3456", secret: true }), + ); + }); + + test("marks everything as unset in a fresh project", async () => { + await list({ json: true }); + expect(JSON.parse(captured.out).every((entry: { set: boolean }) => !entry.set)).toBe(true); + }); +}); + +describe("clear", () => { + test("empties both stores", async () => { + await set("transformer", "firebase"); + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + + await clear({ yes: true }); + + expect(await loadSettings()).toEqual({}); + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + }); + + test("says so rather than claiming to have cleared nothing", async () => { + await clear({ yes: true }); + expect(captured.err).toContain("No migration settings to clear"); + }); + + test("leaves settings the migration does not own", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "OTHER=keep\n"); + await set("firebase-rounds", "8"); + + await clear({ yes: true }); + + expect(envFileContent()).toBe("OTHER=keep\n"); + }); +}); diff --git a/packages/cli-core/src/lib/dotenv.ts b/packages/cli-core/src/lib/dotenv.ts index 2b5d0cb20..16bc28187 100644 --- a/packages/cli-core/src/lib/dotenv.ts +++ b/packages/cli-core/src/lib/dotenv.ts @@ -28,6 +28,74 @@ export async function findExistingEnvFile(cwd: string, fallback: string): Promis return fallback; } +/** + * The env files read back when resolving a value, as opposed to written to. + * + * Deliberately shorter than {@link ENV_FILE_CANDIDATES}: the runtime has + * already loaded every `.env*` variant it recognises into `process.env`, which + * {@link findEnvValue} checks first. This list only has to cover the case where + * the CLI's own process did not load the file — a different cwd at startup, or + * a runtime with no dotenv support. + */ +const ENV_FILES = [".env", ".env.local"]; + +export interface FindEnvValueOptions { + /** Injectable in tests; defaults to the real environment. */ + env?: Record; + /** Lowest priority first — a later file overrides an earlier one. */ + files?: readonly string[]; +} + +export interface LocatedEnvValue { + value: string; + /** Where it came from, for `--verbose` (`CLERK_SECRET_KEY env var`, `.env.local`). */ + source: string; +} + +/** + * Looks for a value under any of `names`, in the order the app itself would + * resolve one: the environment first, then env files with a later file + * overriding an earlier one. + * + * This is the CLI's one way to read a project-level setting. Reading + * `process.env` directly instead skips the file fallback and reports no source, + * so a command that does it cannot explain where its input came from. + */ +export async function findEnvValue( + cwd: string, + names: string[], + options: FindEnvValueOptions = {}, +): Promise { + const { env = process.env, files = ENV_FILES } = options; + + for (const name of new Set(names)) { + const value = env[name]; + if (value) return { value, source: `${name} env var` }; + } + + // Priority is by name, not by position: the framework-specific name beats + // the generic fallback even when the generic one appears later in the same + // file. Within one name, a later file still overrides an earlier one. + const foundByName = new Map(); + for (const envFile of files) { + const file = Bun.file(join(cwd, envFile)); + if (!(await file.exists())) continue; + + for (const line of parseEnvFile(await file.text())) { + if (line.type !== "entry" || !line.value) continue; + if (names.includes(line.key)) { + foundByName.set(line.key, { value: line.value, source: envFile }); + } + } + } + + for (const name of names) { + const located = foundByName.get(name); + if (located) return located; + } + return undefined; +} + export type EnvLine = | { type: "comment"; raw: string } | { type: "blank" } diff --git a/packages/cli-core/src/lib/git.ts b/packages/cli-core/src/lib/git.ts index da4f4e955..697a57de8 100644 --- a/packages/cli-core/src/lib/git.ts +++ b/packages/cli-core/src/lib/git.ts @@ -1,4 +1,4 @@ -import { resolve } from "node:path"; +import { join, resolve } from "node:path"; import { log } from "./log.ts"; const $ = Bun.$; @@ -100,3 +100,22 @@ export function normalizeGitRemoteUrl(raw: string): string { return url.toLowerCase(); } + +/** + * Adds `entry` to the project's `.gitignore` unless it is already listed. + * + * The CLI writes files into a user's repository that must not be committed — + * the keyless breadcrumb, and the migration settings file. Creating one without + * this is how a live credential ends up in a tracked file. + */ +export async function ensureGitignoreEntry(cwd: string, entry: string): Promise { + const gitignorePath = join(cwd, ".gitignore"); + const content = await Bun.file(gitignorePath) + .text() + .catch(() => ""); + const lines = content.split("\n").map((l) => l.trim()); + if (lines.includes(entry)) return; + const separator = content && !content.endsWith("\n") ? "\n" : ""; + await Bun.write(gitignorePath, `${content}${separator}${entry}\n`); + log.debug(`git: added ${entry} to .gitignore`); +} diff --git a/packages/cli-core/src/lib/keyless-target.ts b/packages/cli-core/src/lib/keyless-target.ts index c1f7f5b08..82fa439d3 100644 --- a/packages/cli-core/src/lib/keyless-target.ts +++ b/packages/cli-core/src/lib/keyless-target.ts @@ -11,7 +11,7 @@ import { join } from "node:path"; import { bapiRequest } from "./bapi.ts"; import { resolveAppContext, resolveProfile } from "./config.ts"; import { getStoredSession, hasAccountCredentials, type OAuthSession } from "./credential-store.ts"; -import { parseEnvFile } from "./dotenv.ts"; +import { findEnvValue } from "./dotenv.ts"; import { CliError, ERROR_CODE, throwUsageError } from "./errors.ts"; import { decodePublishableKey } from "./fapi.ts"; import { detectPublishableKeyName, detectSecretKeyName } from "./framework.ts"; @@ -38,8 +38,6 @@ export type InstanceTarget = | { kind: "account"; ctx: AccountContext; label: string } | { kind: "keyless"; keyless: KeylessTarget; label: string }; -const ENV_FILES = [".env", ".env.local"]; - /** * Where the Clerk SDKs park the keys for a keyless app they created themselves * (running `next dev` with no keys configured). Shape: @@ -71,45 +69,6 @@ export async function readSdkKeylessApp( } } -interface LocatedKey { - value: string; - source: string; -} - -/** - * Looks for a key under any of `names`, in the order the app itself would - * resolve one: the environment first, then env files with a later file - * overriding an earlier one. - */ -async function findKeyInProject(cwd: string, names: string[]): Promise { - for (const name of new Set(names)) { - const value = process.env[name]; - if (value) return { value, source: `${name} env var` }; - } - - // Priority is by name, not by position: the framework-specific name beats - // the generic fallback even when the generic one appears later in the same - // file. Within one name, a later file still overrides an earlier one. - const foundByName = new Map(); - for (const envFile of ENV_FILES) { - const file = Bun.file(join(cwd, envFile)); - if (!(await file.exists())) continue; - - for (const line of parseEnvFile(await file.text())) { - if (line.type !== "entry" || !line.value) continue; - if (names.includes(line.key)) { - foundByName.set(line.key, { value: line.value, source: envFile }); - } - } - } - - for (const name of names) { - const located = foundByName.get(name); - if (located) return located; - } - return undefined; -} - /** * The instance secret key a keyless project keeps locally. Falls back to the * keys an SDK created for itself, which it only does when nothing else supplies @@ -117,7 +76,7 @@ async function findKeyInProject(cwd: string, names: string[]): Promise { const names = [await detectSecretKeyName(cwd), "CLERK_SECRET_KEY"]; - const located = await findKeyInProject(cwd, names); + const located = await findEnvValue(cwd, names); const found = located ? { secretKey: located.value, source: located.source } @@ -136,7 +95,7 @@ async function sdkKeylessTarget(cwd: string): Promise /** The publishable key a keyless project holds locally, when one can be found. */ export async function findLocalPublishableKey(cwd: string): Promise { const names = [await detectPublishableKeyName(cwd), "CLERK_PUBLISHABLE_KEY"]; - const located = await findKeyInProject(cwd, names); + const located = await findEnvValue(cwd, names); return located?.value ?? (await readSdkKeylessApp(cwd))?.publishableKey; } diff --git a/packages/cli-core/src/lib/keyless.ts b/packages/cli-core/src/lib/keyless.ts index 46db0a1a7..1e2e3ca96 100644 --- a/packages/cli-core/src/lib/keyless.ts +++ b/packages/cli-core/src/lib/keyless.ts @@ -5,6 +5,7 @@ import { detectPublishableKeyName, detectSecretKeyName, detectEnvFile } from "./ import { parseEnvFile, mergeEnvVars, serializeEnvFile } from "./dotenv.ts"; import { BapiError } from "./errors.ts"; import { loggedFetch } from "./fetch.ts"; +import { ensureGitignoreEntry } from "./git.ts"; import { log } from "./log.ts"; const BREADCRUMB_DIR = ".clerk"; @@ -113,18 +114,6 @@ function breadcrumbPath(cwd: string): string { return join(cwd, BREADCRUMB_DIR, BREADCRUMB_FILE); } -async function ensureGitignoreEntry(cwd: string, entry: string): Promise { - const gitignorePath = join(cwd, ".gitignore"); - const content = await Bun.file(gitignorePath) - .text() - .catch(() => ""); - const lines = content.split("\n").map((l) => l.trim()); - if (lines.includes(entry)) return; - const separator = content && !content.endsWith("\n") ? "\n" : ""; - await Bun.write(gitignorePath, `${content}${separator}${entry}\n`); - log.debug(`Added ${entry} to .gitignore`); -} - export async function writeKeylessBreadcrumb(cwd: string, claimToken: string): Promise { await ensureGitignoreEntry(cwd, BREADCRUMB_DIR + "/"); await mkdir(join(cwd, BREADCRUMB_DIR), { recursive: true }); diff --git a/packages/cli-core/src/test/integration/lib/harness.ts b/packages/cli-core/src/test/integration/lib/harness.ts index 2ad58c8a7..6853e9ca3 100644 --- a/packages/cli-core/src/test/integration/lib/harness.ts +++ b/packages/cli-core/src/test/integration/lib/harness.ts @@ -91,6 +91,7 @@ mock.module( getGitRepoIdentifier: async () => mockState.gitRepoIdentifier, getGitNormalizedRemote: async () => mockState.gitNormalizedRemote, normalizeGitRemoteUrl: (url: string) => url, + ensureGitignoreEntry: async () => {}, }) satisfies typeof import("../../../lib/git.ts"), ); From 194d8f6048a7f351afd98b0fb01816812bae9472 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 16:53:11 -0400 Subject: [PATCH 07/34] feat(migrate): explain each setting in `migrate settings`, and fix the column alignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The list showed seven kebab-case identifiers and nothing else, so `file` and `skip-unsupported-providers` read as jargon rather than as anything a user could act on. A description column now carries the prose the registry already held. The names stay kebab-case on purpose: each one is identical to the `migrate run` flag it backs, so `firebase-signer-key` and `--firebase-signer-key` are one knob reached two ways rather than two spellings to learn. Sentence case belongs in the description, which is where it now is. Also fixes the alignment. `column()` pads to the visible width before colouring; the previous code coloured first and then hand-compensated for the escape bytes with `dim("—").length - 1`, which only held for unset rows — any row with a value pulled `SOURCE` and everything after it out of line. --- .../cli-core/src/commands/migrate/README.md | 14 ++++-- .../src/commands/migrate/settings/list.ts | 50 +++++++++++++------ .../src/commands/migrate/settings/registry.ts | 16 ++++-- .../migrate/settings/settings.test.ts | 37 ++++++++++++++ 4 files changed, 94 insertions(+), 23 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 1e479a8f8..d8d9aeb63 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -453,13 +453,17 @@ clerk migrate settings clear -y ``` ``` -SETTING VALUE SOURCE -transformer firebase clerk config -file ./users.json clerk config -firebase-signer-key aVer…3456 .env.clerk-migrate -firebase-rounds — not set +SETTING VALUE SOURCE DESCRIPTION +transformer firebase clerk config Source platform the export came from +file users.json clerk config Export file to import users from +firebase-signer-key aVer…3456 .env.clerk-migrate Firebase base64 signer key +firebase-rounds — not set Firebase scrypt rounds ``` +Setting names are kebab-case and identical to the `migrate run` flag each one +backs, so `firebase-signer-key` here is `--firebase-signer-key` there rather +than a second spelling to learn. The description column carries the prose. + The source column is the point. A migration reads from flags, the environment, two of the app's env files and the CLI's config, so when a run picks up a stale value the question is never "what is it" but "which of those won". diff --git a/packages/cli-core/src/commands/migrate/settings/list.ts b/packages/cli-core/src/commands/migrate/settings/list.ts index 9fb1683d0..f5e9e43ec 100644 --- a/packages/cli-core/src/commands/migrate/settings/list.ts +++ b/packages/cli-core/src/commands/migrate/settings/list.ts @@ -46,6 +46,7 @@ function toJson(resolved: ResolvedSetting[]) { return resolved.map(({ setting, value, source }) => ({ name: setting.name, store: setting.store, + description: setting.description, // Redacted here too: `--json` is what gets piped into a log or a ticket. value: value === undefined ? null : displayValue(setting, value), set: value !== undefined, @@ -54,6 +55,16 @@ function toJson(resolved: ResolvedSetting[]) { })); } +/** + * Pads to a visible width, then colours. + * + * Colouring first and padding after would count the ANSI escape bytes towards + * the width and pull every later column left by however many they took. + */ +function column(text: string, width: number, paint: (value: string) => string): string { + return paint(text) + " ".repeat(Math.max(0, width - text.length)); +} + export async function list(options: SettingsListOptions = {}): Promise { const resolved = await resolveAll(); @@ -62,23 +73,34 @@ export async function list(options: SettingsListOptions = {}): Promise { return; } - const nameWidth = Math.max(...SETTINGS.map((s) => s.name.length), "SETTING".length) + 2; - const valueWidth = - Math.max( - ...resolved.map(({ setting, value }) => - value === undefined ? 1 : displayValue(setting, value).length, - ), - "VALUE".length, - ) + 2; + const cells = resolved.map(({ setting, value, source }) => ({ + setting, + name: setting.name, + value: value === undefined ? "—" : displayValue(setting, value), + unset: value === undefined, + source: source ?? "not set", + })); + + const width = (header: string, pick: (cell: (typeof cells)[number]) => string) => + Math.max(header.length, ...cells.map((cell) => pick(cell).length)) + 2; - log.info(dim("SETTING".padEnd(nameWidth)) + dim("VALUE".padEnd(valueWidth)) + dim("SOURCE")); + const nameWidth = width("SETTING", (c) => c.name); + const valueWidth = width("VALUE", (c) => c.value); + const sourceWidth = width("SOURCE", (c) => c.source); + + log.info( + column("SETTING", nameWidth, dim) + + column("VALUE", valueWidth, dim) + + column("SOURCE", sourceWidth, dim) + + dim("DESCRIPTION"), + ); - for (const { setting, value, source } of resolved) { - const shown = value === undefined ? dim("—") : displayValue(setting, value); + for (const cell of cells) { log.info( - cyan(setting.name.padEnd(nameWidth)) + - shown.padEnd(valueWidth + (value === undefined ? dim("—").length - 1 : 0)) + - dim(source ?? "not set"), + column(cell.name, nameWidth, cyan) + + column(cell.value, valueWidth, cell.unset ? dim : (value) => value) + + column(cell.source, sourceWidth, dim) + + dim(cell.setting.description), ); } diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index 8063607a2..9d358a5f5 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -17,7 +17,15 @@ export type SettingStore = "config" | "env"; export interface SettingDef { - /** What the user types: `clerk migrate settings set `. */ + /** + * What the user types: `clerk migrate settings set `. + * + * Kebab-case, and identical to the `migrate run` flag it backs. A setting and + * its flag are the same knob reached two ways, so `firebase-signer-key` here + * and `--firebase-signer-key` there must not drift into two spellings the + * user has to learn separately. Sentence-case prose belongs in + * `description`, which is what the list renders alongside it. + */ name: string; store: SettingStore; description: string; @@ -44,19 +52,19 @@ export const SETTINGS: SettingDef[] = [ name: "transformer", store: "config", configKey: "transformer", - description: "Source platform the last run imported from", + description: "Source platform the export came from", }, { name: "file", store: "config", configKey: "file", - description: "Export file the last run imported", + description: "Export file to import users from", }, { name: "skip-unsupported-providers", store: "config", configKey: "skipUnsupportedProviders", - description: "Supabase: skip users whose only social provider is disabled in Clerk", + description: "Supabase: skip users with no provider enabled in Clerk", validate: boolean, }, { diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index 9bee68233..b1a606789 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -139,6 +139,43 @@ describe("list", () => { ); }); + // The names are kebab-case because they mirror the `migrate run` flags; the + // description column is what makes the list readable. + test("explains each setting in prose", async () => { + await list(); + + expect(captured.err).toContain("Source platform the export came from"); + expect(captured.err).toContain("Export file to import users from"); + }); + + test("carries the description into JSON too", async () => { + await list({ json: true }); + + expect(JSON.parse(captured.out)).toContainEqual( + expect.objectContaining({ name: "file", description: "Export file to import users from" }), + ); + }); + + // Colouring before padding counts the ANSI bytes towards the column width, + // which pulls later columns left on exactly the rows that have a value. + test("starts the description at one column, set or not", async () => { + await set("transformer", "supabase"); + captured.clear(); + + await list(); + + const plain = captured.err.replaceAll(/\u001B\[\d+m/g, ""); + const columnOf = (description: string) => + plain + .split("\n") + .find((row) => row.includes(description)) + ?.indexOf(description); + + expect(columnOf("Source platform the export came from")).toBe( + columnOf("Export file to import users from") as number, + ); + }); + test("marks everything as unset in a fresh project", async () => { await list({ json: true }); expect(JSON.parse(captured.out).every((entry: { set: boolean }) => !entry.set)).toBe(true); From 909003bf91ae5dfaabbb6081d77e2cc62848ef24 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 16:59:29 -0400 Subject: [PATCH 08/34] feat(migrate): move the Supabase scope of `skip-unsupported-providers` to a trailing parenthetical Matches how the README heading already scopes it (`--skip-unsupported-providers (Supabase)`), and leaves the description reading as one sentence rather than a label plus a colon. --- packages/cli-core/src/commands/migrate/settings/registry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index 9d358a5f5..e034ec60d 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -64,7 +64,7 @@ export const SETTINGS: SettingDef[] = [ name: "skip-unsupported-providers", store: "config", configKey: "skipUnsupportedProviders", - description: "Supabase: skip users with no provider enabled in Clerk", + description: "Skip users with no provider enabled in Clerk (Supabase)", validate: boolean, }, { From 922890ad1bbe9fdad7f4e626d2c86f2708424c9c Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:11:42 -0400 Subject: [PATCH 09/34] feat(migrate): wrap the logs and transformers subcommands in the gutter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `migrate logs list|clean|convert` and `migrate transformers list` printed flush-left with no intro/outro frame, while `migrate run`, `migrate delete` and every `migrate export` already wrapped — as do the pre-existing commands they mirror (`apps list`, `mcp list`, `unlink`, `config pull`). Follows `commands/mcp/list.ts`: the `--json` early return stays outside the gutter so machine-readable output is unchanged, and only the human path wraps. `withGutter` no-ops outside human mode, so agent output is untouched, and it turns a cancelled prompt into `└ Paused` rather than `└ Failed` — which `logs clean` and `logs convert` both needed. --- .../src/commands/migrate/logs/clean.ts | 73 +++++++++--------- .../src/commands/migrate/logs/convert.ts | 75 ++++++++++--------- .../src/commands/migrate/logs/list.ts | 50 +++++++------ .../migrate/logs/logs-interactive.test.ts | 24 ++++++ .../src/commands/migrate/logs/logs.test.ts | 66 +++++++++++++--- .../migrate/transformers/list.test.ts | 30 ++++++++ .../src/commands/migrate/transformers/list.ts | 37 ++++----- 7 files changed, 234 insertions(+), 121 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/logs/clean.ts b/packages/cli-core/src/commands/migrate/logs/clean.ts index 32d60355d..3c609f5ad 100644 --- a/packages/cli-core/src/commands/migrate/logs/clean.ts +++ b/packages/cli-core/src/commands/migrate/logs/clean.ts @@ -13,6 +13,7 @@ import fs from "node:fs"; import { throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { confirm } from "../../../lib/prompts.ts"; +import { withGutter } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { listLogFiles } from "../lib/log-files.ts"; import { getLogDir } from "../lib/logger.ts"; @@ -22,48 +23,50 @@ export type LogsCleanOptions = { }; export async function clean(options: LogsCleanOptions = {}): Promise { - const files = listLogFiles(); + await withGutter("Cleaning migration logs", async () => { + const files = listLogFiles(); - if (files.length === 0) { - log.info(`No migration logs to clean in ${getLogDir()}.`); - return; - } + if (files.length === 0) { + log.info(`No migration logs to clean in ${getLogDir()}.`); + return; + } - const label = `${files.length} log file${files.length === 1 ? "" : "s"}`; + const label = `${files.length} log file${files.length === 1 ? "" : "s"}`; - if (!options.yes) { - if (isAgent() || !isHuman()) { - throwUsageError( - `\`clerk migrate logs clean\` deletes ${label} from ${getLogDir()} and cannot prompt here. Pass -y to confirm.`, - undefined, - undefined, - [ - { - command: "clerk migrate logs clean -y", - description: "Delete every migration log without prompting", - }, - ], - ); - } + if (!options.yes) { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate logs clean\` deletes ${label} from ${getLogDir()} and cannot prompt here. Pass -y to confirm.`, + undefined, + undefined, + [ + { + command: "clerk migrate logs clean -y", + description: "Delete every migration log without prompting", + }, + ], + ); + } - const proceed = await confirm({ message: `Delete ${label}?`, default: false }); - if (!proceed) throwUserAbort(); - } + const proceed = await confirm({ message: `Delete ${label}?`, default: false }); + if (!proceed) throwUserAbort(); + } - let deleted = 0; - const failures: string[] = []; + let deleted = 0; + const failures: string[] = []; - for (const file of files) { - try { - fs.unlinkSync(file.path); - deleted++; - } catch (error) { - failures.push(`${file.name}: ${(error as Error).message}`); + for (const file of files) { + try { + fs.unlinkSync(file.path); + deleted++; + } catch (error) { + failures.push(`${file.name}: ${(error as Error).message}`); + } } - } - for (const failure of failures) log.warn(`Could not delete ${failure}`); + for (const failure of failures) log.warn(`Could not delete ${failure}`); - log.success(`Deleted ${deleted} log file${deleted === 1 ? "" : "s"}.`); - if (failures.length > 0) process.exitCode = 1; + log.success(`Deleted ${deleted} log file${deleted === 1 ? "" : "s"}.`); + if (failures.length > 0) process.exitCode = 1; + }); } diff --git a/packages/cli-core/src/commands/migrate/logs/convert.ts b/packages/cli-core/src/commands/migrate/logs/convert.ts index 104a236fe..33adf2bae 100644 --- a/packages/cli-core/src/commands/migrate/logs/convert.ts +++ b/packages/cli-core/src/commands/migrate/logs/convert.ts @@ -12,6 +12,7 @@ import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../../ import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { multiselect } from "../../../lib/prompts.ts"; +import { withGutter } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { findLogFile, listLogFiles, readNdjson, type LogFile } from "../lib/log-files.ts"; import { getLogDir } from "../lib/logger.ts"; @@ -81,41 +82,47 @@ async function resolveTargets(options: LogsConvertOptions): Promise { } export async function convert(options: LogsConvertOptions = {}): Promise { - const targets = await resolveTargets(options); - if (targets.length === 0) return; - - let converted = 0; - let malformed = 0; - - for (const file of targets) { - const output = outputPathFor(file); - - try { - const { entries, errors } = readNdjson(file.path); - - // Reported per line, so a truncated final line from an interrupted run - // is visible rather than silently missing from the output. - for (const error of errors) { - malformed++; - log.warn(`${file.name}:${error.line} is not valid JSON and was skipped — ${error.message}`); + // The multiselect lives inside the gutter so cancelling it closes with + // `└ Paused` rather than leaving a half-drawn frame. + await withGutter("Converting migration logs", async () => { + const targets = await resolveTargets(options); + if (targets.length === 0) return; + + let converted = 0; + let malformed = 0; + + for (const file of targets) { + const output = outputPathFor(file); + + try { + const { entries, errors } = readNdjson(file.path); + + // Reported per line, so a truncated final line from an interrupted run + // is visible rather than silently missing from the output. + for (const error of errors) { + malformed++; + log.warn( + `${file.name}:${error.line} is not valid JSON and was skipped — ${error.message}`, + ); + } + + fs.writeFileSync(output, JSON.stringify(entries, null, 2)); + converted++; + const count = `${entries.length} ${entries.length === 1 ? "entry" : "entries"}`; + log.info(`${file.name} → ${output.split("/").pop()} ${dim(`(${count})`)}`); + } catch (error) { + log.warn(`Could not convert ${file.name}: ${(error as Error).message}`); + process.exitCode = 1; } - - fs.writeFileSync(output, JSON.stringify(entries, null, 2)); - converted++; - const count = `${entries.length} ${entries.length === 1 ? "entry" : "entries"}`; - log.info(`${file.name} → ${output.split("/").pop()} ${dim(`(${count})`)}`); - } catch (error) { - log.warn(`Could not convert ${file.name}: ${(error as Error).message}`); - process.exitCode = 1; } - } - if (converted > 0) { - log.success( - `Converted ${converted} log file${converted === 1 ? "" : "s"}. Originals left in place.`, - ); - } - if (malformed > 0) { - log.warn(`${malformed} malformed line${malformed === 1 ? "" : "s"} skipped.`); - } + if (converted > 0) { + log.success( + `Converted ${converted} log file${converted === 1 ? "" : "s"}. Originals left in place.`, + ); + } + if (malformed > 0) { + log.warn(`${malformed} malformed line${malformed === 1 ? "" : "s"} skipped.`); + } + }); } diff --git a/packages/cli-core/src/commands/migrate/logs/list.ts b/packages/cli-core/src/commands/migrate/logs/list.ts index cf30e24e0..1ef7f9405 100644 --- a/packages/cli-core/src/commands/migrate/logs/list.ts +++ b/packages/cli-core/src/commands/migrate/logs/list.ts @@ -8,6 +8,7 @@ import { cyan, dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; +import { withGutter } from "../../../lib/spinner.ts"; import { formatSize, listLogFiles, type LogFile } from "../lib/log-files.ts"; import { getLogDir } from "../lib/logger.ts"; @@ -26,7 +27,7 @@ function toJson(files: LogFile[]) { })); } -export function list(options: LogsListOptions = {}): void { +export async function list(options: LogsListOptions = {}): Promise { const files = listLogFiles(); if (options.json) { @@ -34,31 +35,34 @@ export function list(options: LogsListOptions = {}): void { return; } - if (files.length === 0) { - log.info(`No migration logs in ${getLogDir()}.`); - return; - } - - const kindWidth = Math.max(...files.map((file) => file.kind.length), "TYPE".length) + 2; - const timeWidth = Math.max(...files.map((file) => file.timestamp.length), "TIMESTAMP".length) + 2; - const sizeWidth = Math.max(...files.map((file) => formatSize(file.sizeBytes).length), 4) + 2; + await withGutter("Listing migration logs", async () => { + if (files.length === 0) { + log.info(`No migration logs in ${getLogDir()}.`); + return; + } - log.info( - dim("TYPE".padEnd(kindWidth)) + - dim("TIMESTAMP".padEnd(timeWidth)) + - dim("SIZE".padEnd(sizeWidth)) + - dim("ENTRIES"), - ); + const kindWidth = Math.max(...files.map((file) => file.kind.length), "TYPE".length) + 2; + const timeWidth = + Math.max(...files.map((file) => file.timestamp.length), "TIMESTAMP".length) + 2; + const sizeWidth = Math.max(...files.map((file) => formatSize(file.sizeBytes).length), 4) + 2; - for (const file of files) { log.info( - cyan(file.kind.padEnd(kindWidth)) + - (file.timestamp || dim("—")).padEnd(timeWidth) + - dim(formatSize(file.sizeBytes).padEnd(sizeWidth)) + - String(file.entryCount), + dim("TYPE".padEnd(kindWidth)) + + dim("TIMESTAMP".padEnd(timeWidth)) + + dim("SIZE".padEnd(sizeWidth)) + + dim("ENTRIES"), ); - } - log.info(""); - log.info(dim(`${files.length} log file${files.length === 1 ? "" : "s"} in ${getLogDir()}`)); + for (const file of files) { + log.info( + cyan(file.kind.padEnd(kindWidth)) + + (file.timestamp || dim("—")).padEnd(timeWidth) + + dim(formatSize(file.sizeBytes).padEnd(sizeWidth)) + + String(file.entryCount), + ); + } + + log.info(""); + log.info(dim(`${files.length} log file${files.length === 1 ? "" : "s"} in ${getLogDir()}`)); + }); } diff --git a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts index 36bd10747..4a7cf9eb1 100644 --- a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts @@ -175,3 +175,27 @@ describe("logs convert", () => { expect(mockMultiselect).not.toHaveBeenCalled(); }); }); + +// withGutter turns a UserAbortError into `└ Paused`; a real failure would close +// with `└ Failed`. Declining a prompt is not a failure, so the two must not swap. +describe("cancelling inside the gutter", () => { + test("declining the logs clean confirm closes with Paused, not Failed", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + mockConfirm.mockResolvedValue(false); + + await expect(clean()).rejects.toThrow(UserAbortError); + + expect(captured.err).toContain("Paused"); + expect(captured.err).not.toContain("Failed"); + }); + + test("selecting nothing in the logs convert multiselect closes with Paused", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([]); + + await expect(convert()).rejects.toThrow(UserAbortError); + + expect(captured.err).toContain("Paused"); + expect(captured.err).not.toContain("Failed"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/logs/logs.test.ts b/packages/cli-core/src/commands/migrate/logs/logs.test.ts index 5be2d232b..d0db18fb6 100644 --- a/packages/cli-core/src/commands/migrate/logs/logs.test.ts +++ b/packages/cli-core/src/commands/migrate/logs/logs.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { clean } from "./clean.ts"; @@ -42,21 +43,21 @@ const MIGRATION = "migration-2026-01-01T12-00-00.log"; const DELETION = "user-deletion-2026-02-01T12-00-00.log"; describe("logs list", () => { - test("says so plainly when there is no logs directory", () => { - list(); + test("says so plainly when there is no logs directory", async () => { + await list(); expect(captured.err).toContain("No migration logs in"); }); - test("says so plainly when the directory is empty", () => { + test("says so plainly when the directory is empty", async () => { fs.mkdirSync(getLogDir(), { recursive: true }); - list(); + await list(); expect(captured.err).toContain("No migration logs in"); }); - test("reports type, timestamp, size and entry count", () => { + test("reports type, timestamp, size and entry count", async () => { writeLog(MIGRATION, [{ userId: "u1" }, { userId: "u2" }, { userId: "u3" }]); - list(); + await list(); expect(captured.err).toContain("TYPE"); expect(captured.err).toContain("TIMESTAMP"); @@ -68,21 +69,21 @@ describe("logs list", () => { expect(captured.err).toContain("3"); }); - test("lists every log kind", () => { + test("lists every log kind", async () => { writeLog(MIGRATION, [{ a: 1 }]); writeLog(DELETION, [{ a: 1 }]); - list(); + await list(); expect(captured.err).toContain("migration"); expect(captured.err).toContain("deletion"); expect(captured.err).toContain("2 log files"); }); - test("--json emits a machine-readable listing on stdout", () => { + test("--json emits a machine-readable listing on stdout", async () => { writeLog(MIGRATION, [{ userId: "u1" }]); - list({ json: true }); + await list({ json: true }); const parsed = JSON.parse(captured.out) as Record[]; expect(parsed).toHaveLength(1); @@ -94,8 +95,8 @@ describe("logs list", () => { }); }); - test("--json emits an empty array rather than prose when there are no logs", () => { - list({ json: true }); + test("--json emits an empty array rather than prose when there are no logs", async () => { + await list({ json: true }); expect(JSON.parse(captured.out)).toEqual([]); }); }); @@ -217,3 +218,44 @@ describe("logs convert", () => { expect(captured.err).toContain("3 entries"); }); }); + +describe("human-mode frame", () => { + let originalMode: Mode; + + beforeAll(() => { + originalMode = getMode(); + setMode("human"); + }); + + afterAll(() => { + setMode(originalMode); + }); + + test("logs list wraps its output in an intro/outro gutter", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await list(); + + expect(captured.err).toContain("\u250c"); + expect(captured.err).toContain("Listing migration logs"); + expect(captured.err).toContain("\u2514"); + expect(captured.err).toContain("Done"); + }); + + test("--json stays outside the gutter, on stdout only", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await list({ json: true }); + + expect(JSON.parse(captured.out)).toHaveLength(1); + expect(captured.err).not.toContain("\u250c"); + }); + + test("a failure inside logs convert closes with Failed and still throws", async () => { + writeLog(MIGRATION, [{ a: 1 }]); + + await expect(convert({ files: ["nope.log"] })).rejects.toThrow(CliError); + + expect(captured.err).toContain("Failed"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/list.test.ts b/packages/cli-core/src/commands/migrate/transformers/list.test.ts index 9f432e1d9..26179ee9f 100644 --- a/packages/cli-core/src/commands/migrate/transformers/list.test.ts +++ b/packages/cli-core/src/commands/migrate/transformers/list.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { list } from "./list.ts"; import { transformers } from "./registry.ts"; @@ -114,3 +115,32 @@ describe("a bad --transformer-file", () => { await expect(list({ transformerFile: "./nope.ts" })).rejects.toThrow(CliError); }); }); + +describe("human-mode frame", () => { + let originalMode: Mode; + + beforeAll(() => { + originalMode = getMode(); + setMode("human"); + }); + + afterAll(() => { + setMode(originalMode); + }); + + test("wraps its output in an intro/outro gutter", async () => { + await list(); + + expect(captured.err).toContain("┌"); + expect(captured.err).toContain("Listing transformers"); + expect(captured.err).toContain("└"); + expect(captured.err).toContain("Done"); + }); + + test("--json stays outside the gutter, on stdout only", async () => { + await list({ json: true }); + + expect(() => JSON.parse(captured.out)).not.toThrow(); + expect(captured.err).not.toContain("┌"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/list.ts b/packages/cli-core/src/commands/migrate/transformers/list.ts index 349a53a8f..37da1e0a5 100644 --- a/packages/cli-core/src/commands/migrate/transformers/list.ts +++ b/packages/cli-core/src/commands/migrate/transformers/list.ts @@ -8,6 +8,7 @@ import { bold, cyan, dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; +import { withGutter } from "../../../lib/spinner.ts"; import type { TransformerRegistryEntry } from "../types.ts"; import { loadCustomTransformer } from "./load-custom.ts"; import { transformers } from "./registry.ts"; @@ -44,24 +45,26 @@ export async function list(options: TransformersListOptions = {}): Promise return; } - for (const entry of entries) { - const suffix = entry.builtIn ? "" : ` ${dim(`(custom — ${entry.source})`)}`; - log.info(`${cyan(bold(entry.key))} ${entry.label}${suffix}`); - log.info(` ${dim(entry.description)}`); - log.info(""); - } - - const custom = entries.length - transformers.length; - log.info( - dim( - `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + - (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), - ), - ); + await withGutter("Listing transformers", async () => { + for (const entry of entries) { + const suffix = entry.builtIn ? "" : ` ${dim(`(custom — ${entry.source})`)}`; + log.info(`${cyan(bold(entry.key))} ${entry.label}${suffix}`); + log.info(` ${dim(entry.description)}`); + log.info(""); + } - if (custom === 0) { + const custom = entries.length - transformers.length; log.info( - dim("Migrating from something else? Write a transformer and pass --transformer-file."), + dim( + `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + + (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), + ), ); - } + + if (custom === 0) { + log.info( + dim("Migrating from something else? Write a transformer and pass --transformer-file."), + ); + } + }); } From 05b65365b2fab5707621294d380e3b978985d603 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:13:51 -0400 Subject: [PATCH 10/34] feat(migrate): follow the CLI's spinner, next-steps, pluralization and icon conventions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five formatting mismatches between `migrate` and every command around it. **Summaries escaped the gutter.** `run` and `delete` printed their final summary through `log.raw`, the un-prefixed channel meant for machine-readable output, so it landed flush-left and broke the `┌ … │ … └` frame. `log.info` runs each line through `applyPrefix`, so only the channel changes. **Spinners lost the `...` convention.** All 39 pre-existing `withSpinner` call sites end their message in `...` and none passes a done-message — the vanishing ellipsis *is* the completion signal. Migrate had inverted both halves. Adds the ellipsis to every message and `spinner.update()` string, and drops the four done-message arguments so the stop text derives from the message like everywhere else. **No Next steps blocks.** Every comparable command closes with one. Adds `MIGRATE_DONE`, `MIGRATE_DELETE` and `MIGRATE_EXPORT`, wired through the gutter's `setNextSteps`. `reportExport` now returns the steps rather than hand-rolling a `dim("Next: …")` line, which keeps the six export modules' call shape intact. An export of zero users returns none — there is nothing to import — so `setNextSteps` ignores an empty list instead of rendering a header with no bullets under it. **`user(s)` pluralization.** The CLI's form is `${n} thing${n === 1 ? "" : "s"}`. Migrate used `user(s)`/`row(s)` in ten places while using the correct form in others. **`●`/`○` status icons**, which appear nowhere else in the codebase, become the established `✓`/`✗`/`!` vocabulary from `doctor` and `init`. --- .../cli-core/src/commands/migrate/README.md | 15 ++--- .../src/commands/migrate/delete.test.ts | 4 +- .../cli-core/src/commands/migrate/delete.ts | 29 +++++----- .../src/commands/migrate/export/auth0.test.ts | 11 +++- .../src/commands/migrate/export/auth0.ts | 28 +++++----- .../src/commands/migrate/export/authjs.ts | 22 ++++---- .../src/commands/migrate/export/betterauth.ts | 20 ++++--- .../src/commands/migrate/export/clerk.test.ts | 38 ++++++++++++- .../src/commands/migrate/export/clerk.ts | 26 ++++----- .../migrate/export/db-exports.test.ts | 2 +- .../commands/migrate/export/firebase.test.ts | 11 +++- .../src/commands/migrate/export/firebase.ts | 30 +++++----- .../src/commands/migrate/export/shared.ts | 26 +++++---- .../src/commands/migrate/export/supabase.ts | 20 ++++--- .../src/commands/migrate/import-users.ts | 2 +- .../src/commands/migrate/lib/readiness.ts | 4 +- .../cli-core/src/commands/migrate/run.test.ts | 6 +- packages/cli-core/src/commands/migrate/run.ts | 56 ++++++++++--------- .../src/commands/migrate/settings/clear.ts | 4 +- packages/cli-core/src/lib/next-steps.ts | 10 ++++ packages/cli-core/src/lib/spinner.ts | 4 +- 21 files changed, 227 insertions(+), 141 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index d8d9aeb63..f74ab3606 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -165,13 +165,14 @@ import it, not after: ``` Field coverage - ● 3/3 have an email address - ○ 0/3 have a phone number - ○ 1/3 have a username - ○ 2/3 have a password (not exportable — see below) - -Exported 3 user(s) to /project/exports/clerk-export.json -Next: clerk migrate run --transformer clerk --file exports/clerk-export.json + ✓ 3/3 have an email address + ✗ 0/3 have a phone number + ! 1/3 have a username + ! 2/3 have a password (not exportable — see below) + +Exported 3 users to /project/exports/clerk-export.json +└ Next steps + → Run `clerk migrate run --transformer clerk --file exports/clerk-export.json` to import them ``` Every export also writes `logs/export-.log`, so `migrate logs list` diff --git a/packages/cli-core/src/commands/migrate/delete.test.ts b/packages/cli-core/src/commands/migrate/delete.test.ts index ef110ed00..01711ec97 100644 --- a/packages/cli-core/src/commands/migrate/delete.test.ts +++ b/packages/cli-core/src/commands/migrate/delete.test.ts @@ -382,7 +382,7 @@ describe("deleteMigration", () => { await deleteMigration(baseOptions); expect(deleteCalls()).toEqual([expect.stringContaining("/v1/users/user_1")]); - expect(captured.err).toContain("1 of the file's user(s) are not in this instance"); + expect(captured.err).toContain("1 of the file's users is not in this instance"); }); test("does nothing when none of the migration's users are present", async () => { @@ -399,7 +399,7 @@ describe("deleteMigration", () => { stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); await expect(deleteMigration({ secretKey: "sk_test_x" })).rejects.toThrow( - /permanently deletes 2 user\(s\) and cannot prompt here/, + /permanently deletes 2 users and cannot prompt here/, ); expect(deleteCalls()).toHaveLength(0); }); diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 8b14abe55..db2791205 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -31,6 +31,7 @@ import { } from "../../lib/errors.ts"; import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; import { log } from "../../lib/log.ts"; +import { NEXT_STEPS } from "../../lib/next-steps.ts"; import { confirm } from "../../lib/prompts.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; @@ -130,7 +131,7 @@ export async function findMigratedUsers(options: { const batches = batch(options.externalIds, EXTERNAL_ID_BATCH); for (const [index, ids] of batches.entries()) { - options.spinner?.update(`Finding migrated users: batch ${index + 1}/${batches.length}`); + options.spinner?.update(`Finding migrated users: batch ${index + 1}/${batches.length}...`); const params = new URLSearchParams(); params.set("limit", String(EXTERNAL_ID_BATCH)); @@ -181,7 +182,7 @@ export async function deleteMigratedUsers(options: { const progress = () => spinner?.update( - `Deleting users: [${processed}/${users.length}] (${deleted} deleted, ${failed} failed)`, + `Deleting users: [${processed}/${users.length}] (${deleted} deleted, ${failed} failed)...`, ); // A failure on one user must not abort the rest: a half-undone migration @@ -264,7 +265,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise { + await withGutter("Undoing a migration", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); const limits = resolveLimits(secretKey); @@ -277,15 +278,13 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise findMigratedUsers({ externalIds, secretKey, spinner }), - "Search complete", + const users = await withSpinner("Finding migrated users...", (spinner) => + findMigratedUsers({ externalIds, secretKey, spinner }), ); if (users.length === 0) { log.info( - `None of the ${externalIds.length} user(s) in ${file} are in ${target ?? "this instance"}. Nothing to delete.`, + `None of the ${externalIds.length} user${externalIds.length === 1 ? "" : "s"} in ${file} are in ${target ?? "this instance"}. Nothing to delete.`, ); return; } @@ -297,7 +296,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise deleteMigratedUsers({ users, secretKey, limits, dateTime, spinner }), - "Deletion complete", + const summary = await withSpinner(`Deleting users: [0/${users.length}]...`, (spinner) => + deleteMigratedUsers({ users, secretKey, limits, dateTime, spinner }), ); - log.raw(formatSummary(summary, logFile)); + log.info(formatSummary(summary, logFile)); + + setNextSteps(NEXT_STEPS.MIGRATE_DELETE); if (summary.failed > 0) process.exitCode = 1; }); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index 58ce5add2..ceef2915a 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -1,4 +1,5 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -290,7 +291,15 @@ describe("exportAuth0", () => { test("names the command that consumes the file", async () => { stubAuth0([[auth0User(0)], []]); - await exportAuth0({ ...CREDENTIALS }); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + await exportAuth0({ ...CREDENTIALS }); + } finally { + setMode(originalMode); + } expect(captured.err).toContain( "migrate run --transformer auth0 --file exports/auth0-export.json", ); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index 3d90d661c..bf1657267 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -219,7 +219,7 @@ export async function fetchAllAuth0Users(options: { for (let page = 0; ; page++) { const { users, total } = await fetchAuth0Page(options.credentials, options.token, page); all.push(...users); - options.spinner?.update(`Fetching users from Auth0: ${all.length} so far`); + options.spinner?.update(`Fetching users from Auth0: ${all.length} so far...`); if (users.length < PAGE_SIZE) break; @@ -316,30 +316,30 @@ export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0Exp export async function exportAuth0(options: ExportAuth0Options): Promise { const credentials = await resolveAuth0Credentials(options); - await withGutter("Exporting users from Auth0", async () => { + await withGutter("Exporting users from Auth0", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); log.info(`Exporting from ${credentials.domain}.`); - const token = await withSpinner("Authenticating with Auth0", () => + const token = await withSpinner("Authenticating with Auth0...", () => fetchAuth0Token(credentials), ); - const users = await withSpinner( - "Fetching users from Auth0", - (spinner) => fetchAllAuth0Users({ credentials, token, spinner }), - "Users fetched", + const users = await withSpinner("Fetching users from Auth0...", (spinner) => + fetchAllAuth0Users({ credentials, token, spinner }), ); const { users: exported, coverage } = buildAuth0Export(users, dateTime); const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("auth0")); - reportExport({ - platform: "auth0", - userCount: exported.length, - outputPath, - coverage, - transformerKey: "auth0", - }); + setNextSteps( + reportExport({ + platform: "auth0", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "auth0", + }), + ); if (exported.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index d8183d981..a78df15de 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -113,24 +113,26 @@ export async function exportAuthJs(options: DbExportOptions): Promise { hint: "Postgres, MySQL or a SQLite file — whichever your Auth.js adapter uses.", }); - await withGutter("Exporting users from Auth.js", async () => { + await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); - const { rows, table } = await withSpinner("Reading the user table", () => + const { rows, table } = await withSpinner("Reading the user table...", () => withDbClient(dbUrl, "authjs", fetchAuthJsUsers), ); - log.info(`Read ${rows.length} row(s) from ${table}.`); + log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); const { users, coverage } = buildAuthJsExport(rows, dateTime); const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("authjs")); - reportExport({ - platform: "authjs", - userCount: users.length, - outputPath, - coverage, - transformerKey: "authjs", - }); + setNextSteps( + reportExport({ + platform: "authjs", + userCount: users.length, + outputPath, + coverage, + transformerKey: "authjs", + }), + ); if (users.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index e2dcad93e..85ab190db 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -160,10 +160,10 @@ export async function exportBetterAuth(options: DbExportOptions): Promise hint: "Postgres, MySQL or a SQLite file — whichever your Better Auth install uses.", }); - await withGutter("Exporting users from Better Auth", async () => { + await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); - const { rows, plugins } = await withSpinner("Reading the user table", () => + const { rows, plugins } = await withSpinner("Reading the user table...", () => withDbClient(dbUrl, "betterauth", async (client) => { const plugins = await detectPluginColumns(client); const rows = await client.query(buildBetterAuthQuery(client, plugins)); @@ -180,12 +180,14 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const { users, coverage } = buildBetterAuthExport(rows, dateTime); const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("betterauth")); - reportExport({ - platform: "betterauth", - userCount: users.length, - outputPath, - coverage, - transformerKey: "betterauth", - }); + setNextSteps( + reportExport({ + platform: "betterauth", + userCount: users.length, + outputPath, + coverage, + transformerKey: "betterauth", + }), + ); }); } diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 12a3062db..9ce14d656 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -1,4 +1,5 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -240,12 +241,20 @@ describe("exportClerk", () => { expect(written).toHaveLength(1); expect(written[0]?.id).toBe("u1"); expect(captured.err).toContain("Field coverage"); - expect(captured.err).toContain("Exported 1 user(s)"); + expect(captured.err).toContain("Exported 1 user"); }); test("names the command that consumes the file", async () => { stubPages([[user()], []]); - await exportClerk({ secretKey: "sk_test_x" }); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + await exportClerk({ secretKey: "sk_test_x" }); + } finally { + setMode(originalMode); + } expect(captured.err).toContain( "migrate run --transformer clerk --file exports/clerk-export.json", ); @@ -278,4 +287,29 @@ describe("exportClerk", () => { JSON.parse(fs.readFileSync(path.join(workDir, "exports", "clerk-export.json"), "utf-8")), ).toEqual([]); }); + + test("an empty export warns but does not suggest importing it", async () => { + stubPages([[]]); + const originalMode = getMode(); + setMode("human"); + try { + await exportClerk({ secretKey: "sk_test_x" }); + } finally { + setMode(originalMode); + } + + expect(captured.err).toContain("No users found to export"); + expect(captured.err).not.toContain("Next steps"); + expect(captured.err).not.toContain("migrate run --transformer"); + }); + + test("agent mode suppresses the Next steps block", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("Exported 1 user"); + expect(captured.err).not.toContain("Next steps"); + expect(captured.err).not.toContain("migrate run --transformer"); + }); }); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 37c10d802..a49fa581e 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -172,7 +172,7 @@ export async function fetchAllClerkUsers(options: { const page = Array.isArray(response.body) ? (response.body as BapiUser[]) : []; all.push(...page); - options.spinner?.update(`Fetching users from Clerk: ${all.length} so far`); + options.spinner?.update(`Fetching users from Clerk: ${all.length} so far...`); // A short page means the end; anything else would loop forever on an // instance whose size happens to be a multiple of the page size. @@ -229,29 +229,29 @@ export async function exportClerk(options: ExportClerkOptions): Promise { } const secretKeyOption = options.secretKey ?? options.clerkSecretKey; - await withGutter("Exporting users from Clerk", async () => { + await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); const dateTime = getDateTimeStamp(); log.info(`Exporting from ${target ?? "the resolved instance"}.`); - const users = await withSpinner( - "Fetching users from Clerk", - (spinner) => fetchAllClerkUsers({ secretKey, spinner }), - "Users fetched", + const users = await withSpinner("Fetching users from Clerk...", (spinner) => + fetchAllClerkUsers({ secretKey, spinner }), ); const { users: exported, coverage } = buildClerkExport(users, dateTime); const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("clerk")); - reportExport({ - platform: "clerk", - userCount: exported.length, - outputPath, - coverage, - transformerKey: "clerk", - }); + setNextSteps( + reportExport({ + platform: "clerk", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "clerk", + }), + ); if (exported.length > 0) { log.warn( diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 15116d255..08816b69a 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -213,7 +213,7 @@ describe("authjs export", () => { const written = JSON.parse(fs.readFileSync(path.join(workDir, "authjs.json"), "utf-8")); expect(written).toHaveLength(2); - expect(captured.err).toContain("Read 2 row(s) from"); + expect(captured.err).toContain("Read 2 rows from"); expect(captured.err).toContain("stores no passwords"); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 3772e35de..137d0502d 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -1,4 +1,5 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -414,7 +415,15 @@ describe("exportFirebase", () => { test("names the command that consumes the file", async () => { stubFirebase([[fbUser(0)]], { signIn: {} }); - await exportFirebase({ serviceAccount: "./sa.json" }); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + await exportFirebase({ serviceAccount: "./sa.json" }); + } finally { + setMode(originalMode); + } expect(captured.err).toContain( "migrate run --transformer firebase --file exports/firebase-export.json", ); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 0491227ae..9c70a410b 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -255,7 +255,7 @@ export async function fetchAllFirebaseUsers(options: { const body = (await response.json()) as { users?: FirebaseUser[]; nextPageToken?: string }; all.push(...(body.users ?? [])); - options.spinner?.update(`Fetching users from Firebase: ${all.length} so far`); + options.spinner?.update(`Fetching users from Firebase: ${all.length} so far...`); pageToken = body.nextPageToken; } while (pageToken); @@ -424,28 +424,30 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { + await withGutter("Exporting users from Firebase", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); log.info(`Exporting from the ${account.project_id} project.`); - const token = await withSpinner("Authenticating with Google", () => fetchAccessToken(account)); + const token = await withSpinner("Authenticating with Google...", () => + fetchAccessToken(account), + ); - const users = await withSpinner( - "Fetching users from Firebase", - (spinner) => fetchAllFirebaseUsers({ account, token, spinner }), - "Users fetched", + const users = await withSpinner("Fetching users from Firebase...", (spinner) => + fetchAllFirebaseUsers({ account, token, spinner }), ); const { users: exported, coverage } = buildFirebaseExport(users, dateTime); const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("firebase")); - reportExport({ - platform: "firebase", - userCount: exported.length, - outputPath, - coverage, - transformerKey: "firebase", - }); + setNextSteps( + reportExport({ + platform: "firebase", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "firebase", + }), + ); const passwordCount = coverage.find((entry) => entry.label.includes("password"))?.count ?? 0; const hashConfig = passwordCount > 0 ? await fetchHashConfig(account, token) : null; diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index aa25ea50f..e880d90d3 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -13,6 +13,7 @@ import fs from "node:fs"; import path from "node:path"; import { dim, green, yellow } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; +import { NEXT_STEPS } from "../../../lib/next-steps.ts"; /** Where an export lands when `--output` is not given. */ export function defaultOutputPath(platform: string): string { @@ -36,13 +37,13 @@ export type CoverageField = { label: string; count: number }; /** * How complete an export is, per field. * - * ● every user, ○ some, dim ○ none. The point is to see *before* importing + * ✓ every user, ! some, dim ✗ none. The point is to see *before* importing * that, say, only 3 of 400 users have a password — which changes what the * migration means. */ export function formatFieldCoverage(fields: CoverageField[], total: number): string[] { return fields.map(({ label, count }) => { - const icon = count === total ? green("●") : count > 0 ? yellow("○") : dim("○"); + const icon = count === total ? green("✓") : count > 0 ? yellow("!") : dim("✗"); return ` ${icon} ${dim(`${count}/${total} ${label}`)}`; }); } @@ -56,12 +57,18 @@ export type ExportSummary = { transformerKey: string; }; -/** Reports the coverage table and the exact command that consumes the file. */ -export function reportExport(summary: ExportSummary): void { +/** + * Reports the coverage table. + * + * @returns The next steps for the caller to hand to `setNextSteps`, so the + * suggested import command closes the gutter like every other command's. + * Empty when nothing was exported — there is nothing to import. + */ +export function reportExport(summary: ExportSummary): readonly string[] { log.blank(); if (summary.userCount === 0) { log.warn(`No users found to export. Wrote an empty file to ${summary.outputPath}.`); - return; + return []; } log.info("Field coverage"); @@ -70,12 +77,11 @@ export function reportExport(summary: ExportSummary): void { } log.blank(); - log.success(`Exported ${summary.userCount} user(s) to ${summary.outputPath}`); - log.info( - dim( - `Next: clerk migrate run --transformer ${summary.transformerKey} --file ${relativeIfInside(summary.outputPath)}`, - ), + log.success( + `Exported ${summary.userCount} user${summary.userCount === 1 ? "" : "s"} to ${summary.outputPath}`, ); + + return NEXT_STEPS.MIGRATE_EXPORT(summary.transformerKey, relativeIfInside(summary.outputPath)); } /** Shortens a path for display when it sits under the working directory. */ diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 585ebf9d9..80ff3785e 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -111,23 +111,25 @@ export async function exportSupabase(options: DbExportOptions): Promise { hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", }); - await withGutter("Exporting users from Supabase", async () => { + await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); - const rows = await withSpinner("Reading auth.users", () => + const rows = await withSpinner("Reading auth.users...", () => withDbClient(dbUrl, "supabase", fetchSupabaseUsers), ); const { users, coverage } = buildSupabaseExport(rows, dateTime); const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("supabase")); - reportExport({ - platform: "supabase", - userCount: users.length, - outputPath, - coverage, - transformerKey: "supabase", - }); + setNextSteps( + reportExport({ + platform: "supabase", + userCount: users.length, + outputPath, + coverage, + transformerKey: "supabase", + }), + ); if (users.length > 0) { log.info( diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts index 749142cf4..80402b6d9 100644 --- a/packages/cli-core/src/commands/migrate/import-users.ts +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -299,7 +299,7 @@ export async function importUsers(options: ImportUsersOptions): Promise spinner?.update( - `Importing users: [${processed}/${total}] (${successful} succeeded, ${failed} failed)`, + `Importing users: [${processed}/${total}] (${successful} succeeded, ${failed} failed)...`, ); const recordFailure = (userId: string, message: string, code: string) => { diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts index 353823415..1db17fe3e 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -190,7 +190,7 @@ function renderItem(item: ReadinessItem, total: number): string { return ` ${green("✓")} ${item.label} — ${dim(`enabled in Clerk — ${coverage}`)}`; } // Settings unavailable: state coverage without claiming anything about Clerk. - return ` ${yellow("○")} ${item.label} — ${dim(`${coverage} — check it is enabled in Clerk`)}`; + return ` ${yellow("!")} ${item.label} — ${dim(`${coverage} — check it is enabled in Clerk`)}`; } /** Renders the report for a human, as lines. */ @@ -210,7 +210,7 @@ export function formatReadinessReport(report: ReadinessReport): string[] { if (report.settingsUnavailable) { lines.push( "", - ` ${yellow("○")} ${dim("Could not read this instance's settings, so the checks below are coverage only.")}`, + ` ${yellow("!")} ${dim("Could not read this instance's settings, so the checks below are coverage only.")}`, ` ${dim(` Verify your settings at ${DASHBOARD_URL}`)}`, ); } diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 330d70645..1698a2517 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -263,7 +263,7 @@ describe("run", () => { const created = requests.filter((r) => r.url.endsWith("/v1/users")); expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u1"]); - expect(captured.err).toContain("skipping 1 user(s) without a password"); + expect(captured.err).toContain("skipping 1 user without a password"); }); test("--resume-after skips everyone up to and including that ID", async () => { @@ -279,7 +279,7 @@ describe("run", () => { await run(baseOptions); expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); - expect(captured.err).toContain("1 user(s) failed validation"); + expect(captured.err).toContain("1 user failed validation"); }); test("warns that --clerk-secret-key is deprecated but still honours it", async () => { @@ -750,7 +750,7 @@ describe("run", () => { await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); expect(created()).toEqual(["sb_email", "sb_both"]); - expect(captured.err).toContain("skipping 1 user(s)"); + expect(captured.err).toContain("skipping 1 user "); expect(captured.err).toContain("discord: 1"); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 9b6e9bdf0..7167b854b 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -14,6 +14,7 @@ import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command import { bold, dim, green, red, yellow } from "../../lib/color.ts"; import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; import { log } from "../../lib/log.ts"; +import { NEXT_STEPS } from "../../lib/next-steps.ts"; import { confirm } from "../../lib/prompts.ts"; import { withGutter, withSpinner } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; @@ -270,7 +271,7 @@ async function skipDisabledProviderUsers( return users; } - const settings = await withSpinner("Checking enabled providers", () => + const settings = await withSpinner("Checking enabled providers...", () => fetchInstanceSettings(secretKey), ); const enabled = settings ? enabledSocialProviders(settings) : null; @@ -300,7 +301,7 @@ async function skipDisabledProviderUsers( .map(([provider, count]) => `${provider}: ${count}`) .join(", "); log.warn( - `--skip-unsupported-providers: skipping ${excludedIds.size} user(s) whose only provider is not enabled in Clerk (${breakdown}).`, + `--skip-unsupported-providers: skipping ${excludedIds.size} user${excludedIds.size === 1 ? "" : "s"} whose only provider is not enabled in Clerk (${breakdown}).`, ); return users.filter((user) => !excludedIds.has(user.userId)); @@ -328,7 +329,7 @@ async function showReadinessReport(input: { }): Promise { if (input.skipReport) return; - const settings = await withSpinner("Checking instance settings", () => + const settings = await withSpinner("Checking instance settings...", () => fetchInstanceSettings(input.secretKey), ); @@ -443,7 +444,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const { transformer, file } = validateRunOptions(options); const firebaseHashConfig = await resolveFirebaseHashConfig(options); - await withGutter("Migrating users to Clerk", async () => { + await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); const limits = resolveLimits(secretKey); @@ -451,9 +452,8 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const logFile = getLogFilePath("migration", dateTime); const { users: loaded, validationFailed } = await withSpinner( - `Loading users from ${file}`, + `Loading users from ${file}...`, () => loadUsersFromFile(file, transformer, dateTime, { context: { firebaseHashConfig } }), - "Users loaded", ); let users = applyResumeAfter(loaded, options.resumeAfter); @@ -469,14 +469,16 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const withPassword = users.filter((user) => Boolean(user.password)); const dropped = users.length - withPassword.length; if (dropped > 0) { - log.info(`--require-password: skipping ${dropped} user(s) without a password.`); + log.info( + `--require-password: skipping ${dropped} user${dropped === 1 ? "" : "s"} without a password.`, + ); } users = withPassword; } if (validationFailed > 0) { log.warn( - `${validationFailed} user(s) failed validation and will be skipped. See ${logFile}.`, + `${validationFailed} user${validationFailed === 1 ? "" : "s"} failed validation and will be skipped. See ${logFile}.`, ); } @@ -493,9 +495,12 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ); } + // `target` already carries the instance's environment ("My App + // (development)"), so the detected type is only worth spelling out when + // there is no app context to name — an explicit `--secret-key`. log.info( - `Importing ${users.length} user(s) via the ${transformer} transformer into ` + - `${target ?? "the resolved instance"} (${limits.instanceType}).`, + `Importing ${users.length} user${users.length === 1 ? "" : "s"} via the ${transformer} transformer into ` + + `${target ?? `the resolved instance (${limits.instanceType})`}.`, ); await showReadinessReport({ @@ -509,7 +514,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (!options.yes && isHuman() && !isAgent()) { const proceed = await confirm({ - message: `Import ${users.length} user(s)?`, + message: `Import ${users.length} user${users.length === 1 ? "" : "s"}?`, default: false, }); if (!proceed) throwUserAbort(); @@ -523,22 +528,23 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ...(options.skipUnsupportedProviders ? { skipUnsupportedProviders: true } : {}), }); - const summary = await withSpinner( - `Importing users: [0/${users.length}]`, - (spinner) => - importUsers({ - users, - secretKey, - limits, - dateTime, - skipPasswordRequirement: !options.requirePassword, - validationFailed, - spinner, - }), - "Import complete", + const summary = await withSpinner(`Importing users: [0/${users.length}]...`, (spinner) => + importUsers({ + users, + secretKey, + limits, + dateTime, + skipPasswordRequirement: !options.requirePassword, + validationFailed, + spinner, + }), ); - log.raw(formatSummary(summary, logFile)); + log.info(formatSummary(summary, logFile)); + + // Offered even when some users failed: a partial import is exactly when + // reading the log and knowing how to undo it matters most. + setNextSteps(NEXT_STEPS.MIGRATE_DONE); if (summary.failed > 0) process.exitCode = 1; }); diff --git a/packages/cli-core/src/commands/migrate/settings/clear.ts b/packages/cli-core/src/commands/migrate/settings/clear.ts index 933d1e540..cb588e0eb 100644 --- a/packages/cli-core/src/commands/migrate/settings/clear.ts +++ b/packages/cli-core/src/commands/migrate/settings/clear.ts @@ -53,6 +53,8 @@ export async function clear(options: SettingsClearOptions = {}): Promise { if (hadConfig) log.success("Cleared the saved transformer and file."); if (dropped.length > 0) { - log.success(`Removed ${dropped.length} credential(s) from ${MIGRATE_ENV_FILE}.`); + log.success( + `Removed ${dropped.length} credential${dropped.length === 1 ? "" : "s"} from ${MIGRATE_ENV_FILE}.`, + ); } } diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index e447ddd2e..613fe1042 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -71,6 +71,16 @@ export const NEXT_STEPS = { "Run `clerk apps list` to see your other applications", "Run `clerk config pull` to inspect the live configuration of this instance", ], + MIGRATE_DONE: [ + "Run `clerk migrate logs list` to inspect the import log", + "Run `clerk migrate delete` to undo this migration", + ], + MIGRATE_DELETE: ["Run `clerk migrate logs list` to inspect the deletion log"], + // The only parameterized entry: a suggested import is worthless unless it + // names the transformer that reads this export and the file just written. + MIGRATE_EXPORT: (transformerKey: string, file: string) => [ + `Run \`clerk migrate run --transformer ${transformerKey} --file ${file}\` to import them`, + ], } as const; /** diff --git a/packages/cli-core/src/lib/spinner.ts b/packages/cli-core/src/lib/spinner.ts index b668c35c1..707ba4d16 100644 --- a/packages/cli-core/src/lib/spinner.ts +++ b/packages/cli-core/src/lib/spinner.ts @@ -107,7 +107,9 @@ export async function withGutter( let nextSteps: readonly string[] | undefined; const controls: GutterControls = { setNextSteps(steps) { - nextSteps = steps; + // Empty is ignored rather than stored: `outro([])` would render the + // "Next steps" header with no bullets under it. Matches printNextSteps. + if (steps.length > 0) nextSteps = steps; }, }; From 68c7057f399da7e2fcb45bfc60631f39ddfe46a5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:14:06 -0400 Subject: [PATCH 11/34] feat(prompts): advertise select-all in the multiselect footer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `MultiSelectPrompt` has always bound `a` to toggle every option, but clack's instruction footer never listed it and takes no override — so a genuinely useful key stayed undiscoverable unless each call site spelled it out in its own message, which is worse: it is a property of the prompt, not of any one question. `MULTISELECT_INSTRUCTIONS` is the only seam, and it is read fresh on every render. Inserted second-to-last so `Enter: confirm` stays where readers expect it. `i` (invert) is left out deliberately: it is rarely what anyone wants, and a four-item legend stops being scannable. The test lives outside `prompts.test.ts`, which mocks the whole module. What is worth checking is that the real clack export is still a live array read at render time — an upgrade that froze it, replaced it, or rendered a copy would drop `a: all` silently and nothing else in the suite would notice. --- .../src/lib/prompts-instructions.test.ts | 29 +++++++++++++++++++ packages/cli-core/src/lib/prompts.ts | 18 ++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 packages/cli-core/src/lib/prompts-instructions.test.ts diff --git a/packages/cli-core/src/lib/prompts-instructions.test.ts b/packages/cli-core/src/lib/prompts-instructions.test.ts new file mode 100644 index 000000000..b159c1a2e --- /dev/null +++ b/packages/cli-core/src/lib/prompts-instructions.test.ts @@ -0,0 +1,29 @@ +/** + * The multiselect footer hack, checked against the real @clack/prompts. + * + * Kept out of `prompts.test.ts`, which mocks the whole module — the one thing + * worth verifying here is that the real export is still a live array clack + * reads at render time. A clack upgrade that froze it, replaced it, or rendered + * a copy would drop `a: all` from the legend silently, and nothing else in the + * suite would notice. + */ + +import { test, expect } from "bun:test"; +import { MULTISELECT_INSTRUCTIONS } from "@clack/prompts"; + +// Importing for the module-level side effect is the point. +await import("./prompts.ts"); + +const legend = () => MULTISELECT_INSTRUCTIONS.join(" • ").replaceAll(/\[[0-9;]*m/g, ""); + +test("the multiselect legend advertises select-all", () => { + expect(legend()).toContain("a: all"); +}); + +test("confirm stays last, where readers expect it", () => { + expect(legend().endsWith("Enter: confirm")).toBe(true); +}); + +test("the keys clack actually binds are the ones named", () => { + expect(legend()).toBe("↑/↓ to navigate • Space: select • a: all • Enter: confirm"); +}); diff --git a/packages/cli-core/src/lib/prompts.ts b/packages/cli-core/src/lib/prompts.ts index 533924e89..ed8a466c5 100644 --- a/packages/cli-core/src/lib/prompts.ts +++ b/packages/cli-core/src/lib/prompts.ts @@ -7,16 +7,34 @@ import { confirm as clackConfirm, isCancel, + MULTISELECT_INSTRUCTIONS, text as clackText, password as clackPassword, multiselect as clackMultiselect, type Option as ClackOption, } from "@clack/prompts"; import { editAsync } from "external-editor"; +import { dim } from "./color.ts"; import { throwUserAbort } from "./errors.ts"; import { ttyContext } from "./listage.ts"; import { log } from "./log.ts"; +/** + * Advertise select-all in the multiselect footer. + * + * `MultiSelectPrompt` binds `a` to toggle every option (and `i` to invert), but + * clack's instruction footer has never listed them and takes no override — the + * array below is the only seam, and it is read fresh on every render. So a + * genuinely useful key stays undiscoverable unless each call site spells it out + * in its own message, which is worse: it is a property of the prompt, not of + * any one question. + * + * Inserted second-to-last so `Enter: confirm` stays where readers expect it. + * `i` is left out deliberately — inverting is rarely what anyone wants, and a + * four-item legend stops being scannable. + */ +MULTISELECT_INSTRUCTIONS.splice(MULTISELECT_INSTRUCTIONS.length - 1, 0, `${dim("a:")} all`); + type ValidationResult = string | Error | true | undefined; type Validate = (value: string | undefined) => ValidationResult | Promise; type SyncValidate = (value: string | undefined) => string | Error | undefined; From e9a66929f0c50c421583c4801952e9bf39184b00 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:14:54 -0400 Subject: [PATCH 12/34] feat(migrate): offer to fix the settings the readiness report flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The report already knew which instance settings would cost users; acting on it meant leaving the CLI for the dashboard. A human run now offers one selectable change per flagged row, before the import confirmation, and writes the selection as a single `PATCH` of the instance config document — the same document `clerk config patch` writes. These are offers, not corrections. A flagged setting is not a wrong setting: an instance that genuinely requires an email address is configured exactly as its owner intended, and fixing the export may well be the right answer. Nothing is preselected, and selecting nothing continues to the import with the instance untouched. The redraw after a write is computed from the write, not from a second settings fetch. Clerk's Frontend API is eventually consistent, so a `/v1/environment` read issued this soon after routinely still reports the pre-write settings and would redraw every row the operator just cleared. The offer then repeats while anything is still flagged: applying one change routinely leaves others worth making, so reaching the second never costs a second run of the command. Email and phone take two writes rather than one — they are verifiable attributes, and Clerk rejects one that is on with no way to verify it, while switching it off empties `verification_strategies`. To make the offer answerable, the report itself now leads with **outcomes** rather than per-field coverage: each user is classified once, into the worst outcome that applies to them, so the ✗/⚠/✓ totals add up to the file. A required identifier rejects a user outright; a required password does not, because the import sends `skip_password_requirement`. The field rows below no longer restate user counts, which read as contradicting that block. "If you import them, this applies to them too" names what is masked behind a rejection. A user who is not being created cannot lose a field, so a setting affecting only rejected users costs nothing today — right up until the requirement rejecting them is relaxed, at which point all of it lands at once. Surfacing it up front collapses apply → re-check → discover → apply into one decision. Stands down with a warning rather than a failed run when the instance cannot be resolved, and for keyless applications, whose Backend API has no route for any of these settings. --- .../cli-core/src/commands/migrate/README.md | 161 +++++++++- .../migrate/lib/modify-settings.test.ts | 303 ++++++++++++++++++ .../commands/migrate/lib/modify-settings.ts | 191 +++++++++++ .../commands/migrate/lib/readiness.test.ts | 187 ++++++++++- .../src/commands/migrate/lib/readiness.ts | 258 ++++++++++++++- .../commands/migrate/run-interactive.test.ts | 264 ++++++++++++++- .../cli-core/src/commands/migrate/run.test.ts | 2 +- packages/cli-core/src/commands/migrate/run.ts | 167 ++++++++-- 8 files changed, 1469 insertions(+), 64 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/modify-settings.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index f74ab3606..3c5b180a3 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -40,8 +40,9 @@ platform and file from the last run so a repeat migration is mostly pressing enter. Anything already passed as a flag is not asked for. Firebase's hash parameters are never pre-filled — see [below](#--firebase--firebase). -Then it prints the [Migration Readiness report](#migration-readiness-report) -and waits for confirmation. Declining writes nothing to Clerk. +Then it prints the [Migration Readiness report](#migration-readiness-report), +offers to [change whatever it flagged](#changing-the-flagged-settings), and +waits for confirmation. Declining writes nothing to Clerk. **Agent mode never prompts.** `clerk migrate` with no flags exits with a usage error naming exactly what to pass: @@ -695,35 +696,163 @@ lecture" and should not pay for the two extra round-trips. Agent runs without It cross-references the file against the destination instance's live settings (BAPI `/v1/domains` → that instance's Frontend API `/v1/environment`) and -flags the two failure modes a migration otherwise discovers halfway through: - -- **Required in Clerk, missing from the file.** Those users fail one at a time, - mid-import, after earlier users already exist. -- **Present in the file, disabled in Clerk.** Social providers users actually - signed up with, or an identifier the instance has switched off. +answers the two questions worth answering before writing anything: **who won't +be imported**, and **who will arrive incomplete**. ``` Migration readiness - 120 users ready to import + 120 users in this file 3 failed validation and will be skipped + ✗ 12 users will not be imported + 12 have no email, which this instance requires + If you import them, this applies to them too: + 12 have a phone, which this instance is not set up to store + ⚠ 20 users will be imported, but not everything they carry + 14 have no password, which this instance requires — they will have to reset it to sign in + 6 have a username, which this instance is not set up to store + ✓ 88 users will be imported in full + Identifiers - ⚠ Email — required in Clerk, but 12 users lack it — 108/120 users - ✓ Username — enabled in Clerk — all users + ⚠ Email — required in Clerk, and not every user has one — 108/120 users + ⚠ Username — not enabled in Clerk — 6/120 users Social connections ✓ Google — enabled in Clerk — 40/120 users ⚠ Discord — not enabled in Clerk — 12/120 users -⚠ 2 settings need attention +⚠ 3 settings need attention ``` +### The two blocks + +**The outcome block** classifies each user **once**, into the worst outcome that +applies to them, so its three totals add up to the file. This matters: per-field +coverage cannot answer "how many won't be imported", because the users missing +an email and the users missing a password overlap by an amount only a per-user +pass knows. A user rejected for their missing email is not also counted under +the missing password they happen to share. + +**"If you import them, this applies to them too"** is the part that stops the +settings interacting invisibly. A user who is not being created cannot lose a +field, so a setting that only affects rejected users costs nothing _today_ and +would otherwise never be mentioned — right up until the operator relaxes the +requirement rejecting them, at which point all of it lands at once. Naming it +up front is what turns + +> make email optional → re-check → discover the phones are being dropped → +> enable phone → re-check + +into a single decision with both offers visible. It is also why a setting can +be flagged in the section rows while contributing nothing to the ✗/⚠/✓ totals. + +**The section rows below** are the other question — per-field coverage against +each setting — and deliberately do not restate user counts, which would read as +contradicting the block above. + +### Which settings cost what + +| Setting | Consequence | +| ------------------------------------------ | --------------------------------------------------------------------------------------------- | +| Identifier (email/phone/username) required | **Not imported.** `POST /v1/users` enforces the sign-up identifier requirements. | +| Password required, user has none | **Imported without a password.** The import sends `skip_password_requirement`, so the user is | +| | created and has to reset their password before they can sign in with one. | +| Attribute disabled in Clerk | **Imported without that field.** The instance has nowhere to put it. | +| Social provider disabled | **Imported**, but that sign-in method is unavailable to them. | + +Social rows are not part of the per-user outcome counts: which providers a user +signed up with lives in the raw export rather than the transformed user, so it +cannot be attributed per user. Their coverage row still names them. + If the instance settings cannot be read — the secret key is rejected, or FAPI is unreachable — the report degrades to a coverage-only listing with a note. Nothing is flagged in that case: "could not read" is not the same as "switched off", and treating it as such would raise alarms about settings that are perfectly fine. +### Changing the flagged settings + +When the report flags anything, a human run offers one selectable change per +flagged row before the import confirmation, so acting on the report does not +mean leaving the CLI for the dashboard: + +``` +Update this instance's settings first? (enter to skip) + ◻ Make Email optional at sign-up + ◻ Enable Discord sign-in + ↑/↓ to navigate • Space: select • a: all • Enter: confirm +``` + +**Nothing is preselected** — relaxing an instance's sign-up requirements is a +real decision, not a default — and selecting nothing continues to the import +prompt with the instance untouched, which is what "enter to skip" is there to +say. + +`a: all` is added to clack's legend in `lib/prompts.ts`: `MultiSelectPrompt` +has always bound `a` to toggle everything (and `i` to invert), but clack's +footer never listed them and takes no override, so the key was undiscoverable. +It applies to every multiselect in the CLI, because it is a property of the +prompt rather than of any one question. + +These are offers, not corrections: **a flagged setting is not a wrong setting.** +An instance that genuinely requires an email address is configured exactly as +its owner intended, and the right answer may well be to fix the export instead. + +Whatever is selected becomes a single `PATCH` of the instance config document, +the same document `clerk config patch` writes. The report is then redrawn so +the confirmation that follows is against the settings the write established. + +**The offer repeats while anything is still flagged.** A redraw is another +decision point, not a receipt: applying one change routinely leaves others +worth making, and each round re-offers only what is left. It ends when the +report has nothing flagged, when the operator selects nothing, or when there is +nothing offerable for the rows that remain — so reaching the second change +never costs a second run of the command. + +The redraw is computed from the write, **not** from a second settings fetch. +Clerk's Frontend API is eventually consistent, so a `/v1/environment` read +issued this soon after the config write routinely still reports the pre-write +settings — which would redraw the report with every row the operator just +cleared still flagged. The Platform API accepting the write is the +authoritative statement of what took, exactly as `clerk config patch` treats +it (see that command's [round-trip verification](../config/README.md#round-trip-verification) +notes for the same reasoning). + +The config leaves each option writes are not shown in the prompt — internal +detail an operator cannot act on — but they are fixed and listed here: + +| Flagged row | Change offered | +| ------------------------------- | --------------------------------------------------------------------------------- | +| Email/Phone/Username — required | `auth_.required_for_sign_up → false` | +| Email — disabled | `auth_email.used_for_sign_up → true` + `verification_strategies → ["email_code"]` | +| Phone — disabled | `auth_phone.used_for_sign_up → true` + `verification_strategies → ["phone_code"]` | +| Username — disabled | `auth_username.used_for_sign_up → true` | +| Password — required / disabled | `auth_password.required → false` / `auth_password.enabled → true` | +| First/Last name | `user_model..required → false` / `user_model..enabled → true` | +| Social provider — disabled | `connection_oauth_.enabled → true` | + +`used_for_sign_up` is the enable field that matters: `POST /v1/users` validates +an import against the instance's sign-up requirements, not its sign-in +strategies. + +**Email and phone take two writes, not one.** They are _verifiable_ attributes, +and Clerk rejects one that is on with no way to verify it: + +``` +422 phone_number: verifiable attributes need to have at least one verification +``` + +Switching the attribute off empties `verification_strategies`, so whatever +turns it back on has to put a strategy back in the same request. Username, +password and the name fields are not verifiable and take one write each. + +The offer is skipped entirely for `-y` and in agent mode, both of which say +"don't prompt". It also stands down, with a warning rather than a failed run, +when the instance to configure cannot be resolved (a bare `--secret-key` in an +unlinked directory) or when it is a **keyless** application — the Backend API +those are reachable through has no route for any of these settings, so +`clerk auth login` is the way in. + ## Artifacts Both are written relative to the **current working directory**, not to the @@ -784,6 +913,14 @@ NDJSON is. The original `.log` stays put. | `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | | `GET` | `/v1/domains` | Readiness report and `--skip-unsupported-providers` — resolves the Frontend API host | +The readiness report also reads the instance's Frontend API +`GET /v1/environment` (bootstrapping a dev browser first on development +instances), and its settings-change offer writes through the Platform API: + +| Method | Path | Used by | +| ------- | ----------------------------------------------------------------- | ------------------------------------------------------ | +| `PATCH` | `/v1/platform/applications/{appID}/instances/{instanceID}/config` | Applying the settings changes selected from the report | + Two exports talk to their own platform rather than to Clerk: | Method | Path | Used by | diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts new file mode 100644 index 000000000..e9b55e8e0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts @@ -0,0 +1,303 @@ +import { describe, expect, test } from "bun:test"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import type { FieldAnalysis } from "./analysis.ts"; +import { buildReadinessReport } from "./readiness.ts"; +import { applyChanges, buildChangePayload, buildSettingChanges } from "./modify-settings.ts"; + +/** Instance settings carrying only the attributes and providers a test names. */ +function settings(config: { + attributes?: Record; + social?: Record; +}): UserSettingsJSON { + return { + attributes: Object.fromEntries( + Object.entries(config.attributes ?? {}).map(([name, value]) => [ + name, + { enabled: value.enabled, required: value.required ?? false }, + ]), + ), + social: config.social ?? {}, + } as unknown as UserSettingsJSON; +} + +function analysis(overrides: Partial & { totalUsers: number }): FieldAnalysis { + return { + identifiers: { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: overrides.totalUsers, + ...overrides.identifiers, + }, + fieldCounts: overrides.fieldCounts ?? {}, + totalUsers: overrides.totalUsers, + }; +} + +/** + * Changes are built from a real report rather than hand-written rows, so a row + * whose `key` stops matching the path table fails here instead of silently + * dropping out of the offer. + */ +function changesFor(input: Parameters[0]) { + return buildSettingChanges(buildReadinessReport(input).blocking); +} + +describe("what gets offered", () => { + test("a required field not every user has is offered as a relaxation", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + + expect(changes).toEqual([ + { + id: "email_address", + label: "Make Email optional at sign-up", + section: "identifiers", + kind: "relax", + writes: [{ path: ["auth_email", "required_for_sign_up"], value: false }], + }, + ]); + }); + + test("a field the file carries but Clerk has switched off is offered as an enable", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { username: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { username: { enabled: false } } }), + }); + + expect(changes).toEqual([ + { + id: "username", + label: "Enable Username", + section: "identifiers", + kind: "enable", + writes: [{ path: ["auth_username", "used_for_sign_up"], value: true }], + }, + ]); + }); + + /** + * Clerk refuses a verifiable attribute that is on with no way to verify it — + * `422 phone_number: verifiable attributes need to have at least one + * verification` — and switching the attribute off empties the strategies, so + * every enable that turned one off has to put one back. + */ + test.each([ + ["phone_number", "auth_phone", "phone_code"], + ["email_address", "auth_email", "email_code"], + ])("enabling %s also restores its verification strategy", (attribute, group, strategy) => { + const carriesIt = attribute === "phone_number" ? { verifiedPhones: 2 } : { verifiedEmails: 2 }; + + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { ...carriesIt, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { [attribute]: { enabled: false } } }), + }); + + expect(changes[0]?.writes).toEqual([ + { path: [group, "used_for_sign_up"], value: true }, + { path: [group, "verification_strategies"], value: [strategy] }, + ]); + expect(buildChangePayload(changes)).toEqual({ + [group]: { used_for_sign_up: true, verification_strategies: [strategy] }, + }); + }); + + // Not verifiable, so no strategy to restore — one write is the whole change. + test("enabling username takes a single write", () => { + const changes = changesFor({ + analysis: analysis({ totalUsers: 2, identifiers: { username: 2 } as never }), + settings: settings({ attributes: { username: { enabled: false } } }), + }); + expect(changes[0]?.writes).toHaveLength(1); + }); + + test("a disabled social provider is offered under Clerk's own strategy name", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_x: { enabled: false } }, + }), + // Supabase calls it `twitter`; the config document calls it `oauth_x`. + providerCounts: { twitter: 2 }, + }); + + expect(changes).toEqual([ + { + id: "twitter", + label: "Enable Twitter (X) sign-in", + section: "social", + kind: "enable", + writes: [{ path: ["connection_oauth_x", "enabled"], value: true }], + }, + ]); + }); + + // "Could not read" is not "switched off", so nothing is flagged and nothing + // is offered — the report already degrades to a coverage-only listing. + test("nothing is offered when the instance settings could not be read", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 1, hasAnyIdentifier: 2 } as never, + }), + settings: null, + }); + + expect(changes).toEqual([]); + }); + + test("nothing is offered when every field is already configured", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + + expect(changes).toEqual([]); + }); +}); + +describe("the payload", () => { + test("collapses changes that share a parent into one branch", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 3 } as never, + fieldCounts: { firstName: 2, lastName: 1 }, + }), + settings: settings({ + attributes: { + email_address: { enabled: true }, + first_name: { enabled: true, required: true }, + last_name: { enabled: true, required: true }, + }, + }), + }); + + expect(buildChangePayload(changes)).toEqual({ + user_model: { first_name: { required: false }, last_name: { required: false } }, + }); + }); + + test("carries only the changes it is given", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 3 } as never, + fieldCounts: { password: 1 }, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + password: { enabled: true, required: true }, + }, + }), + }); + expect(changes.map((change) => change.id)).toEqual(["email_address", "password"]); + + expect(buildChangePayload(changes.filter((change) => change.id === "password"))).toEqual({ + auth_password: { required: false }, + }); + }); + + test("is empty when nothing was selected", () => { + expect(buildChangePayload([])).toEqual({}); + }); +}); + +/** + * The redraw after a write comes from `applyChanges`, not a second fetch: + * Clerk's Frontend API is eventually consistent, so re-reading straight after + * the patch returns the pre-write settings and redraws every row just cleared. + */ +describe("the settings after a write", () => { + /** The two fields a change touches, as `settings()` above builds them. */ + const attr = (value: { enabled: boolean; required: boolean }) => + value as unknown as UserSettingsJSON["attributes"]["email_address"]; + + test("drops the requirement a relaxation removed", () => { + const before = settings({ attributes: { email_address: { enabled: true, required: true } } }); + const input = { + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, + }), + settings: before, + }; + + const after = applyChanges(before, changesFor(input)); + + expect(after?.attributes.email_address).toEqual(attr({ enabled: true, required: false })); + // The report is rebuilt from this, so the row must stop being flagged. + expect(buildReadinessReport({ ...input, settings: after }).blocking).toEqual([]); + }); + + test("turns on what an enable switched on", () => { + const before = settings({ attributes: { username: { enabled: false } } }); + const changes = changesFor({ + analysis: analysis({ totalUsers: 2, identifiers: { username: 2 } as never }), + settings: before, + }); + + expect(applyChanges(before, changes)?.attributes.username).toEqual( + attr({ enabled: true, required: false }), + ); + }); + + test("enables a provider under Clerk's strategy name, not the source platform's", () => { + const before = settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_x: { enabled: false } }, + }); + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: before, + providerCounts: { twitter: 2 }, + }); + + expect(applyChanges(before, changes)?.social).toEqual({ + oauth_x: { enabled: true }, + } as unknown as UserSettingsJSON["social"]); + }); + + test("leaves the settings it was given untouched", () => { + const before = settings({ attributes: { email_address: { enabled: true, required: true } } }); + const changes = changesFor({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, + }), + settings: before, + }); + + applyChanges(before, changes); + + expect(before.attributes.email_address).toMatchObject({ required: true }); + }); + + test("passes null through — unreadable settings flag nothing to change", () => { + expect(applyChanges(null, [])).toBeNull(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts new file mode 100644 index 000000000..8136f6ca1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts @@ -0,0 +1,191 @@ +/** + * Turning a flagged Migration Readiness row into the instance-config change + * that would stop it being flagged. + * + * The report already knows which settings will cost users; without this the + * only way to act on it is to leave the CLI, find the setting in the dashboard, + * and come back. Each change is a single leaf in the Platform API's config + * document, so they compose into one `PATCH` however many the operator picks. + * + * These are offers, not corrections. A flagged setting is not a wrong setting — + * an instance that genuinely requires an email address is configured exactly as + * its owner intended, and the right answer may well be to fix the export + * instead. Nothing here is preselected and nothing is applied unasked. + * + * Only the two verdicts `buildReadinessReport` produces are mapped: "required + * in Clerk" (relax the requirement) and "not enabled in Clerk" (turn it on). A + * row this file has no path for is simply not offered — the report still names + * it and still points at the dashboard. + */ + +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { toClerkStrategy } from "./clerk-config.ts"; +import type { ReadinessItem, ReadinessSection } from "./readiness.ts"; + +/** One leaf of the config document, and what to set it to. */ +export type SettingWrite = { path: string[]; value: boolean | string[] }; + +/** One offered change: what it says, and the config leaves it writes. */ +export type SettingChange = { + /** Stable identity for the multiselect, and for tests. */ + id: string; + label: string; + section: ReadinessSection; + /** `enable` turns something on; `relax` drops a requirement. */ + kind: "enable" | "relax"; + writes: SettingWrite[]; +}; + +type ChangeWrites = { enable: SettingWrite[]; relax: SettingWrite[] }; + +/** + * Where each attribute lives in the config document. `used_for_sign_up` is the + * enable field that matters here: `POST /v1/users` validates an import against + * the instance's sign-up requirements, not its sign-in strategies. + * + * Email and phone are **verifiable** attributes, so enabling one takes two + * writes rather than one. Clerk rejects a verifiable attribute that is on with + * no way to verify it — `422 phone_number: verifiable attributes need to have + * at least one verification` — and switching the attribute off empties + * `verification_strategies`, so whatever turns it back on has to put a strategy + * back. Username, password and the name fields are not verifiable and take one + * write each. + */ +const ATTRIBUTE_WRITES: Record = { + email_address: { + enable: [ + { path: ["auth_email", "used_for_sign_up"], value: true }, + { path: ["auth_email", "verification_strategies"], value: ["email_code"] }, + ], + relax: [{ path: ["auth_email", "required_for_sign_up"], value: false }], + }, + phone_number: { + enable: [ + { path: ["auth_phone", "used_for_sign_up"], value: true }, + { path: ["auth_phone", "verification_strategies"], value: ["phone_code"] }, + ], + relax: [{ path: ["auth_phone", "required_for_sign_up"], value: false }], + }, + username: { + enable: [{ path: ["auth_username", "used_for_sign_up"], value: true }], + relax: [{ path: ["auth_username", "required_for_sign_up"], value: false }], + }, + password: { + enable: [{ path: ["auth_password", "enabled"], value: true }], + relax: [{ path: ["auth_password", "required"], value: false }], + }, + first_name: { + enable: [{ path: ["user_model", "first_name", "enabled"], value: true }], + relax: [{ path: ["user_model", "first_name", "required"], value: false }], + }, + last_name: { + enable: [{ path: ["user_model", "last_name", "enabled"], value: true }], + relax: [{ path: ["user_model", "last_name", "required"], value: false }], + }, +}; + +/** `github` → `connection_oauth_github`, via Clerk's own strategy name. */ +function socialPath(provider: string): string[] { + return [`connection_oauth_${toClerkStrategy(provider).replace(/^oauth_/, "")}`, "enabled"]; +} + +function changeFor(item: ReadinessItem): SettingChange | undefined { + // Required-but-not-universal is the only verdict that relaxes rather than + // enables; every other flagged row is something switched off in Clerk. + const relax = item.clerkRequired === true; + + if (item.section === "social") { + // A provider has no "required" in Clerk, so there is nothing to relax. + if (relax) return undefined; + return { + id: item.key, + label: `Enable ${item.label} sign-in`, + section: item.section, + kind: "enable", + writes: [{ path: socialPath(item.key), value: true }], + }; + } + + const writes = ATTRIBUTE_WRITES[item.key]; + if (!writes) return undefined; + + return { + id: item.key, + label: relax ? `Make ${item.label} optional at sign-up` : `Enable ${item.label}`, + section: item.section, + kind: relax ? "relax" : "enable", + writes: relax ? writes.relax : writes.enable, + }; +} + +/** The changes offerable for a report's flagged rows, in report order. */ +export function buildSettingChanges(flagged: ReadinessItem[]): SettingChange[] { + return flagged.map(changeFor).filter((change): change is SettingChange => change !== undefined); +} + +/** + * Collapses the chosen changes into one config payload. + * + * Changes share parents — `first_name` and `last_name` both write `user_model` + * — so leaves are written into a shared tree rather than merged after the fact. + */ +export function buildChangePayload(changes: SettingChange[]): Record { + const payload: Record = {}; + + for (const write of changes.flatMap((change) => change.writes)) { + let node = payload; + for (const key of write.path.slice(0, -1)) { + node = (node[key] ??= {}) as Record; + } + node[write.path[write.path.length - 1] as string] = write.value; + } + + return payload; +} + +/** + * The instance's settings as they stand once `changes` have been written. + * + * Deliberately not a re-read. Clerk's Frontend API is eventually consistent, so + * a `/v1/environment` fetch issued straight after the config write routinely + * still reports the pre-write settings — which would redraw the report with + * every row it just cleared still flagged. The Platform API answering the write + * is the authoritative statement of what took, exactly as `clerk config patch` + * treats it. + * + * @param settings - `null` passes through: when the settings could not be read + * nothing is ever flagged, so there is nothing to have changed. + */ +export function applyChanges( + settings: UserSettingsJSON | null, + changes: SettingChange[], +): UserSettingsJSON | null { + if (!settings) return null; + + const next = structuredClone(settings); + + for (const change of changes) { + if (change.section === "social") { + const social = next.social as Record; + const strategy = toClerkStrategy(change.id); + social[strategy] = { ...social[strategy], enabled: true }; + continue; + } + + const attributes = next.attributes as Record; + attributes[change.id] = + change.kind === "enable" + ? { + ...attributes[change.id], + enabled: true, + required: attributes[change.id]?.required ?? false, + } + : { + ...attributes[change.id], + enabled: attributes[change.id]?.enabled ?? true, + required: false, + }; + } + + return next; +} diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts index cdac7d866..842e9c4b5 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import type { UserSettingsJSON } from "../../../lib/fapi.ts"; -import type { FieldAnalysis } from "./analysis.ts"; +import { analyzeFields, type FieldAnalysis } from "./analysis.ts"; import { buildReadinessReport, formatReadinessReport, type ReadinessItem } from "./readiness.ts"; /** Instance settings carrying only the attributes and providers a test names. */ @@ -97,7 +97,9 @@ describe("required in Clerk but missing from the file", () => { const email = item(report, "Email"); expect(email?.blocking).toBe(true); - expect(email?.detail).toContain("3 users lack it"); + expect(email?.detail).toContain("required in Clerk"); + // A required identifier is the one verdict Clerk refuses the user over. + expect(email?.consequence).toBe("rejects"); expect(report.blocking).toHaveLength(1); }); @@ -126,20 +128,23 @@ describe("required in Clerk but missing from the file", () => { expect(report.blocking).toHaveLength(0); }); - test("uses the singular form for a single missing user", () => { + // The import sends `skip_password_requirement`, so a required password costs + // the user their password rather than their whole account. + test("a required password drops rather than rejects", () => { const report = buildReadinessReport({ analysis: analysis({ - totalUsers: 2, - identifiers: { verifiedEmails: 1, hasAnyIdentifier: 2, username: 2 } as never, + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + fieldCounts: { password: 1 }, }), settings: settings({ attributes: { - email_address: { enabled: true, required: true }, - username: { enabled: true }, + email_address: { enabled: true }, + password: { enabled: true, required: true }, }, }), }); - expect(item(report, "Email")?.detail).toContain("1 user lacks it"); + expect(item(report, "Password")?.consequence).toBe("drops"); }); }); @@ -255,6 +260,168 @@ describe("file-level totals", () => { }); }); +/** + * The counts an operator actually decides on. Built from the users themselves, + * because per-field coverage cannot answer them: the users missing an email and + * the users missing a password overlap by an amount only a per-user pass knows. + */ +describe("what the settings mean for these users", () => { + const REQUIRE_EMAIL_AND_PASSWORD = settings({ + attributes: { + email_address: { enabled: true, required: true }, + password: { enabled: true, required: true }, + }, + }); + + /** Two with everything, two with no email, one with an email but no password. */ + const USERS = [ + { userId: "a", email: "a@x.dev", password: "hash" }, + { userId: "b", email: "b@x.dev", password: "hash" }, + { userId: "c", username: "c" }, + { userId: "d", username: "d" }, + { userId: "e", email: "e@x.dev" }, + ] as never; + + const outcomes = () => + buildReadinessReport({ + analysis: analyzeFields(USERS), + users: USERS, + settings: REQUIRE_EMAIL_AND_PASSWORD, + }).outcomes; + + test("the three totals account for every user exactly once", () => { + const result = outcomes(); + expect(result).toMatchObject({ rejected: 2, incomplete: 1, complete: 2 }); + expect((result?.rejected ?? 0) + (result?.incomplete ?? 0) + (result?.complete ?? 0)).toBe(5); + }); + + // The file has three users without a password, but two of them are already + // rejected for the email — counting them twice would overstate the damage. + test("a rejected user is not also counted as incomplete", () => { + expect(outcomes()?.incompleteReasons).toEqual([ + { + label: "Password", + count: 1, + detail: expect.stringContaining("1 has no password, which this instance requires"), + }, + ]); + }); + + test("names why the rejected users are rejected", () => { + expect(outcomes()?.rejectedReasons).toEqual([ + { label: "Email", count: 2, detail: "2 have no email, which this instance requires" }, + ]); + }); + + /** + * The rejected users lose nothing today — they are not being created. But the + * moment the operator relaxes the requirement rejecting them (one of the + * changes on offer) every masked setting lands at once. Surfacing it here is + * what saves an apply → re-check → discover → apply → re-check loop. + */ + describe("what is masked behind a rejection", () => { + // b and c have no email, so both are rejected; b also carries a phone the + // instance is not set up to store. Exactly the shape the supabase sample + // hits: every phone belongs to a user who has no email. + const MASKED_USERS = [ + { userId: "a", email: "a@x.dev" }, + { userId: "b", username: "b", phone: "+15551234567" }, + { userId: "c", username: "c" }, + ] as never; + + const report = (attributes: Record) => + buildReadinessReport({ + analysis: analyzeFields(MASKED_USERS), + users: MASKED_USERS, + settings: settings({ attributes }), + }); + + const REQUIRE_EMAIL_PHONE_OFF = { + email_address: { enabled: true, required: true }, + phone_number: { enabled: false }, + username: { enabled: true }, + }; + + test("counts a setting that only bites once the rejected users get in", () => { + const outcomes = report(REQUIRE_EMAIL_PHONE_OFF).outcomes; + + expect(outcomes).toMatchObject({ rejected: 2, incomplete: 0, complete: 1 }); + expect(outcomes?.maskedReasons).toEqual([ + { + label: "Phone", + count: 1, + detail: "1 has a phone, which this instance is not set up to store", + }, + ]); + }); + + test("keeps it out of the incomplete count, which is about users being imported", () => { + expect(report(REQUIRE_EMAIL_PHONE_OFF).outcomes?.incompleteReasons).toEqual([]); + }); + + test("renders it under the rejected group", () => { + const output = formatReadinessReport(report(REQUIRE_EMAIL_PHONE_OFF)).join("\n"); + + expect(output).toContain("If you import them, this applies to them too:"); + expect(output).toContain("1 has a phone, which this instance is not set up to store"); + }); + + // Enabling phone is the other change on offer, and it empties the block — + // which is the check that the two offers really do interact this way. + test("is empty once the masked setting is no longer a problem", () => { + const outcomes = report({ + email_address: { enabled: true, required: true }, + phone_number: { enabled: true }, + username: { enabled: true }, + }).outcomes; + + expect(outcomes).toMatchObject({ rejected: 2 }); + expect(outcomes?.maskedReasons).toEqual([]); + }); + }); + + test("a disabled attribute costs the users who carry it, not the ones who don't", () => { + const users = [ + { userId: "a", email: "a@x.dev", username: "a" }, + { userId: "b", email: "b@x.dev" }, + ] as never; + + const result = buildReadinessReport({ + analysis: analyzeFields(users), + users, + settings: settings({ + attributes: { email_address: { enabled: true }, username: { enabled: false } }, + }), + }).outcomes; + + expect(result).toMatchObject({ rejected: 0, incomplete: 1, complete: 1 }); + expect(result?.incompleteReasons[0]?.detail).toContain("not set up to store"); + }); + + test("is omitted when the caller passes no users", () => { + const report = buildReadinessReport({ + analysis: analyzeFields(USERS), + settings: REQUIRE_EMAIL_AND_PASSWORD, + }); + expect(report.outcomes).toBeUndefined(); + }); + + test("renders each outcome with the reasons behind it", () => { + const output = formatReadinessReport( + buildReadinessReport({ + analysis: analyzeFields(USERS), + users: USERS, + settings: REQUIRE_EMAIL_AND_PASSWORD, + }), + ).join("\n"); + + expect(output).toContain("2 users will not be imported"); + expect(output).toContain("1 user will be imported, but not everything they carry"); + expect(output).toContain("2 users will be imported in full"); + expect(output).toContain("they will have to reset it to sign in"); + }); +}); + describe("rendering", () => { const blocked = () => buildReadinessReport({ @@ -273,7 +440,7 @@ describe("rendering", () => { test("leads with the counts an operator needs before confirming", () => { const output = formatReadinessReport(blocked()).join("\n"); - expect(output).toContain("10 users ready to import"); + expect(output).toContain("10 users in this file"); expect(output).toContain("2 failed validation"); expect(output).toContain("2 without any identifier"); }); @@ -281,7 +448,7 @@ describe("rendering", () => { test("names the blocking rows and points at the dashboard", () => { const output = formatReadinessReport(blocked()).join("\n"); expect(output).toContain("1 setting needs attention"); - expect(output).toContain("3 users lack it"); + expect(output).toContain("required in Clerk, and not every user has one"); expect(output).toContain("dashboard.clerk.com"); }); diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts index 1db17fe3e..e476da4a1 100644 --- a/packages/cli-core/src/commands/migrate/lib/readiness.ts +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -16,10 +16,11 @@ import type { UserSettingsJSON } from "../../../lib/fapi.ts"; import { bold, dim, green, red, yellow } from "../../../lib/color.ts"; // Pure attribute lookups, shared with the `users` create wizard. import { isEnabled, isRequired, type AttributeName } from "../../users/interactive/attributes.ts"; -import type { FieldAnalysis } from "./analysis.ts"; +import type { User } from "../types.ts"; +import { hasValue, type FieldAnalysis } from "./analysis.ts"; import { providerLabel, toClerkStrategy } from "./clerk-config.ts"; -const DASHBOARD_URL = "https://dashboard.clerk.com/~/user-authentication"; +export const DASHBOARD_URL = "https://dashboard.clerk.com/~/user-authentication"; export type ReadinessSection = "identifiers" | "auth" | "social" | "model"; @@ -32,16 +33,68 @@ export type ReadinessSection = "identifiers" | "auth" | "social" | "model"; */ export type ReadinessItem = { label: string; + /** + * What the row is about, machine-side: an {@link AttributeName} for every + * section but `social`, and the source platform's provider key for that one. + * `label` is for humans; this is what `modify-settings.ts` looks up. + */ + key: string; section: ReadinessSection; /** Users in the file that carry this field or provider. */ userCount: number; clerkEnabled: boolean | null; clerkRequired: boolean | null; blocking: boolean; + /** + * What this row costs the users it affects. + * + * - `rejects` — Clerk refuses the user outright. Only a required identifier + * does this: `POST /v1/users` enforces the instance's sign-up identifier + * requirements, and a user carrying none of them has nothing to be created + * under. + * - `drops` — the user is created, but this piece of them is not. A required + * password is in this group rather than `rejects` because the import sends + * `skip_password_requirement` (see `import-users.ts`), so the user lands + * without one and has to reset it before they can sign in that way. + */ + consequence?: "rejects" | "drops"; /** Why it blocks — omitted when it does not. */ detail?: string; }; +/** One reason users are affected, and how many of them it affects. */ +export type OutcomeReason = { label: string; count: number; detail: string }; + +/** + * What the settings mean for the users in the file, counted per user rather + * than per field. + * + * Per-field coverage cannot answer "how many users will not be imported" — + * the users missing an email and the users missing a username overlap by an + * unknown amount. Each user is classified once, into the worst outcome that + * applies to them, so the three totals add up to the file. + */ +export type ImportOutcomes = { + rejected: number; + rejectedReasons: OutcomeReason[]; + /** + * What *else* affects the rejected users — surfaced now rather than after + * they become importable. + * + * A user who is not being created cannot lose a field, so these settings cost + * nothing today and would otherwise go unmentioned. But the moment the + * operator relaxes the requirement rejecting them, every one of these lands. + * Reporting it only afterwards turns one decision into a apply → re-check → + * discover → apply → re-check loop, which is exactly what the report exists + * to prevent. + */ + maskedReasons: OutcomeReason[]; + incomplete: number; + incompleteReasons: OutcomeReason[]; + /** Imported with everything the file carries for them. */ + complete: number; +}; + export type ReadinessReport = { totalUsers: number; /** Users with no identifier at all; they cannot be imported under any settings. */ @@ -52,6 +105,8 @@ export type ReadinessReport = { blocking: ReadinessItem[]; /** True when the instance settings could not be read. */ settingsUnavailable: boolean; + /** Omitted when the caller passed no users to classify. */ + outcomes?: ImportOutcomes; }; type BuildInput = { @@ -61,8 +116,21 @@ type BuildInput = { validationFailed?: number; /** Source-platform provider key → user count. Supabase exports only. */ providerCounts?: Record; + /** + * The users themselves, for the per-user outcome counts. Optional so callers + * that only need the coverage rows (and tests working from a synthetic + * {@link FieldAnalysis}) do not have to supply them. + */ + users?: User[]; }; +/** + * Identifiers Clerk creates a user *under*. A required one that a user does not + * carry leaves nothing to create them with, so the API refuses them — which is + * why these are the only attributes whose consequence is `rejects`. + */ +const IDENTIFIER_ATTRIBUTES = new Set(["email_address", "phone_number", "username"]); + /** An identifier or user-model row, with its blocking verdict. */ function buildAttributeItem( label: string, @@ -81,15 +149,18 @@ function buildAttributeItem( if (required === true && missing > 0) { return { label, + key: attribute, section, userCount, clerkEnabled: enabled, clerkRequired: required, blocking: true, - detail: - missing === 1 - ? "required in Clerk, but 1 user lacks it" - : `required in Clerk, but ${missing} users lack it`, + consequence: IDENTIFIER_ATTRIBUTES.has(attribute) ? "rejects" : "drops", + // How many users this costs is the outcome block's job. Restating it here + // reads as a contradiction, because that block counts each user once and + // this row counts the field — a user missing both an email and a password + // appears in both rows but only in the first outcome. + detail: "required in Clerk, and not every user has one", }; } @@ -97,17 +168,20 @@ function buildAttributeItem( if (enabled === false && userCount > 0) { return { label, + key: attribute, section, userCount, clerkEnabled: enabled, clerkRequired: required, blocking: true, + consequence: "drops", detail: "not enabled in Clerk", }; } return { label, + key: attribute, section, userCount, clerkEnabled: enabled, @@ -153,22 +227,135 @@ export function buildReadinessReport(input: BuildInput): ReadinessReport { : null; items.push({ label: providerLabel(provider), + key: provider, section: "social", userCount: count, clerkEnabled: enabled, clerkRequired: null, blocking: enabled === false, - ...(enabled === false ? { detail: "not enabled in Clerk" } : {}), + ...(enabled === false + ? { consequence: "drops" as const, detail: "not enabled in Clerk" } + : {}), }); } + const blocking = items.filter((item) => item.blocking); + return { totalUsers: total, withoutIdentifier: total - analysis.identifiers.hasAnyIdentifier, validationFailed, items, - blocking: items.filter((item) => item.blocking), + blocking, settingsUnavailable: settings === null, + ...(input.users ? { outcomes: countOutcomes(input.users, blocking) } : {}), + }; +} + +/** Whether a user carries the field an attribute row is about. */ +const CARRIES: Record) => boolean> = { + email_address: (u) => + hasValue(u.email) || hasValue(u.emailAddresses) || hasValue(u.unverifiedEmailAddresses), + phone_number: (u) => + hasValue(u.phone) || hasValue(u.phoneNumbers) || hasValue(u.unverifiedPhoneNumbers), + username: (u) => hasValue(u.username), + password: (u) => hasValue(u.password), + first_name: (u) => hasValue(u.firstName), + last_name: (u) => hasValue(u.lastName), +}; + +/** Which users a flagged row actually affects: the ones missing it, or carrying it. */ +function affects(item: ReadinessItem, user: Record): boolean { + const carries = CARRIES[item.key]; + if (!carries) return false; + // A required row costs the users without it; a disabled row costs the ones with it. + return item.clerkRequired === true ? !carries(user) : carries(user); +} + +/** + * One reason line: how many users, what they are missing or carrying, and what + * the instance does about it. Count first, because that is what is being + * decided on. + */ +function describe(item: ReadinessItem, count: number): string { + const noun = item.label.toLowerCase(); + const have = count === 1 ? "has" : "have"; + + if (item.clerkRequired === true) { + const consequence = item.key === "password" ? " — they will have to reset it to sign in" : ""; + return `${count} ${have} no ${noun}, which this instance requires${consequence}`; + } + return `${count} ${have} a ${noun}, which this instance is not set up to store`; +} + +function toReasons(counts: Map): OutcomeReason[] { + return [...counts].map(([label, { item, count }]) => ({ + label, + count, + detail: describe(item, count), + })); +} + +/** + * Classifies every user into exactly one outcome, worst first. + * + * Social rows are left out: which providers a user signed up with lives in the + * raw export rather than the transformed `User`, so they cannot be counted per + * user here. Their coverage row still names them. + */ +function countOutcomes(users: User[], blocking: ReadinessItem[]): ImportOutcomes { + const rejecting = blocking.filter((item) => item.consequence === "rejects"); + const dropping = blocking.filter( + (item) => item.consequence === "drops" && item.section !== "social", + ); + + type Tally = Map; + const rejectedBy: Tally = new Map(); + const droppedBy: Tally = new Map(); + const maskedBy: Tally = new Map(); + let rejected = 0; + let incomplete = 0; + let complete = 0; + + const tally = (into: Tally, item: ReadinessItem) => { + const entry = into.get(item.label) ?? { item, count: 0 }; + entry.count++; + into.set(item.label, entry); + }; + + for (const entry of users) { + const user = entry as unknown as Record; + const gaps = dropping.filter((item) => affects(item, user)); + + const refusals = rejecting.filter((item) => affects(item, user)); + if (refusals.length > 0) { + rejected++; + for (const item of refusals) tally(rejectedBy, item); + // Their gaps are still tallied, into a separate bucket. Dropping them + // here is what makes the settings interact invisibly: relaxing the + // requirement that rejects these users lets them in, and only then does + // whatever else affects them show up — a second round trip to learn + // something that was knowable now. + for (const item of gaps) tally(maskedBy, item); + continue; + } + + if (gaps.length === 0) { + complete++; + continue; + } + + incomplete++; + for (const item of gaps) tally(droppedBy, item); + } + + return { + rejected, + rejectedReasons: toReasons(rejectedBy), + maskedReasons: toReasons(maskedBy), + incomplete, + incompleteReasons: toReasons(droppedBy), + complete, }; } @@ -193,11 +380,59 @@ function renderItem(item: ReadinessItem, total: number): string { return ` ${yellow("!")} ${item.label} — ${dim(`${coverage} — check it is enabled in Clerk`)}`; } +const users = (count: number) => `${count} user${count === 1 ? "" : "s"}`; + +/** + * The three outcomes, each with the reasons behind it. + * + * This is the part of the report that answers "so what": which users the + * instance will refuse, which will arrive with something missing, and why. + * Per-field coverage lives further down and is a different question. + */ +function renderOutcomes(outcomes: ImportOutcomes): string[] { + const lines: string[] = []; + + const group = ( + symbol: string, + colour: (text: string) => string, + headline: string, + reasons: OutcomeReason[], + ) => { + lines.push(` ${colour(symbol)} ${colour(headline)}`); + for (const reason of reasons) lines.push(` ${dim(reason.detail)}`); + }; + + if (outcomes.rejected > 0) { + group("✗", red, `${users(outcomes.rejected)} will not be imported`, outcomes.rejectedReasons); + + // Named here rather than left for a second run of the report: these are the + // settings that start costing something the moment the rejection above is + // lifted, and lifting it is one of the changes on offer. + if (outcomes.maskedReasons.length > 0) { + lines.push(` ${dim("If you import them, this applies to them too:")}`); + for (const reason of outcomes.maskedReasons) lines.push(` ${dim(reason.detail)}`); + } + } + if (outcomes.incomplete > 0) { + group( + "⚠", + yellow, + `${users(outcomes.incomplete)} will be imported, but not everything they carry`, + outcomes.incompleteReasons, + ); + } + if (outcomes.complete > 0) { + lines.push(` ${green("✓")} ${green(`${users(outcomes.complete)} will be imported in full`)}`); + } + + return lines; +} + /** Renders the report for a human, as lines. */ export function formatReadinessReport(report: ReadinessReport): string[] { const lines: string[] = [bold("Migration readiness")]; - lines.push(` ${report.totalUsers} user${report.totalUsers === 1 ? "" : "s"} ready to import`); + lines.push(` ${users(report.totalUsers)} in this file`); if (report.validationFailed > 0) { lines.push(` ${yellow(`${report.validationFailed} failed validation and will be skipped`)}`); } @@ -207,6 +442,11 @@ export function formatReadinessReport(report: ReadinessReport): string[] { ); } + if (report.outcomes) { + const outcomeLines = renderOutcomes(report.outcomes); + if (outcomeLines.length > 0) lines.push("", ...outcomeLines); + } + if (report.settingsUnavailable) { lines.push( "", diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index c5355da20..44198a6f7 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -14,23 +14,46 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { getMode, setMode, type Mode } from "../../mode.ts"; -import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; +import { keylessTargetStubs, listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; +import type { InstanceTarget } from "../../lib/keyless-target.ts"; const mockSelect = mock(async () => "clerk" as unknown); const mockText = mock(async () => "export.json" as unknown); +type MultiselectConfig = { options: { value: string; label: string; hint?: string }[] }; +const mockMultiselect = mock(async (_config: MultiselectConfig) => [] as unknown[]); let confirmAnswer = true; let originalMode: Mode; +const ACCOUNT_TARGET: InstanceTarget = { + kind: "account", + ctx: { + appId: "app_1", + appLabel: "Migration Test", + instanceId: "ins_1", + instanceLabel: "development", + }, + label: "Migration Test (development)", +}; +let instanceTarget: InstanceTarget | Error = ACCOUNT_TARGET; + mock.module("../../lib/listage.ts", () => ({ ...listageStubs, select: (...args: unknown[]) => mockSelect(...(args as [])), })); +mock.module("../../lib/keyless-target.ts", () => ({ + ...keylessTargetStubs, + resolveInstanceTarget: async () => { + if (instanceTarget instanceof Error) throw instanceTarget; + return instanceTarget; + }, +})); + // Every export of the real module must appear here — a missing one is a link // error at import time, which takes down the whole file rather than one prompt. mock.module("../../lib/prompts.ts", () => ({ confirm: async () => confirmAnswer, - multiselect: async () => [], + multiselect: (...args: unknown[]) => mockMultiselect(...(args as [MultiselectConfig])), text: (...args: unknown[]) => mockText(...(args as [])), password: async () => "", editor: async () => "{}", @@ -57,11 +80,17 @@ const EXPORT = [ const baseOptions = { transformer: "clerk", file: "export.json", secretKey: "sk_test_x" }; +let originalPlatformKey: string | undefined; + beforeAll(() => { originalMode = getMode(); setMode("human"); originalCwd = process.cwd(); originalFetch = globalThis.fetch; + // Pinned rather than inherited: the settings-fix write goes through the + // Platform API, and CI has neither a `.env.local` nor a login session. + originalPlatformKey = process.env.CLERK_PLATFORM_API_KEY; + process.env.CLERK_PLATFORM_API_KEY = "ak_test"; workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-config-")); _setConfigDir(configDir); @@ -71,6 +100,8 @@ beforeAll(() => { afterAll(() => { setMode(originalMode); globalThis.fetch = originalFetch; + if (originalPlatformKey === undefined) delete process.env.CLERK_PLATFORM_API_KEY; + else process.env.CLERK_PLATFORM_API_KEY = originalPlatformKey; _setConfigDir(undefined); process.chdir(originalCwd); fs.rmSync(workDir, { recursive: true, force: true }); @@ -80,10 +111,13 @@ afterAll(() => { beforeEach(() => { requests = []; confirmAnswer = true; + instanceTarget = ACCOUNT_TARGET; mockSelect.mockReset(); mockText.mockReset(); + mockMultiselect.mockReset(); mockSelect.mockResolvedValue("clerk"); mockText.mockResolvedValue("export.json"); + mockMultiselect.mockResolvedValue([]); fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); fs.rmSync(path.join(configDir, "config.json"), { force: true }); fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); @@ -94,8 +128,16 @@ afterEach(() => { process.exitCode = 0; }); +type StubSettings = { attributes?: object; social?: object } | null; + +let currentSettings: StubSettings = null; +/** What the instance reports once a config PATCH lands, when a test sets one. */ +let settingsAfterFix: StubSettings = null; + /** Stubs BAPI plus the FAPI environment lookup the readiness report needs. */ -function stubInstanceSettings(settings: { attributes?: object; social?: object } | null) { +function stubInstanceSettings(settings: StubSettings) { + currentSettings = settings; + settingsAfterFix = null; globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { const url = input.toString(); requests.push({ @@ -104,13 +146,17 @@ function stubInstanceSettings(settings: { attributes?: object; social?: object } body: init?.body ? JSON.parse(init.body as string) : null, }); if (url.endsWith("/v1/domains")) { - if (!settings) return new Response("nope", { status: 500 }); + if (!currentSettings) return new Response("nope", { status: 500 }); return Response.json({ data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], }); } if (url.includes("/v1/dev_browser")) return Response.json({ token: "jwt" }); - if (url.includes("/v1/environment")) return Response.json({ user_settings: settings }); + if (url.includes("/v1/environment")) return Response.json({ user_settings: currentSettings }); + if (url.endsWith("/instances/ins_1/config")) { + if (settingsAfterFix) currentSettings = settingsAfterFix; + return Response.json({ config_version: "v1_patched" }); + } return Response.json({ id: "user_created" }); }) as unknown as typeof fetch; } @@ -151,7 +197,7 @@ describe("the readiness report", () => { test("renders before the confirmation", async () => { await run(baseOptions); expect(captured.err).toContain("Migration readiness"); - expect(captured.err).toContain("2 users ready to import"); + expect(captured.err).toContain("2 users in this file"); }); // The whole point of the report: seeing what will go wrong, then backing out @@ -190,7 +236,10 @@ describe("the readiness report", () => { await run(baseOptions); - expect(captured.err).toContain("1 user lacks it"); + // The outcome block is the point: one user has no email, and an instance + // that requires one will refuse them. + expect(captured.err).toContain("1 user will not be imported"); + expect(captured.err).toContain("1 has no email, which this instance requires"); expect(captured.err).toContain("1 setting needs attention"); }); @@ -212,6 +261,207 @@ describe("the readiness report", () => { }); }); +describe("fixing the instance's settings from the report", () => { + /** An export whose second user has no email, against a required-email instance. */ + function blockedOnRequiredEmail() { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", username: "bob" }, + ]), + ); + } + + /** Two flagged rows: email required with a user lacking it, username switched off. */ + function blockedOnTwoSettings() { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: false }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev", username: "alice" }, + { id: "u2", username: "bob" }, + ]), + ); + } + + const patched = () => requests.filter((r) => r.url.endsWith("/instances/ins_1/config")); + const offered = (round = 0) => mockMultiselect.mock.calls[round]?.[0]?.options ?? []; + + // Plain labels only: the config leaf each one writes is internal detail an + // operator cannot act on and does not need to read. + test("offers one change per blocking row, named in plain terms", async () => { + blockedOnRequiredEmail(); + + await run(baseOptions); + + expect(offered()).toEqual([ + { value: "email_address", label: "Make Email optional at sign-up" }, + ]); + }); + + test("does not ask when nothing is blocking", async () => { + await run(baseOptions); + + expect(mockMultiselect).not.toHaveBeenCalled(); + expect(created()).toHaveLength(2); + }); + + // Relaxing an instance's sign-up requirements is a real decision, so nothing + // is preselected and an empty answer must leave the instance untouched. + test("selecting nothing changes nothing and continues to the import", async () => { + blockedOnRequiredEmail(); + mockMultiselect.mockResolvedValue([]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(created()).toHaveLength(2); + }); + + test("selecting a change patches the instance and re-renders the report", async () => { + blockedOnRequiredEmail(); + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(1); + expect(patched()[0]).toMatchObject({ + method: "PATCH", + body: { auth_email: { required_for_sign_up: false } }, + }); + expect(captured.err).toContain("Updated 1 setting"); + // The redraw clears the row that was just fixed, so the confirmation that + // follows is against the settings the write established. + expect(captured.err).toContain("Every field in this file is configured in Clerk"); + expect(created()).toHaveLength(2); + }); + + /** + * The redraw must not re-read the Frontend API. It is eventually consistent, + * so a fetch this soon after the write returns the pre-write settings and + * redraws the report with every row the operator just cleared still flagged. + */ + test("redraws from the write rather than re-reading stale settings", async () => { + blockedOnRequiredEmail(); + // Anything read back now would still say "required" — as it did in practice. + settingsAfterFix = { + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }; + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(requests.filter((r) => r.url.includes("/v1/environment"))).toHaveLength(1); + expect(captured.err).toContain("Every field in this file is configured in Clerk"); + // Flagged in the first report, and only there — the redraw is clean even + // though a re-read at this moment would still have reported it. + expect(captured.err.split("setting needs attention")).toHaveLength(2); + }); + + // A keyless application is only reachable through the Backend API, which has + // no route for any of these settings — saying so beats a confusing rejection. + test("stands down for a keyless application and still imports", async () => { + blockedOnRequiredEmail(); + instanceTarget = { + kind: "keyless", + keyless: { secretKey: "sk_test_x", source: ".env" }, + label: "keyless", + }; + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(captured.err).toContain("clerk auth login"); + expect(created()).toHaveLength(2); + }); + + /** + * Each redraw is another decision point, not a receipt. Applying one change + * routinely leaves others still worth making, and an operator should not have + * to re-run the whole command to reach them. + */ + describe("offering again while anything is still flagged", () => { + test("re-offers what is left, without the change already applied", async () => { + blockedOnTwoSettings(); + mockMultiselect.mockResolvedValueOnce(["email_address"]); + mockMultiselect.mockResolvedValueOnce(["username"]); + + await run(baseOptions); + + expect(offered(0).map((option) => option.value)).toEqual(["email_address", "username"]); + expect(offered(1).map((option) => option.value)).toEqual(["username"]); + expect(patched()).toHaveLength(2); + expect(patched()[1]).toMatchObject({ + body: { auth_username: { used_for_sign_up: true } }, + }); + }); + + test("stops once nothing is flagged, rather than asking again", async () => { + blockedOnTwoSettings(); + mockMultiselect.mockResolvedValueOnce(["email_address"]); + mockMultiselect.mockResolvedValueOnce(["username"]); + + await run(baseOptions); + + expect(mockMultiselect).toHaveBeenCalledTimes(2); + expect(captured.err).toContain("Every field in this file is configured in Clerk"); + expect(created()).toHaveLength(2); + }); + + test("stops when the operator skips, leaving the rest flagged", async () => { + blockedOnTwoSettings(); + mockMultiselect.mockResolvedValueOnce(["email_address"]); + mockMultiselect.mockResolvedValueOnce([]); + + await run(baseOptions); + + expect(mockMultiselect).toHaveBeenCalledTimes(2); + expect(patched()).toHaveLength(1); + expect(created()).toHaveLength(2); + }); + + // A selection naming nothing on offer is the same as no selection, and must + // not become an empty PATCH. + test("sends nothing when the selection matches no offered change", async () => { + blockedOnRequiredEmail(); + mockMultiselect.mockResolvedValue(["not_a_real_change"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(created()).toHaveLength(2); + }); + }); + + test("warns instead of failing the run when the instance cannot be resolved", async () => { + blockedOnRequiredEmail(); + instanceTarget = new Error("not linked"); + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(captured.err).toContain("nothing was changed"); + expect(created()).toHaveLength(2); + }); +}); + describe("guards that still apply interactively", () => { test("the dev-instance 500-user cap", async () => { fs.writeFileSync( diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 1698a2517..4cd88959d 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -409,7 +409,7 @@ describe("run", () => { await run({ ...baseOptions, yes: false }); expect(captured.err).toContain("Migration readiness"); - expect(captured.err).toContain("1 user lacks it"); + expect(captured.err).toContain("1 user will not be imported"); // The report was printed before the first POST /v1/users. const reportIndex = requests.findIndex((r) => r.url.includes("/v1/environment")); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 7167b854b..712aa756b 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -13,11 +13,13 @@ import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; import { bold, dim, green, red, yellow } from "../../lib/color.ts"; import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; +import { resolveInstanceTarget, type InstanceTarget } from "../../lib/keyless-target.ts"; import { log } from "../../lib/log.ts"; import { NEXT_STEPS } from "../../lib/next-steps.ts"; -import { confirm } from "../../lib/prompts.ts"; +import { confirm, multiselect } from "../../lib/prompts.ts"; import { withGutter, withSpinner } from "../../lib/spinner.ts"; import { isAgent, isHuman } from "../../mode.ts"; +import { writeInstanceConfig } from "../config/io.ts"; import { importUsers } from "./import-users.ts"; import { analyzeFields } from "./lib/analysis.ts"; import { findMigrateEnvValue } from "./lib/env-file.ts"; @@ -26,7 +28,18 @@ import { fetchInstanceSettings, toClerkStrategy, } from "./lib/clerk-config.ts"; -import { buildReadinessReport, formatReadinessReport } from "./lib/readiness.ts"; +import { + buildReadinessReport, + DASHBOARD_URL, + formatReadinessReport, + type ReadinessReport, +} from "./lib/readiness.ts"; +import { + applyChanges, + buildChangePayload, + buildSettingChanges, + type SettingChange, +} from "./lib/modify-settings.ts"; import { DEV_USER_LIMIT, resolveLimits } from "./lib/instance.ts"; import { getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; import { saveSettings } from "./lib/settings.ts"; @@ -307,32 +320,19 @@ async function skipDisabledProviderUsers( return users.filter((user) => !excludedIds.has(user.userId)); } -/** - * Prints the Migration Readiness report: what the file contains, cross- - * referenced against what the destination instance accepts. - * - * Rendered immediately before the confirmation prompt, so declining that - * prompt aborts with nothing written to Clerk. - * - * Skipped only for `-y`, which says "don't ask, don't lecture" and should not - * pay for two extra network round-trips. Agent mode still gets it: an agent - * driving a migration can act on "this field is required and 40 users lack it" - * exactly as a human would. - */ -async function showReadinessReport(input: { +type ReportInput = { users: User[]; file: string; transformer: string; secretKey: string; validationFailed: number; - skipReport: boolean; -}): Promise { - if (input.skipReport) return; - - const settings = await withSpinner("Checking instance settings...", () => - fetchInstanceSettings(input.secretKey), - ); +}; +/** + * Everything the report needs except the instance's settings — the half that + * comes from the file, and so does not change when the instance does. + */ +async function readFileSide(input: ReportInput) { // Only Supabase exports record per-user providers, so only they can be // cross-referenced against the instance's social connections. let providerCounts: Record | undefined; @@ -344,18 +344,134 @@ async function showReadinessReport(input: { } } - const report = buildReadinessReport({ + return { analysis: analyzeFields(input.users), - settings, validationFailed: input.validationFailed, providerCounts, - }); + }; +} +function printReport(report: ReadinessReport): void { log.blank(); for (const line of formatReadinessReport(report)) log.info(line); log.blank(); } +/** + * Offers to change the instance's settings, one selectable change per flagged + * row. + * + * Without this the report names something the operator has to leave the CLI to + * act on. Nothing is preselected and selecting nothing continues to the import + * prompt unchanged: a flagged setting is not a wrong setting, and relaxing an + * instance's sign-up requirements is a real decision rather than a default. + * + * @returns The changes that were written, so the caller can redraw the report. + */ +async function offerSettingChanges( + report: ReadinessReport, + options: MigrateRunOptions, +): Promise { + const changes = buildSettingChanges(report.blocking); + if (changes.length === 0) return []; + + // Navigation keys are in the prompt's own footer; what that footer cannot say + // is that selecting nothing is a valid answer rather than an unfinished one. + const chosen = await multiselect({ + message: "Update this instance's settings first? (enter to skip)", + options: changes.map((change) => ({ value: change.id, label: change.label })), + initialValues: [], + required: false, + }); + // Filtered before anything is resolved or sent: a selection that matches no + // offered change is the same as no selection, and must not become an empty + // PATCH. + const applied = changes.filter((change) => chosen.includes(change.id)); + if (applied.length === 0) return []; + + // Resolved here rather than up front: an operator who selects nothing should + // not pay for a Platform API round-trip, and a target that cannot be resolved + // (a bare `--secret-key` against an unlinked directory) should not fail the + // whole run before the report has even been offered. + let target: InstanceTarget; + try { + target = await resolveInstanceTarget({ app: options.app, instance: options.instance }); + } catch (error) { + log.warn( + "Could not resolve which instance to configure, so nothing was changed. " + + "Link a project with `clerk link`, or pass `--app `.", + ); + log.debug(`migrate: settings change target unresolved: ${String(error)}`); + return []; + } + + // The Backend API a keyless application is reachable through has no route for + // any of these settings — `config patch` rejects the same payload by name. + if (target.kind === "keyless") { + log.warn( + "These settings need an account to change. Run `clerk auth login` to claim this application, " + + `then re-run, or update them at ${DASHBOARD_URL}.`, + ); + return []; + } + + await withSpinner(`Updating settings on ${target.label}...`, () => + writeInstanceConfig(target, buildChangePayload(applied), { + method: "PATCH", + failureContext: "Failed to update instance settings", + }), + ); + log.success(`Updated ${applied.length} setting${applied.length === 1 ? "" : "s"}.`); + + return applied; +} + +/** + * Prints the Migration Readiness report: what the file contains, cross- + * referenced against what the destination instance accepts. + * + * Rendered immediately before the confirmation prompt, so declining that + * prompt aborts with nothing written to Clerk. + * + * Skipped only for `-y`, which says "don't ask, don't lecture" and should not + * pay for two extra network round-trips. Agent mode still gets it: an agent + * driving a migration can act on "10 users will not be imported, because email + * is required" exactly as a human would — but not the prompt, which needs one. + */ +async function showReadinessReport( + input: ReportInput & { skipReport: boolean; options: MigrateRunOptions }, +): Promise { + if (input.skipReport) return; + + let settings = await withSpinner("Checking instance settings...", () => + fetchInstanceSettings(input.secretKey), + ); + const fileSide = { ...(await readFileSide(input)), users: input.users }; + + let report = buildReadinessReport({ ...fileSide, settings }); + printReport(report); + + if (!isHuman() || isAgent()) return; + + // Every redraw is another decision point, not a receipt. Applying one change + // routinely leaves others still worth making — and can surface consequences + // that were masked behind the row just cleared — so the offer repeats for as + // long as the report has something to offer. + while (report.blocking.length > 0) { + const applied = await offerSettingChanges(report, input.options); + // Nothing selected, nothing offerable, or nowhere to write it: the operator + // has said their piece and the import prompt is next. + if (applied.length === 0) return; + + // Redrawn from the write, not from a re-read. Clerk's Frontend API is + // eventually consistent, so fetching settings again here routinely returns + // the pre-write ones and redraws every row the operator just cleared. + settings = applyChanges(settings, applied); + report = buildReadinessReport({ ...fileSide, settings }); + printReport(report); + } +} + /** * Fills in a missing `--transformer`/`--file` interactively, or explains what * to pass. @@ -510,6 +626,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { secretKey, validationFailed, skipReport: Boolean(options.yes), + options, }); if (!options.yes && isHuman() && !isAgent()) { From 18cef5e4440ca47aabb0bb42d07df4555ebf48b5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:26:06 -0400 Subject: [PATCH 13/34] feat(migrate): document `clerk migrate` rather than `clerk migrate run` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run` is registered `isDefault`, so `clerk migrate --transformer clerk --file users.json` has always worked and is the shorter spelling. Every example, error message, next-step line and README invocation now uses it. `clerk migrate run` stays addressable — scripts and older docs use it — but nothing points there. The group's own help follows `clerk config`: options stay on the subcommands, and `migrate --help` is a list of subcommands plus examples covering each one. `transformers` had no examples block at all; it does now. Two fixes this turned up: - A partial `CLERK_FIREBASE_*` set left in `.env.clerk-migrate` failed *every* subsequent run, including a Supabase one that never asked for Firebase. That was a regression from routing those values through the env file: previously only explicit flags could trigger the all-four-or-nothing check. A partial set that came from saved config is now warned about and ignored; a partial set that came from flags still fails, because there the user did ask. - `readme.test.ts` resolved a documented command to its group and read only that group's options, so every `clerk migrate --transformer …` example looked like it used a flag the binary rejects. It now follows the default subcommand, the same way Commander does. `migrate delete`'s description said "in this directory"; the record it reads has been keyed by project since the `.settings` removal. --- .../cli-core/src/commands/migrate/README.md | 32 +++++----- .../cli-core/src/commands/migrate/delete.ts | 2 +- .../src/commands/migrate/export/auth0.test.ts | 4 +- .../src/commands/migrate/export/clerk.test.ts | 8 +-- .../src/commands/migrate/export/clerk.ts | 2 +- .../commands/migrate/export/firebase.test.ts | 2 +- .../src/commands/migrate/export/firebase.ts | 2 +- .../src/commands/migrate/export/index.ts | 2 +- .../cli-core/src/commands/migrate/index.ts | 59 ++++++++++++++----- .../src/commands/migrate/readme.test.ts | 18 +++++- .../cli-core/src/commands/migrate/run.test.ts | 17 +++++- packages/cli-core/src/commands/migrate/run.ts | 34 ++++++++--- .../migrate/settings/settings.test.ts | 1 + .../cli-core/src/commands/migrate/wizard.ts | 2 +- packages/cli-core/src/lib/config.ts | 2 +- packages/cli-core/src/lib/next-steps.ts | 2 +- 16 files changed, 130 insertions(+), 59 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 3c5b180a3..5f49a5b10 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -5,7 +5,7 @@ Clerk instance. ## Targeting And Auth -`migrate run` resolves its Backend API key through the CLI's standard chain: +`clerk migrate` resolves its Backend API key through the CLI's standard chain: | Flag | Description | | ------------------------ | ---------------------------------------------------------------- | @@ -26,7 +26,7 @@ defaults and the hard development-instance cap below. ### `clerk migrate` (interactive) -Bare `clerk migrate` dispatches to `migrate run`, which walks a human through +Bare `clerk migrate` walks a human through the migration instead of demanding flags — mirroring how bare `clerk deploy` dispatches to `deploy run`. @@ -52,13 +52,13 @@ error naming exactly what to pass: Pass --transformer and --file . ``` -### `clerk migrate run` +### `clerk migrate` Reads an exported user file, maps it onto Clerk's user schema, validates every record, and creates the users through the Backend API. ```sh -clerk migrate run -y --transformer clerk --file users.json +clerk migrate -y --transformer clerk --file users.json ``` | Flag | Description | @@ -121,7 +121,7 @@ limit — the run fails before any request is sent. ### `clerk migrate export` Gets users **out** of a source platform, so there is something to feed -`migrate run`. +`clerk migrate`. ```sh clerk migrate export # pick a platform @@ -158,7 +158,7 @@ other path flag here. | `--client-secret ` | `auth0` | Machine-to-machine application client secret | `export clerk` also takes the targeting flags — it reads from a Clerk instance, -so it resolves a key exactly the way `migrate run` does. +so it resolves a key exactly the way `clerk migrate` does. After each export you get a field-coverage table — which Clerk-relevant fields were present on how many users — so you know the data is thin _before_ you @@ -173,7 +173,7 @@ Field coverage Exported 3 users to /project/exports/clerk-export.json └ Next steps - → Run `clerk migrate run --transformer clerk --file exports/clerk-export.json` to import them + → Run `clerk migrate --transformer clerk --file exports/clerk-export.json` to import them ``` Every export also writes `logs/export-.log`, so `migrate logs list` @@ -259,7 +259,7 @@ prints the exact import command: ``` Password hash parameters Read from the project. Import with: - clerk migrate run -y --transformer firebase --file exports/firebase-export.json \ + clerk migrate -y --transformer firebase --file exports/firebase-export.json \ --firebase-signer-key "…" --firebase-salt-separator "…" \ --firebase-rounds 8 --firebase-mem-cost 14 ``` @@ -297,7 +297,7 @@ returning the first thousand would read as "that is everyone". ### `clerk migrate delete` -The undo for a bad migration. Deletes the users a previous `clerk migrate run` +The undo for a bad migration. Deletes the users a previous `clerk migrate` created in this directory, matched by the `external_id` the import stamped on each one. @@ -306,7 +306,7 @@ clerk migrate delete # confirms first clerk migrate delete -y # non-interactive ``` -Takes the same targeting flags as `migrate run` (`--secret-key`, `--app`, +Takes the same targeting flags as `clerk migrate` (`--secret-key`, `--app`, `--instance`). Flat rather than under a noun group: it is the one command in this tree that @@ -462,7 +462,7 @@ firebase-signer-key aVer…3456 .env.clerk-migrate Firebase base64 sig firebase-rounds — not set Firebase scrypt rounds ``` -Setting names are kebab-case and identical to the `migrate run` flag each one +Setting names are kebab-case and identical to the `clerk migrate` flag each one backs, so `firebase-signer-key` here is `--firebase-signer-key` there rather than a second spelling to learn. The description column carries the prose. @@ -500,7 +500,7 @@ so the output is safe to paste into an issue. Migrating from a platform with no built-in, without recompiling the CLI: ```sh -clerk migrate run --transformer-file ./my-platform.ts --file users.json +clerk migrate --transformer-file ./my-platform.ts --file users.json ``` The file lives in **your** project, not in the CLI, and is imported at runtime. @@ -572,7 +572,7 @@ alongside each digest. Find them in the Firebase console under **Authentication → Users → (⋮) → Password hash parameters**. ```sh -clerk migrate run -y -t firebase -f users.json \ +clerk migrate -y -t firebase -f users.json \ --firebase-signer-key --firebase-salt-separator \ --firebase-rounds 8 --firebase-mem-cost 14 ``` @@ -905,9 +905,9 @@ NDJSON is. The original `.log` stays put. | Method | Path | Used by | | -------- | -------------------------- | ------------------------------------------------------------------------------------ | -| `POST` | `/v1/users` | `migrate run` — creates each user | -| `POST` | `/v1/email_addresses` | `migrate run` — attaches additional emails | -| `POST` | `/v1/phone_numbers` | `migrate run` — attaches additional phones | +| `POST` | `/v1/users` | `clerk migrate` — creates each user | +| `POST` | `/v1/email_addresses` | `clerk migrate` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `clerk migrate` — attaches additional phones | | `GET` | `/v1/users?external_id=…` | `migrate delete` — finds this migration's users, 100 IDs a call | | `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | | `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index db2791205..39394ceff 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -74,7 +74,7 @@ export async function resolveMigrationToUndo(): Promise<{ file: string; key: str if (!settings.file || !settings.transformer) { throw new CliError( - "No migration to undo: this project has no record of a previous `clerk migrate run`.\n" + + "No migration to undo: this project has no record of a previous `clerk migrate`.\n" + "Run `clerk migrate delete` from the project you migrated from.", { code: ERROR_CODE.FILE_NOT_FOUND }, ); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index ceef2915a..9746f79cb 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -300,9 +300,7 @@ describe("exportAuth0", () => { } finally { setMode(originalMode); } - expect(captured.err).toContain( - "migrate run --transformer auth0 --file exports/auth0-export.json", - ); + expect(captured.err).toContain("migrate --transformer auth0 --file exports/auth0-export.json"); }); test("--output controls the destination", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 9ce14d656..b628a8408 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -255,9 +255,7 @@ describe("exportClerk", () => { } finally { setMode(originalMode); } - expect(captured.err).toContain( - "migrate run --transformer clerk --file exports/clerk-export.json", - ); + expect(captured.err).toContain("migrate --transformer clerk --file exports/clerk-export.json"); }); test("--output controls the destination, relative to the working directory", async () => { @@ -300,7 +298,7 @@ describe("exportClerk", () => { expect(captured.err).toContain("No users found to export"); expect(captured.err).not.toContain("Next steps"); - expect(captured.err).not.toContain("migrate run --transformer"); + expect(captured.err).not.toContain("migrate --transformer"); }); test("agent mode suppresses the Next steps block", async () => { @@ -310,6 +308,6 @@ describe("exportClerk", () => { expect(captured.err).toContain("Exported 1 user"); expect(captured.err).not.toContain("Next steps"); - expect(captured.err).not.toContain("migrate run --transformer"); + expect(captured.err).not.toContain("migrate --transformer"); }); }); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index a49fa581e..564bcabd2 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -5,7 +5,7 @@ * onto `bapiRequest` instead of `@clerk/backend` so it shares the CLI's auth * resolution, `--verbose` request tracing and error taxonomy. * - * The output feeds `clerk migrate run --transformer clerk` unedited, which is + * The output feeds `clerk migrate --transformer clerk` unedited, which is * what makes development → production a two-command operation. * * **Passwords do not come out of this endpoint.** Clerk never returns password diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 137d0502d..1de54a3c8 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -425,7 +425,7 @@ describe("exportFirebase", () => { setMode(originalMode); } expect(captured.err).toContain( - "migrate run --transformer firebase --file exports/firebase-export.json", + "migrate --transformer firebase --file exports/firebase-export.json", ); }); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 9c70a410b..aedb8fb9c 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -397,7 +397,7 @@ export function formatHashConfigGuidance( bold("Password hash parameters"), "Read from the project. Import with:", dim( - ` clerk migrate run -y --transformer firebase --file ${outputPath} \\\n` + + ` clerk migrate -y --transformer firebase --file ${outputPath} \\\n` + ` --firebase-signer-key "${config.signerKey}" \\\n` + ` --firebase-salt-separator "${config.saltSeparator}" \\\n` + ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 2ac69b5b3..773e07f22 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -84,7 +84,7 @@ const DB_PLATFORMS = [ export function registerMigrateExport(migrateCommand: Command<[], Record>): void { const exportCommand = migrateCommand .command("export") - .description("Export users from a source platform, ready for `migrate run`") + .description("Export users from a source platform, ready for `clerk migrate`") .setExamples([ { command: "clerk migrate export", description: "Pick a platform interactively" }, { diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index a1a56e1c7..b17195af4 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -16,19 +16,37 @@ export function registerMigrate(program: Program): void { .command("migrate") .description("Migrate users into Clerk from another auth provider") .setExamples([ + { command: "clerk migrate", description: "Walk through a migration interactively" }, { - command: "clerk migrate", - description: "Walk through a migration interactively", + command: "clerk migrate -y --transformer clerk --file users.json", + description: "Import users from a Clerk export", }, { - command: "clerk migrate run -y --transformer clerk --file users.json", - description: "Import users from a Clerk export", + command: "clerk migrate -y -t supabase -f users.json --skip-unsupported-providers", + description: "Skip Supabase users whose provider is not enabled", + }, + { + command: "clerk migrate export supabase", + description: "Export users from Supabase, ready to import", }, + { command: "clerk migrate settings", description: "Show what a run here would pick up" }, + { + command: "clerk migrate settings set firebase-signer-key abc123", + description: "Save a credential to .env.clerk-migrate", + }, + { command: "clerk migrate logs", description: "List the local migration logs" }, + { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { command: "clerk migrate delete", description: "Undo the last migration" }, ]); - // `isDefault` so bare `clerk migrate` runs the wizard, mirroring how bare - // `clerk deploy` dispatches to `deploy run`. Not hidden: unlike deploy's, - // this subcommand is documented and carries every flag. + // `isDefault` so `clerk migrate` is the whole command: bare, it runs the + // wizard; with flags, they fall through to here. `run` stays addressable + // because scripts and older docs use it, but `clerk migrate` is the spelling + // every example gives. + // + // The flags stay here rather than on `migrate`, matching how `config` keeps + // its own on `pull`/`patch`/`put` — a group's help is a list of subcommands + // and examples, not a merge of everything underneath it. migrateCommand .command("run", { isDefault: true }) .description("Import users from an exported JSON or CSV file") @@ -51,10 +69,10 @@ export function registerMigrate(program: Program): void { ) .option("--firebase-signer-key ", "Firebase base64 signer key") .option("--firebase-salt-separator ", "Firebase base64 salt separator") - .option("--firebase-rounds ", "Firebase scrypt rounds", (value) => + .option("--firebase-rounds ", "Firebase scrypt rounds", (value: string) => parseIntegerOption(value, "--firebase-rounds", { min: 1 }), ) - .option("--firebase-mem-cost ", "Firebase scrypt memory cost", (value) => + .option("--firebase-mem-cost ", "Firebase scrypt memory cost", (value: string) => parseIntegerOption(value, "--firebase-mem-cost", { min: 1 }), ) .option("-y, --yes", "Skip the confirmation prompt") @@ -64,19 +82,19 @@ export function registerMigrate(program: Program): void { .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") .setExamples([ { - command: "clerk migrate run -y --transformer clerk --file users.json", + command: "clerk migrate -y --transformer clerk --file users.json", description: "Import a Clerk Dashboard export", }, { - command: "clerk migrate run -y -t clerk -f users.csv --require-password", + command: "clerk migrate -y -t clerk -f users.csv --require-password", description: "Import only the users that carry a password digest", }, { - command: "clerk migrate run -y -t clerk -f users.json -r user_2x9k", + command: "clerk migrate -y -t clerk -f users.json -r user_2x9k", description: "Resume a partial migration after the last imported user", }, { - command: "clerk migrate run -y -t supabase -f users.json --skip-unsupported-providers", + command: "clerk migrate -y -t supabase -f users.json --skip-unsupported-providers", description: "Skip Supabase users whose only provider is not enabled in Clerk", }, ]) @@ -88,7 +106,7 @@ export function registerMigrate(program: Program): void { // destroys data in Clerk, and it is worth keeping short and prominent. migrateCommand .command("delete") - .description("Delete the users created by the last migration in this directory") + .description("Delete the users created by the last migration for this project") .option("-y, --yes", "Skip the confirmation prompt") .option("--secret-key ", "Backend API secret key to use") .option("--clerk-secret-key ", "Deprecated alias for --secret-key") @@ -111,7 +129,18 @@ export function registerMigrate(program: Program): void { // need a command rather than only appearing in the interactive picker. const transformersCommand = migrateCommand .command("transformers") - .description("Inspect the available source-platform transformers"); + .description("Inspect the available source-platform transformers") + .setExamples([ + { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { + command: "clerk migrate transformers list --json", + description: "Machine-readable, including each one's ID field", + }, + { + command: "clerk migrate transformers list --transformer-file ./my-transformer.ts", + description: "Include one you wrote", + }, + ]); transformersCommand .command("list", { isDefault: true }) diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index ef5a05f85..f41aec567 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -60,10 +60,26 @@ function resolve(tokens: string[]): { command: Command; rest: string[] } { return { command, rest: tokens.slice(index) }; } +/** + * The flags a command accepts, including those of a default subcommand. + * + * `clerk migrate` carries no options of its own — `run` is registered + * `isDefault`, so Commander hands it everything after the group name. The + * documented spelling is `clerk migrate --transformer …`, and this has to see + * the same flags Commander does or every such example reads as unsupported. + */ function flagsOf(command: Command): string[] { - return command.options.flatMap( + const own = command.options.flatMap( (option) => [option.short, option.long].filter(Boolean) as string[], ); + + const defaultChild = command.commands.find( + // Commander records the default subcommand on the parent, not the child. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (child) => child.name() === (command as any)._defaultCommandName, + ); + + return defaultChild ? [...own, ...flagsOf(defaultChild)] : own; } /** Every command under `migrate`, so no subcommand escapes the flag sweep. */ diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 4cd88959d..ad30e0e03 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -98,6 +98,7 @@ describe("resolveFirebaseHashConfig", () => { }); describe("environment fallback", () => { + const captured = useCaptureLog(); const ENV = { CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER", CLERK_FIREBASE_SALT_SEPARATOR: "Bw==", @@ -142,9 +143,21 @@ describe("resolveFirebaseHashConfig", () => { }); }); - test("still demands the full set when the environment supplies only part", async () => { + // Stale saved config, not an instruction: a signer key left over from a + // Firebase migration must not fail the Supabase run that follows it. The + // flag path stays strict — see "rejects a set missing %s" above. + test("ignores a partial set rather than failing a run that never asked for it", async () => { setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); - await expect(resolveFirebaseHashConfig({})).rejects.toThrow(/--firebase-salt-separator/); + + expect(await resolveFirebaseHashConfig({})).toBeUndefined(); + expect(captured.err).toContain("Ignoring an incomplete Firebase hash configuration"); + }); + + test("still fails when a flag supplied part of the set", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); + await expect(resolveFirebaseHashConfig({ firebaseRounds: 8 })).rejects.toThrow( + /--firebase-salt-separator/, + ); }); // An empty var is how a shell spells "unset", and treating it as set would diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 712aa756b..1d8b4c16d 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -1,13 +1,15 @@ /** - * `clerk migrate run` — non-interactive user import. + * `clerk migrate` — the user import itself. * * Ported from the standalone migration-tool's `src/migrate/cli.ts` * (`runNonInteractive`), with auth moved onto the CLI's standard secret-key * resolution chain and every failure raised as a `CliError` instead of * `console.error` + `process.exit`. * - * The interactive wizard that a bare `clerk migrate` will launch is a separate - * command; this path is the one an agent or a script drives. + * Registered as the `run` subcommand and marked default, so `clerk migrate` and + * `clerk migrate run` both land here. Whatever the flags did not supply is + * filled in by `wizard.ts` for a human, or raised as a usage error naming the + * missing flags for an agent, which cannot answer a prompt. */ import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; @@ -122,6 +124,7 @@ async function withFirebaseEnv(options: MigrateRunOptions): Promise { + const fromFlags = FIREBASE_FLAGS.filter(([key]) => rawOptions[key] !== undefined); const options = await withFirebaseEnv(rawOptions); const provided = FIREBASE_FLAGS.filter(([key]) => options[key] !== undefined); @@ -131,6 +134,20 @@ export async function resolveFirebaseHashConfig( const missing = FIREBASE_FLAGS.filter(([key]) => options[key] === undefined).map( ([, flag]) => flag, ); + + // A partial set nobody asked for on this command line is stale saved + // config, not an instruction: a `CLERK_FIREBASE_SIGNER_KEY` left in + // `.env.clerk-migrate` after a Firebase migration must not fail the + // Supabase run that follows it. Warned rather than dropped silently, + // because on a Firebase run it is the reason passwords will not import. + if (fromFlags.length === 0) { + log.warn( + `Ignoring an incomplete Firebase hash configuration (no ${missing.join(", ")}). ` + + "Run `clerk migrate settings` to see what is set.", + ); + return undefined; + } + throwUsageError( `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", @@ -167,8 +184,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: - "clerk migrate run -y --transformer-file ./my-transformer.ts --file users.json", + command: "clerk migrate -y --transformer-file ./my-transformer.ts --file users.json", description: "Import with a custom transformer", }, ], @@ -190,7 +206,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate run -y --transformer clerk --file users.json", + command: "clerk migrate -y --transformer clerk --file users.json", description: "Import a Clerk export", }, ], @@ -208,7 +224,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate run -y --transformer clerk --file users.json", + command: "clerk migrate -y --transformer clerk --file users.json", description: "Import a Clerk export", }, ], @@ -530,11 +546,11 @@ async function applyCustomTransformer(options: MigrateRunOptions): Promise { await list(); + // eslint-disable-next-line no-control-regex const plain = captured.err.replaceAll(/\u001B\[\d+m/g, ""); const columnOf = (description: string) => plain diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index d5b556edb..d855e9b26 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -151,7 +151,7 @@ export function throwAgentFlagsRequired(missing: { transformer: boolean; file: b undefined, [ { - command: `clerk migrate run -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, + command: `clerk migrate -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, description: "Run non-interactively", }, ], diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index 8139d9ca0..f77a9602e 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -50,7 +50,7 @@ interface RelayEntry { token: string; } -/** What `clerk migrate run` last imported for a project, and how. */ +/** What `clerk migrate` last imported for a project, and how. */ interface MigrationEntry { transformer?: string; file?: string; diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index 613fe1042..41d415f90 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -79,7 +79,7 @@ export const NEXT_STEPS = { // The only parameterized entry: a suggested import is worthless unless it // names the transformer that reads this export and the file just written. MIGRATE_EXPORT: (transformerKey: string, file: string) => [ - `Run \`clerk migrate run --transformer ${transformerKey} --file ${file}\` to import them`, + `Run \`clerk migrate --transformer ${transformerKey} --file ${file}\` to import them`, ], } as const; From 85afd339e6476ae5e335bc47356d0df50f14a4fe Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 6 Aug 2026 17:34:13 -0400 Subject: [PATCH 14/34] fix(migrate): read the Firebase hash parameters only when the transformer is firebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `migrate run` is one command serving every platform, so a `CLERK_FIREBASE_SIGNER_KEY` left in `.env.clerk-migrate` after a Firebase migration was in scope for whatever ran next. A complete leftover set was resolved and passed along on a Supabase import; a partial one failed that import outright, naming four `--firebase-*` flags the user had not used and did not need. The gate now sits before the lookup rather than being a filter after it: any transformer but `firebase` returns immediately, without reading the environment, the env files, or even its own flags. Nothing downstream misused the value — only the Firebase transformer reads it off `TransformContext` — but resolving it at all is what let stale config warn and fail unrelated runs. The previous fix only covered the partial case, and did it transformer-blind. Moved to `lib/firebase-hash.ts` so the wizard can resolve after the platform is picked without importing from `run.ts`, which imports the wizard. That also keeps the interactive path: choosing Firebase with all four already set skips the prompt, choosing anything else never looks. The per-platform export commands need no equivalent gate — `migrate export auth0` reads `AUTH0_*` and nothing else, because there the command *is* the platform. `migrate run` is the only one that spans them. --- .../migrate/lib/firebase-hash.test.ts | 172 ++++++++++++++++++ .../src/commands/migrate/lib/firebase-hash.ts | 119 ++++++++++++ .../cli-core/src/commands/migrate/run.test.ts | 114 +----------- packages/cli-core/src/commands/migrate/run.ts | 110 +---------- .../cli-core/src/commands/migrate/wizard.ts | 24 ++- 5 files changed, 317 insertions(+), 222 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/firebase-hash.ts diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts new file mode 100644 index 000000000..ccf5fa75e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts @@ -0,0 +1,172 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { resolveFirebaseHashConfig } from "./firebase-hash.ts"; + +const captured = useCaptureLog(); + +const ALL_FLAGS = { + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, +}; + +const ENV = { + CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER", + CLERK_FIREBASE_SALT_SEPARATOR: "Bw==", + CLERK_FIREBASE_ROUNDS: "8", + CLERK_FIREBASE_MEM_COST: "14", +}; + +let workDir: string; +let originalCwd: string; + +const setEnv = (vars: Partial) => Object.assign(process.env, vars); + +beforeEach(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-fbhash-"))); + process.chdir(workDir); +}); + +afterEach(() => { + for (const name of Object.keys(ENV)) delete process.env[name]; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("gating on the transformer", () => { + // `migrate run` is one command for every platform, so a signer key left in + // .env.clerk-migrate after a Firebase migration is in scope for whatever runs + // next unless the transformer says otherwise. + test.each([["clerk"], ["supabase"], ["auth0"], ["authjs"], ["betterauth"]])( + "reads nothing for the %s transformer", + async (transformer) => { + setEnv(ENV); + expect(await resolveFirebaseHashConfig({}, transformer)).toBeUndefined(); + }, + ); + + test("stays silent about a complete set on another platform's run", async () => { + setEnv(ENV); + await resolveFirebaseHashConfig({}, "supabase"); + expect(captured.err).toBe(""); + }); + + // The case that regressed: half a set used to fail every later run. + test("stays silent about a partial set on another platform's run", async () => { + fs.writeFileSync(path.join(workDir, ".env.clerk-migrate"), "CLERK_FIREBASE_SIGNER_KEY=left\n"); + + expect(await resolveFirebaseHashConfig({}, "supabase")).toBeUndefined(); + expect(captured.err).toBe(""); + }); + + test("ignores even explicit flags when the platform is not firebase", async () => { + expect(await resolveFirebaseHashConfig(ALL_FLAGS, "supabase")).toBeUndefined(); + }); + + test("resolves nothing before the platform is known", async () => { + setEnv(ENV); + expect(await resolveFirebaseHashConfig({}, undefined)).toBeUndefined(); + }); +}); + +describe("on a firebase run", () => { + test("builds the config from flags", async () => { + expect(await resolveFirebaseHashConfig(ALL_FLAGS, "firebase")).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + test("falls back to the environment", async () => { + setEnv(ENV); + expect((await resolveFirebaseHashConfig({}, "firebase"))?.base64_signer_key).toBe("ENV_SIGNER"); + }); + + test("reads .env.clerk-migrate when the variable is not exported", async () => { + fs.writeFileSync( + path.join(workDir, ".env.clerk-migrate"), + Object.entries(ENV) + .map(([key, value]) => `${key}=${value}`) + .join("\n"), + ); + + expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); + }); + + test("prefers a flag over the environment", async () => { + setEnv(ENV); + expect((await resolveFirebaseHashConfig(ALL_FLAGS, "firebase"))?.base64_signer_key).toBe( + "SIGNER", + ); + }); + + test("fills only the gaps the flags left", async () => { + setEnv({ CLERK_FIREBASE_ROUNDS: "8", CLERK_FIREBASE_MEM_COST: "14" }); + + expect( + await resolveFirebaseHashConfig( + { firebaseSignerKey: "SIGNER", firebaseSaltSeparator: "Bw==" }, + "firebase", + ), + ).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + // A digest built from a partial set is well-formed but verifies against + // nothing, so every migrated user would silently fail to sign in. + test.each([ + ["firebaseSignerKey", "--firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost"], + ] as const)("rejects a flag set missing %s, naming it", async (omit, flag) => { + const partial = { ...ALL_FLAGS }; + delete (partial as Record)[omit]; + + await expect(resolveFirebaseHashConfig(partial, "firebase")).rejects.toThrow(new RegExp(flag)); + }); + + test("names every missing flag at once", async () => { + await expect( + resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" }, "firebase"), + ).rejects.toThrow(/--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/); + }); + + // Saved config is a leftover, not an instruction — but on a Firebase import + // it is the reason the passwords will not come across, so it is said aloud. + test("warns and continues when only saved config is partial", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); + + expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); + expect(captured.err).toContain("Ignoring an incomplete Firebase hash configuration"); + }); + + test("still fails when a flag supplied part of the set", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); + + await expect(resolveFirebaseHashConfig({ firebaseRounds: 8 }, "firebase")).rejects.toThrow( + /--firebase-salt-separator/, + ); + }); + + // An empty variable is how a shell spells "unset". + test("ignores an empty variable", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "" }); + expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); + }); + + test("returns nothing when neither flags nor the environment supply a config", async () => { + expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts new file mode 100644 index 000000000..836d374ea --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts @@ -0,0 +1,119 @@ +/** + * Firebase's four scrypt parameters: where they come from, and when they are + * looked for at all. + * + * **Only read when the transformer is `firebase`.** `migrate run` is one + * command serving every platform, so a `CLERK_FIREBASE_SIGNER_KEY` left in + * `.env.clerk-migrate` after a Firebase migration is in scope for the Supabase + * run that follows it unless something says otherwise. Nothing downstream would + * misuse it — only the Firebase transformer reads the config off + * {@link TransformContext} — but resolving it means a stale or partial set can + * warn, or fail, a run that never mentioned Firebase. So the gate is here, + * before the lookup, rather than a filter after it. + * + * The per-platform export commands need no such gate: `migrate export auth0` + * reads `AUTH0_*` and nothing else, because the command itself is the platform. + * This is the only place one command spans them all. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { findMigrateEnvValue } from "./env-file.ts"; +import type { FirebaseHashConfig } from "../types.ts"; + +/** The `--firebase-*` flags, and the variable each falls back to. */ +export const FIREBASE_FLAGS = [ + ["firebaseSignerKey", "--firebase-signer-key", "CLERK_FIREBASE_SIGNER_KEY"], + ["firebaseSaltSeparator", "--firebase-salt-separator", "CLERK_FIREBASE_SALT_SEPARATOR"], + ["firebaseRounds", "--firebase-rounds", "CLERK_FIREBASE_ROUNDS"], + ["firebaseMemCost", "--firebase-mem-cost", "CLERK_FIREBASE_MEM_COST"], +] as const; + +const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); + +export type FirebaseHashFlags = { + firebaseSignerKey?: string; + firebaseSaltSeparator?: string; + firebaseRounds?: number; + firebaseMemCost?: number; +}; + +/** + * Overlays the `CLERK_FIREBASE_*` values onto whichever flags were not passed. + * + * Resolved through {@link findMigrateEnvValue}: the environment first, then + * `.env.clerk-migrate`, then the app's own `.env` files. The signer key is a + * Firebase secret, so it is never written to the CLI's config — + * `.env.clerk-migrate` is gitignored on creation. + */ +async function withFirebaseEnv(flags: FirebaseHashFlags): Promise { + const merged: FirebaseHashFlags = { ...flags }; + for (const [key, , envVar] of FIREBASE_FLAGS) { + if (merged[key] !== undefined) continue; + const located = await findMigrateEnvValue([envVar]); + if (!located || located.value.trim() === "") continue; + // A non-numeric round count is left to fail the flag's own validation + // rather than silently becoming NaN. + (merged as Record)[key] = FIREBASE_NUMERIC.has(key) + ? Number(located.value) + : located.value; + } + return merged; +} + +/** + * Resolves the four parameters from flags, then the `CLERK_FIREBASE_*` + * variables, then the project's env files. + * + * The four are required as a set: a digest built from a partial set is + * well-formed but verifies against nothing, so every migrated user would fail + * to sign in with no error at import time. How a partial set is treated depends + * on where it came from — flags are an instruction, saved config is not. + * + * @param transformer - The platform being migrated. Anything but `firebase` + * returns immediately, without reading the environment. + * @returns The config, or `undefined` when none was supplied — which is fine + * for an export that carries no password hashes. + */ +export async function resolveFirebaseHashConfig( + flags: FirebaseHashFlags, + transformer: string | undefined, +): Promise { + if (transformer !== "firebase") return undefined; + + const fromFlags = FIREBASE_FLAGS.filter(([key]) => flags[key] !== undefined); + const resolved = await withFirebaseEnv(flags); + const provided = FIREBASE_FLAGS.filter(([key]) => resolved[key] !== undefined); + + if (provided.length === 0) return undefined; + + if (provided.length < FIREBASE_FLAGS.length) { + const missing = FIREBASE_FLAGS.filter(([key]) => resolved[key] === undefined).map( + ([, flag]) => flag, + ); + + // Saved config is a leftover, not an instruction: half a set in + // `.env.clerk-migrate` should not fail the run, but on a Firebase import it + // is the reason the passwords will not come across, so it is said out loud. + if (fromFlags.length === 0) { + log.warn( + `Ignoring an incomplete Firebase hash configuration (no ${missing.join(", ")}). ` + + "Run `clerk migrate settings` to see what is set.", + ); + return undefined; + } + + throwUsageError( + `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + + "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + "https://clerk.com/docs/guides/development/migrating/firebase", + ); + } + + return { + base64_signer_key: resolved.firebaseSignerKey as string, + base64_salt_separator: resolved.firebaseSaltSeparator as string, + rounds: resolved.firebaseRounds as number, + mem_cost: resolved.firebaseMemCost as number, + }; +} diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index ad30e0e03..9d9272f95 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -8,7 +8,7 @@ import { useCaptureLog } from "../../test/lib/stubs.ts"; import { getLogDir } from "./lib/logger.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; import { loadSettings } from "./lib/settings.ts"; -import { applyResumeAfter, resolveFirebaseHashConfig, run, validateRunOptions } from "./run.ts"; +import { applyResumeAfter, run, validateRunOptions } from "./run.ts"; import type { User } from "./types.ts"; let workDir: string; @@ -61,118 +61,6 @@ describe("validateRunOptions", () => { }); }); -describe("resolveFirebaseHashConfig", () => { - const ALL = { - firebaseSignerKey: "SIGNER", - firebaseSaltSeparator: "Bw==", - firebaseRounds: 8, - firebaseMemCost: 14, - }; - - test("builds the config when all four flags are present", async () => { - expect(await resolveFirebaseHashConfig(ALL)).toEqual({ - base64_signer_key: "SIGNER", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }); - }); - - // A digest built from a partial set is well-formed but verifies against - // nothing, so every migrated user would silently fail to sign in. - test.each([ - ["firebaseSignerKey", "--firebase-signer-key"], - ["firebaseSaltSeparator", "--firebase-salt-separator"], - ["firebaseRounds", "--firebase-rounds"], - ["firebaseMemCost", "--firebase-mem-cost"], - ] as const)("rejects a set missing %s, naming the flag", async (omit, flag) => { - const partial = { ...ALL }; - delete (partial as Record)[omit]; - await expect(resolveFirebaseHashConfig(partial)).rejects.toThrow(new RegExp(flag)); - }); - - test("names every missing flag at once", async () => { - await expect(resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" })).rejects.toThrow( - /--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/, - ); - }); - - describe("environment fallback", () => { - const captured = useCaptureLog(); - const ENV = { - CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER", - CLERK_FIREBASE_SALT_SEPARATOR: "Bw==", - CLERK_FIREBASE_ROUNDS: "8", - CLERK_FIREBASE_MEM_COST: "14", - }; - - afterEach(() => { - for (const name of Object.keys(ENV)) delete process.env[name]; - }); - - const setEnv = (vars: Partial) => Object.assign(process.env, vars); - - test("builds the config when no flag is passed", async () => { - setEnv(ENV); - expect(await resolveFirebaseHashConfig({})).toEqual({ - base64_signer_key: "ENV_SIGNER", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }); - }); - - test("prefers a flag over the environment", async () => { - setEnv(ENV); - expect((await resolveFirebaseHashConfig(ALL))?.base64_signer_key).toBe("SIGNER"); - }); - - // Half from the environment and half from flags is still a complete set. - test("fills only the gaps the flags left", async () => { - setEnv({ CLERK_FIREBASE_ROUNDS: "8", CLERK_FIREBASE_MEM_COST: "14" }); - expect( - await resolveFirebaseHashConfig({ - firebaseSignerKey: "SIGNER", - firebaseSaltSeparator: "Bw==", - }), - ).toEqual({ - base64_signer_key: "SIGNER", - base64_salt_separator: "Bw==", - rounds: 8, - mem_cost: 14, - }); - }); - - // Stale saved config, not an instruction: a signer key left over from a - // Firebase migration must not fail the Supabase run that follows it. The - // flag path stays strict — see "rejects a set missing %s" above. - test("ignores a partial set rather than failing a run that never asked for it", async () => { - setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); - - expect(await resolveFirebaseHashConfig({})).toBeUndefined(); - expect(captured.err).toContain("Ignoring an incomplete Firebase hash configuration"); - }); - - test("still fails when a flag supplied part of the set", async () => { - setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); - await expect(resolveFirebaseHashConfig({ firebaseRounds: 8 })).rejects.toThrow( - /--firebase-salt-separator/, - ); - }); - - // An empty var is how a shell spells "unset", and treating it as set would - // demand the other three for a config nobody asked for. - test("ignores an empty variable", async () => { - setEnv({ CLERK_FIREBASE_SIGNER_KEY: "" }); - expect(await resolveFirebaseHashConfig({})).toBeUndefined(); - }); - }); - - test("returns nothing when neither flags nor the environment supply a config", async () => { - expect(await resolveFirebaseHashConfig({})).toBeUndefined(); - }); -}); - describe("applyResumeAfter", () => { test("returns everything when no ID is given", () => { expect(applyResumeAfter(users("a", "b"), undefined)).toHaveLength(2); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 1d8b4c16d..bdcd865e9 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -24,7 +24,7 @@ import { isAgent, isHuman } from "../../mode.ts"; import { writeInstanceConfig } from "../config/io.ts"; import { importUsers } from "./import-users.ts"; import { analyzeFields } from "./lib/analysis.ts"; -import { findMigrateEnvValue } from "./lib/env-file.ts"; +import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebase-hash.ts"; import { enabledSocialProviders, fetchInstanceSettings, @@ -54,7 +54,7 @@ import { import { fileExists, getFileType, loadUsersFromFile } from "./lib/transform.ts"; import { loadCustomTransformer } from "./transformers/load-custom.ts"; import { registerCustomTransformer, transformerKeys } from "./transformers/registry.ts"; -import type { FirebaseHashConfig, ImportSummary, User } from "./types.ts"; +import type { ImportSummary, User } from "./types.ts"; import { runWizard, throwAgentFlagsRequired } from "./wizard.ts"; export type MigrateRunOptions = { @@ -72,96 +72,7 @@ export type MigrateRunOptions = { transformerFile?: string; /** Supabase: drop users whose only social provider is disabled in Clerk. */ skipUnsupportedProviders?: boolean; - firebaseSignerKey?: string; - firebaseSaltSeparator?: string; - firebaseRounds?: number; - firebaseMemCost?: number; -}; - -const FIREBASE_FLAGS = [ - ["firebaseSignerKey", "--firebase-signer-key", "CLERK_FIREBASE_SIGNER_KEY"], - ["firebaseSaltSeparator", "--firebase-salt-separator", "CLERK_FIREBASE_SALT_SEPARATOR"], - ["firebaseRounds", "--firebase-rounds", "CLERK_FIREBASE_ROUNDS"], - ["firebaseMemCost", "--firebase-mem-cost", "CLERK_FIREBASE_MEM_COST"], -] as const; - -const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); - -/** - * Overlays the `CLERK_FIREBASE_*` values onto whichever flags were not passed. - * - * Resolved through {@link findMigrateEnvValue}: the environment first, then - * `.env.clerk-migrate`, then the app's own `.env` files. The signer key is a - * Firebase secret, so it is never written to the CLI's config — - * `.env.clerk-migrate` is gitignored on creation. - */ -async function withFirebaseEnv(options: MigrateRunOptions): Promise { - const merged = { ...options }; - for (const [key, , envVar] of FIREBASE_FLAGS) { - if (merged[key] !== undefined) continue; - const located = await findMigrateEnvValue([envVar]); - if (!located || located.value.trim() === "") continue; - // A non-numeric round count is left to fail the flag's own validation - // rather than silently becoming NaN. - (merged as Record)[key] = FIREBASE_NUMERIC.has(key) - ? Number(located.value) - : located.value; - } - return merged; -} - -/** - * Resolves Firebase's four hash parameters from flags, falling back to the - * `CLERK_FIREBASE_*` environment variables and the project's `.env` files. - * - * The four are required as a set: a digest built from a partial set is - * well-formed but verifies against nothing, so every migrated user would fail - * to sign in with no error at import time. - * - * @returns The config, or `undefined` when none was supplied — which is fine - * for an export that carries no password hashes. - */ -export async function resolveFirebaseHashConfig( - rawOptions: MigrateRunOptions, -): Promise { - const fromFlags = FIREBASE_FLAGS.filter(([key]) => rawOptions[key] !== undefined); - const options = await withFirebaseEnv(rawOptions); - const provided = FIREBASE_FLAGS.filter(([key]) => options[key] !== undefined); - - if (provided.length === 0) return undefined; - - if (provided.length < FIREBASE_FLAGS.length) { - const missing = FIREBASE_FLAGS.filter(([key]) => options[key] === undefined).map( - ([, flag]) => flag, - ); - - // A partial set nobody asked for on this command line is stale saved - // config, not an instruction: a `CLERK_FIREBASE_SIGNER_KEY` left in - // `.env.clerk-migrate` after a Firebase migration must not fail the - // Supabase run that follows it. Warned rather than dropped silently, - // because on a Firebase run it is the reason passwords will not import. - if (fromFlags.length === 0) { - log.warn( - `Ignoring an incomplete Firebase hash configuration (no ${missing.join(", ")}). ` + - "Run `clerk migrate settings` to see what is set.", - ); - return undefined; - } - - throwUsageError( - `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + - "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", - "https://clerk.com/docs/guides/development/migrating/firebase", - ); - } - - return { - base64_signer_key: options.firebaseSignerKey as string, - base64_salt_separator: options.firebaseSaltSeparator as string, - rounds: options.firebaseRounds as number, - mem_cost: options.firebaseMemCost as number, - }; -} +} & FirebaseHashFlags; /** * Validates the flags a run needs before anything is read or sent. @@ -504,14 +415,11 @@ async function resolveMissingOptions(options: MigrateRunOptions): Promise { const secretKeyOption = options.secretKey ?? options.clerkSecretKey; const { transformer, file } = validateRunOptions(options); - const firebaseHashConfig = await resolveFirebaseHashConfig(options); + const firebaseHashConfig = await resolveFirebaseHashConfig(options, transformer); await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index d855e9b26..ff6e7173e 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -14,6 +14,7 @@ import { throwUsageError } from "../../lib/errors.ts"; import { select } from "../../lib/listage.ts"; import { log } from "../../lib/log.ts"; import { text } from "../../lib/prompts.ts"; +import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebase-hash.ts"; import { loadSettings } from "./lib/settings.ts"; import { fileExists, getFileType } from "./lib/transform.ts"; import { transformers } from "./transformers/registry.ts"; @@ -112,11 +113,13 @@ async function askNumber(label: string): Promise { * * @param provided - Flags the caller already supplied; those are not asked for. */ -export async function runWizard(provided: { - transformer?: string; - file?: string; - firebaseHashConfig?: FirebaseHashConfig; -}): Promise { +export async function runWizard( + provided: { + transformer?: string; + file?: string; + firebaseHashConfig?: FirebaseHashConfig; + } & FirebaseHashFlags, +): Promise { const saved = await loadSettings(); const transformer = provided.transformer ?? (await pickTransformer(saved.transformer)); @@ -124,9 +127,14 @@ export async function runWizard(provided: { let firebaseHashConfig = provided.firebaseHashConfig; if (transformer === "firebase" && !firebaseHashConfig) { - // Never prefilled: the signer key is a secret the CLI does not keep. A - // repeat run supplies it through `--firebase-*` or `CLERK_FIREBASE_*`, - // which short-circuits this prompt entirely. + // Looked up here rather than before the picker: until the platform is + // chosen there is no reason to read Firebase's variables at all, and a + // migration from anywhere else must not see them. + firebaseHashConfig = await resolveFirebaseHashConfig(provided, "firebase"); + } + if (transformer === "firebase" && !firebaseHashConfig) { + // Prompted, never prefilled: the signer key is a secret the CLI does not + // keep, so there is nothing to offer back. firebaseHashConfig = await askFirebaseHashConfig(); } From 3a667d791f469b275610495e63eb02707f0c9a8d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:28:29 -0400 Subject: [PATCH 15/34] chore: ignore local migration exports and service account keys `clerk migrate export` writes real user records to ./exports, and the Firebase export is driven by a service account key people download into the checkout. Neither belongs in the repository, and both are one `git add -A` away from it. --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index c5f69f393..9ab506561 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,7 @@ test/e2e/.har # Local planning/spec docs docs/superpowers/ + +# Local migration exports and the credentials that produced them +exports/ +*service-account*.json From 829d97985fb578cf3b83eb3ad1b24a89f37daa6d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:28:38 -0400 Subject: [PATCH 16/34] fix(config): persist the migration entry to disk `setMigrationEntry` mutated the in-memory config and returned without writing it, so nothing recorded what the last import did. `clerk migrate delete` reads that entry to find the users to undo, and with it never written the undo path had nothing to work from. --- packages/cli-core/src/lib/config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index 6d2faf342..c93b359e1 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -238,6 +238,7 @@ export async function setMigrationEntry(key: string, entry: MigrationEntry): Pro const config = await readConfig(); if (!config.migrations) config.migrations = {}; config.migrations[key] = entry; + await writeConfig(config); } /** Persistent random machine id for telemetry. Generated on first use. */ From 9a96503a3977b2b845ce841d794acee745a8caa9 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:29:51 -0400 Subject: [PATCH 17/34] refactor(migrate)!: rename `migrate run` to `migrate import`, drop --clerk-secret-key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run` was registered `isDefault`, so `clerk migrate` on its own meant "import". That reads fine until `migrate export` sits beside it: one direction is implied by the bare group name and the other has to be spelled out. Both are named now, and bare `clerk migrate` prints help. The `--clerk-secret-key` alias goes with it. It was carried over from the standalone migration tool, but `migrate` ships new in this CLI — there is no released spelling to stay compatible with, so there is nothing to deprecate. --- .changeset/migrate-cli.md | 2 +- .../cli-core/src/commands/migrate/README.md | 80 ++++++++++--------- .../cli-core/src/commands/migrate/delete.ts | 12 +-- .../src/commands/migrate/export/clerk.ts | 14 +--- .../src/commands/migrate/export/firebase.ts | 4 +- .../src/commands/migrate/export/index.ts | 3 +- .../src/commands/migrate/index.test.ts | 42 ++++------ .../cli-core/src/commands/migrate/index.ts | 28 +++---- .../src/commands/migrate/lib/clerk-config.ts | 2 +- .../migrate/lib/firebase-hash.test.ts | 2 +- .../src/commands/migrate/lib/firebase-hash.ts | 2 +- .../src/commands/migrate/lib/retry.ts | 2 +- .../src/commands/migrate/readme.test.ts | 8 +- .../commands/migrate/run-interactive.test.ts | 4 +- .../cli-core/src/commands/migrate/run.test.ts | 7 -- packages/cli-core/src/commands/migrate/run.ts | 35 ++++---- .../src/commands/migrate/settings/registry.ts | 2 +- .../migrate/settings/settings.test.ts | 2 +- .../commands/migrate/transformers/registry.ts | 2 +- .../cli-core/src/commands/migrate/wizard.ts | 6 +- packages/cli-core/src/lib/config.ts | 2 +- packages/cli-core/src/lib/next-steps.ts | 2 +- 22 files changed, 116 insertions(+), 147 deletions(-) diff --git a/.changeset/migrate-cli.md b/.changeset/migrate-cli.md index 42864a475..6ba3096b0 100644 --- a/.changeset/migrate-cli.md +++ b/.changeset/migrate-cli.md @@ -2,4 +2,4 @@ "clerk": minor --- -Add `clerk migrate` for importing users, exporting from supported auth providers, reviewing migration logs, undoing a migration, and extending imports with custom transformers. +Add `clerk migrate` for importing users with `migrate import`, exporting from supported auth providers, reviewing migration logs, undoing a migration, and extending imports with custom transformers. diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 5f49a5b10..53dcc2b44 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -5,14 +5,14 @@ Clerk instance. ## Targeting And Auth -`clerk migrate` resolves its Backend API key through the CLI's standard chain: +`clerk migrate import` resolves its Backend API key through the CLI's standard +chain: -| Flag | Description | -| ------------------------ | ---------------------------------------------------------------- | -| `--secret-key ` | Use a specific Backend API secret key directly | -| `--clerk-secret-key ` | **Deprecated** alias for `--secret-key`; warns and keeps working | -| `--app ` | Target an application directly, even outside a linked project | -| `--instance ` | Target `dev`, `prod`, or a full instance ID | +| Flag | Description | +| -------------------- | ------------------------------------------------------------- | +| `--secret-key ` | Use a specific Backend API secret key directly | +| `--app ` | Target an application directly, even outside a linked project | +| `--instance ` | Target `dev`, `prod`, or a full instance ID | Resolution order: `--secret-key` → `--app` + Platform API lookup → `CLERK_SECRET_KEY` → the keyless project's own key → a linked project profile @@ -24,14 +24,18 @@ defaults and the hard development-instance cap below. ## Commands -### `clerk migrate` (interactive) +`clerk migrate` on its own is a group name, not a command: it prints its help +and lists the subcommands below. The direction is always spelled out — +`migrate import` moves users **into** Clerk, `migrate export` gets them **out** +of a source platform — so neither is implied by the group. -Bare `clerk migrate` walks a human through -the migration instead of demanding flags — mirroring how bare `clerk deploy` -dispatches to `deploy run`. +### `clerk migrate import` (interactive) + +Bare `clerk migrate import` walks a human through the import instead of +demanding flags. ```sh -clerk migrate +clerk migrate import ``` It picks the transformer from a list built off the registry, asks for the file, @@ -44,21 +48,21 @@ Then it prints the [Migration Readiness report](#migration-readiness-report), offers to [change whatever it flagged](#changing-the-flagged-settings), and waits for confirmation. Declining writes nothing to Clerk. -**Agent mode never prompts.** `clerk migrate` with no flags exits with a usage -error naming exactly what to pass: +**Agent mode never prompts.** `clerk migrate import` with no flags exits with a +usage error naming exactly what to pass: ``` -`clerk migrate` is interactive and cannot prompt in agent mode. +`clerk migrate import` is interactive and cannot prompt in agent mode. Pass --transformer and --file . ``` -### `clerk migrate` +### `clerk migrate import` Reads an exported user file, maps it onto Clerk's user schema, validates every record, and creates the users through the Backend API. ```sh -clerk migrate -y --transformer clerk --file users.json +clerk migrate import -y --transformer clerk --file users.json ``` | Flag | Description | @@ -75,8 +79,8 @@ clerk migrate -y --transformer clerk --file users.json | `--firebase-mem-cost ` | Firebase scrypt memory cost | | `-y, --yes` | Skip the confirmation prompt | -Plus the targeting flags from the table above: `--secret-key`, -`--clerk-secret-key`, `--app` and `--instance`. +Plus the targeting flags from the table above: `--secret-key`, `--app` and +`--instance`. `--transformer` and `--file` are required. Omitting either fails with a usage error that names the valid values. @@ -121,7 +125,7 @@ limit — the run fails before any request is sent. ### `clerk migrate export` Gets users **out** of a source platform, so there is something to feed -`clerk migrate`. +`clerk migrate import`. ```sh clerk migrate export # pick a platform @@ -259,7 +263,7 @@ prints the exact import command: ``` Password hash parameters Read from the project. Import with: - clerk migrate -y --transformer firebase --file exports/firebase-export.json \ + clerk migrate import -y --transformer firebase --file exports/firebase-export.json \ --firebase-signer-key "…" --firebase-salt-separator "…" \ --firebase-rounds 8 --firebase-mem-cost 14 ``` @@ -297,16 +301,16 @@ returning the first thousand would read as "that is everyone". ### `clerk migrate delete` -The undo for a bad migration. Deletes the users a previous `clerk migrate` -created in this directory, matched by the `external_id` the import stamped on -each one. +The undo for a bad migration. Deletes the users a previous +`clerk migrate import` created in this directory, matched by the `external_id` +the import stamped on each one. ```sh clerk migrate delete # confirms first clerk migrate delete -y # non-interactive ``` -Takes the same targeting flags as `clerk migrate` (`--secret-key`, `--app`, +Takes the same targeting flags as `clerk migrate import` (`--secret-key`, `--app`, `--instance`). Flat rather than under a noun group: it is the one command in this tree that @@ -462,7 +466,7 @@ firebase-signer-key aVer…3456 .env.clerk-migrate Firebase base64 sig firebase-rounds — not set Firebase scrypt rounds ``` -Setting names are kebab-case and identical to the `clerk migrate` flag each one +Setting names are kebab-case and identical to the `clerk migrate import` flag each one backs, so `firebase-signer-key` here is `--firebase-signer-key` there rather than a second spelling to learn. The description column carries the prose. @@ -500,7 +504,7 @@ so the output is safe to paste into an issue. Migrating from a platform with no built-in, without recompiling the CLI: ```sh -clerk migrate --transformer-file ./my-platform.ts --file users.json +clerk migrate import --transformer-file ./my-platform.ts --file users.json ``` The file lives in **your** project, not in the CLI, and is imported at runtime. @@ -572,7 +576,7 @@ alongside each digest. Find them in the Firebase console under **Authentication → Users → (⋮) → Password hash parameters**. ```sh -clerk migrate -y -t firebase -f users.json \ +clerk migrate import -y -t firebase -f users.json \ --firebase-signer-key --firebase-salt-separator \ --firebase-rounds 8 --firebase-mem-cost 14 ``` @@ -859,13 +863,13 @@ Both are written relative to the **current working directory**, not to the CLI's config directory, because they describe "which file am I migrating" rather than "which project is linked here". -| Path | Contents | -| -------------------------------------- | --------------------------------------------------------------------- | -| `./logs/migration-.log` | NDJSON: one line per user, plus validation failures and retry notices | -| `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | -| `./logs/export-.log` | NDJSON: one line per exported user | -| `./exports/-export.json` | The export itself, unless `--output` says otherwise | -| `./.env.clerk-migrate` | Migration credentials, written by `settings set` and gitignored | +| Path | Contents | +| ------------------------------------------ | --------------------------------------------------------------------- | +| `./logs/migration-.log` | NDJSON: one line per user, plus validation failures and retry notices | +| `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | +| `./logs/export-.log` | NDJSON: one line per exported user | +| `./exports/-export-.json` | The export itself, unless `--output` says otherwise | +| `./.env.clerk-migrate` | Migration credentials, written by `settings set` and gitignored | The transformer and file of the last run are **not** written here. They go to the `migrations` section of the CLI's own config file, keyed by project the @@ -905,9 +909,9 @@ NDJSON is. The original `.log` stays put. | Method | Path | Used by | | -------- | -------------------------- | ------------------------------------------------------------------------------------ | -| `POST` | `/v1/users` | `clerk migrate` — creates each user | -| `POST` | `/v1/email_addresses` | `clerk migrate` — attaches additional emails | -| `POST` | `/v1/phone_numbers` | `clerk migrate` — attaches additional phones | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | | `GET` | `/v1/users?external_id=…` | `migrate delete` — finds this migration's users, 100 IDs a call | | `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | | `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 39394ceff..06781d3cd 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -50,7 +50,6 @@ const EXTERNAL_ID_BATCH = 100; export type MigrateDeleteOptions = { yes?: boolean; secretKey?: string; - clerkSecretKey?: string; app?: string; instance?: string; }; @@ -74,7 +73,7 @@ export async function resolveMigrationToUndo(): Promise<{ file: string; key: str if (!settings.file || !settings.transformer) { throw new CliError( - "No migration to undo: this project has no record of a previous `clerk migrate`.\n" + + "No migration to undo: this project has no record of a previous `clerk migrate import`.\n" + "Run `clerk migrate delete` from the project you migrated from.", { code: ERROR_CODE.FILE_NOT_FOUND }, ); @@ -258,16 +257,11 @@ function formatSummary(summary: DeleteSummary, logFile: string): string { } export async function deleteMigration(options: MigrateDeleteOptions): Promise { - if (options.clerkSecretKey) { - log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); - } - const secretKeyOption = options.secretKey ?? options.clerkSecretKey; - const { file, key } = await resolveMigrationToUndo(); await withGutter("Undoing a migration", async ({ setNextSteps }) => { - const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); - const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const limits = resolveLimits(secretKey); const dateTime = getDateTimeStamp(); const logFile = getLogFilePath("user-deletion", dateTime); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 564bcabd2..a9ff2f062 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -5,7 +5,7 @@ * onto `bapiRequest` instead of `@clerk/backend` so it shares the CLI's auth * resolution, `--verbose` request tracing and error taxonomy. * - * The output feeds `clerk migrate --transformer clerk` unedited, which is + * The output feeds `clerk migrate import --transformer clerk` unedited, which is * what makes development → production a two-command operation. * * **Passwords do not come out of this endpoint.** Clerk never returns password @@ -28,7 +28,6 @@ const PAGE_SIZE = 500; export type ExportClerkOptions = { output?: string; secretKey?: string; - clerkSecretKey?: string; app?: string; instance?: string; }; @@ -67,7 +66,7 @@ type IdentifierWithId = BapiIdentifier & { id?: string }; /** * Splits identifiers into verified and unverified, primary first. * - * The primary has to lead: `migrate run` puts the first entry on + * The primary has to lead: `migrate import` puts the first entry on * `POST /v1/users` and attaches the rest afterwards, so a reordered list would * silently change which address the user signs in with. */ @@ -224,14 +223,9 @@ export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExpo } export async function exportClerk(options: ExportClerkOptions): Promise { - if (options.clerkSecretKey) { - log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); - } - const secretKeyOption = options.secretKey ?? options.clerkSecretKey; - await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { - const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); - const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const dateTime = getDateTimeStamp(); log.info(`Exporting from ${target ?? "the resolved instance"}.`); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index aedb8fb9c..8106840f1 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -371,7 +371,7 @@ export function buildFirebaseExport(users: FirebaseUser[], dateTime: string) { }; } -/** The exact `migrate run` invocation, with the project's own parameters. */ +/** The exact `migrate import` invocation, with the project's own parameters. */ export function formatHashConfigGuidance( config: HashConfig | null, outputPath: string, @@ -397,7 +397,7 @@ export function formatHashConfigGuidance( bold("Password hash parameters"), "Read from the project. Import with:", dim( - ` clerk migrate -y --transformer firebase --file ${outputPath} \\\n` + + ` clerk migrate import -y --transformer firebase --file ${outputPath} \\\n` + ` --firebase-signer-key "${config.signerKey}" \\\n` + ` --firebase-salt-separator "${config.saltSeparator}" \\\n` + ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 773e07f22..a0067a2a7 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -84,7 +84,7 @@ const DB_PLATFORMS = [ export function registerMigrateExport(migrateCommand: Command<[], Record>): void { const exportCommand = migrateCommand .command("export") - .description("Export users from a source platform, ready for `clerk migrate`") + .description("Export users from a source platform, ready for `clerk migrate import`") .setExamples([ { command: "clerk migrate export", description: "Pick a platform interactively" }, { @@ -104,7 +104,6 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Where to write the export, relative to the current directory") .option("--secret-key ", "Backend API secret key to use") - .option("--clerk-secret-key ", "Deprecated alias for --secret-key") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") .setExamples([ diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index 07454ad98..cb24f0384 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -18,24 +18,15 @@ describe("registerMigrate", () => { expect(migrate?.description()).toContain("Migrate users"); }); - test("registers the run subcommand", () => { - expect(findCommand(["migrate", "run"])).toBeDefined(); + test("registers the import subcommand", () => { + expect(findCommand(["migrate", "import"])).toBeDefined(); }); - // Bare `clerk migrate` dispatches to `migrate run`, mirroring how bare - // `clerk deploy` dispatches to `deploy run`. - test("makes run the default subcommand, so bare `clerk migrate` starts the wizard", () => { - const run = findCommand(["migrate", "run"]); - expect(run as unknown as { _defaultCommandName?: unknown }).toBeDefined(); + // The direction is never implied: `import` and `export` are siblings, so a + // default would make one of them the meaning of the bare group name. + test("leaves migrate with no default subcommand", () => { const migrate = findCommand(["migrate"]) as unknown as { _defaultCommandName?: string }; - expect(migrate._defaultCommandName).toBe("run"); - }); - - test("keeps run visible in help, unlike deploy's hidden default", () => { - expect(findCommand(["migrate", "run"])?.parent?.commands.map((c) => c.name())).toContain("run"); - expect( - (findCommand(["migrate", "run"]) as unknown as { _hidden?: boolean })._hidden, - ).toBeFalsy(); + expect(migrate._defaultCommandName).toBeFalsy(); }); test.each([ @@ -50,11 +41,10 @@ describe("registerMigrate", () => { "--firebase-mem-cost", "--yes", "--secret-key", - "--clerk-secret-key", "--app", "--instance", - ])("migrate run accepts %s", (flag) => { - const flags = findCommand(["migrate", "run"])?.options.map((option) => option.long); + ])("migrate import accepts %s", (flag) => { + const flags = findCommand(["migrate", "import"])?.options.map((option) => option.long); expect(flags).toContain(flag); }); @@ -120,10 +110,10 @@ describe("registerMigrate", () => { test("documents the default output location in help", () => { expect(findCommand(["migrate", "export", "clerk"])?.description()).toContain( - "./exports/clerk-export.json", + "./exports/clerk-export-.json", ); expect(findCommand(["migrate", "export", "auth0"])?.description()).toContain( - "./exports/auth0-export.json", + "./exports/auth0-export-.json", ); }); @@ -140,8 +130,8 @@ describe("registerMigrate", () => { ); }); - test("migrate run accepts --transformer-file", () => { - expect(findCommand(["migrate", "run"])?.options.map((o) => o.long)).toContain( + test("migrate import accepts --transformer-file", () => { + expect(findCommand(["migrate", "import"])?.options.map((o) => o.long)).toContain( "--transformer-file", ); }); @@ -153,7 +143,7 @@ describe("registerMigrate", () => { expect(findCommand(["migrate", "delete"])?.description()).toContain("last migration"); }); - test.each(["--yes", "--secret-key", "--clerk-secret-key", "--app", "--instance"])( + test.each(["--yes", "--secret-key", "--app", "--instance"])( "migrate delete accepts %s", (flag) => { expect(findCommand(["migrate", "delete"])?.options.map((o) => o.long)).toContain(flag); @@ -191,7 +181,9 @@ describe("registerMigrate", () => { }); test("constrains --transformer to the registered transformers, for validation and completion", () => { - const option = findCommand(["migrate", "run"])?.options.find((o) => o.long === "--transformer"); + const option = findCommand(["migrate", "import"])?.options.find( + (o) => o.long === "--transformer", + ); // Tracks the registry so adding a platform needs no edit here. expect(option?.argChoices).toEqual(transformerKeys()); }); @@ -202,7 +194,7 @@ describe("registerMigrate", () => { ["-r", "--resume-after"], ["-y", "--yes"], ])("exposes %s as the short form of %s", (short, long) => { - const option = findCommand(["migrate", "run"])?.options.find((o) => o.long === long); + const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === long); expect(option?.short).toBe(short); }); }); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index b17195af4..e006cb6dd 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -14,15 +14,15 @@ const migrate = { run, delete: deleteMigration, transformersList }; export function registerMigrate(program: Program): void { const migrateCommand = program .command("migrate") - .description("Migrate users into Clerk from another auth provider") + .description("Migrate users into Clerk from another auth provider or another Clerk instance") .setExamples([ - { command: "clerk migrate", description: "Walk through a migration interactively" }, + { command: "clerk migrate import", description: "Walk through an import interactively" }, { - command: "clerk migrate -y --transformer clerk --file users.json", + command: "clerk migrate import -y --transformer clerk --file users.json", description: "Import users from a Clerk export", }, { - command: "clerk migrate -y -t supabase -f users.json --skip-unsupported-providers", + command: "clerk migrate import -y -t supabase -f users.json --skip-unsupported-providers", description: "Skip Supabase users whose provider is not enabled", }, { @@ -39,16 +39,16 @@ export function registerMigrate(program: Program): void { { command: "clerk migrate delete", description: "Undo the last migration" }, ]); - // `isDefault` so `clerk migrate` is the whole command: bare, it runs the - // wizard; with flags, they fall through to here. `run` stays addressable - // because scripts and older docs use it, but `clerk migrate` is the spelling - // every example gives. + // Named, not `isDefault`. `import` and `export` are the two directions this + // group moves users in, and neither is implied by the bare group name — a + // default would make `clerk migrate --file users.json` mean "import" while + // its sibling has to be spelled out. Bare `clerk migrate` prints help. // // The flags stay here rather than on `migrate`, matching how `config` keeps // its own on `pull`/`patch`/`put` — a group's help is a list of subcommands // and examples, not a merge of everything underneath it. migrateCommand - .command("run", { isDefault: true }) + .command("import") .description("Import users from an exported JSON or CSV file") .addOption( createOption( @@ -77,24 +77,23 @@ export function registerMigrate(program: Program): void { ) .option("-y, --yes", "Skip the confirmation prompt") .option("--secret-key ", "Backend API secret key to use") - .option("--clerk-secret-key ", "Deprecated alias for --secret-key") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") .setExamples([ { - command: "clerk migrate -y --transformer clerk --file users.json", + command: "clerk migrate import -y --transformer clerk --file users.json", description: "Import a Clerk Dashboard export", }, { - command: "clerk migrate -y -t clerk -f users.csv --require-password", + command: "clerk migrate import -y -t clerk -f users.csv --require-password", description: "Import only the users that carry a password digest", }, { - command: "clerk migrate -y -t clerk -f users.json -r user_2x9k", + command: "clerk migrate import -y -t clerk -f users.json -r user_2x9k", description: "Resume a partial migration after the last imported user", }, { - command: "clerk migrate -y -t supabase -f users.json --skip-unsupported-providers", + command: "clerk migrate import -y -t supabase -f users.json --skip-unsupported-providers", description: "Skip Supabase users whose only provider is not enabled in Clerk", }, ]) @@ -109,7 +108,6 @@ export function registerMigrate(program: Program): void { .description("Delete the users created by the last migration for this project") .option("-y, --yes", "Skip the confirmation prompt") .option("--secret-key ", "Backend API secret key to use") - .option("--clerk-secret-key ", "Deprecated alias for --secret-key") .option("--app ", "Application ID to target (works from any directory)") .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") .setExamples([ diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts index 7b6cd7dfa..58bb6dceb 100644 --- a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -4,7 +4,7 @@ * * Ported from the standalone migration-tool's `src/lib/clerk.ts`, rewritten * onto the CLI's own primitives: the FAPI host comes from BAPI `/v1/domains` - * — a secret key is all `migrate run` is given — and the settings come from + * — a secret key is all `migrate import` is given — and the settings come from * `lib/fapi.ts` rather than a bespoke fetch. */ diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts index ccf5fa75e..45c65c452 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts @@ -39,7 +39,7 @@ afterEach(() => { }); describe("gating on the transformer", () => { - // `migrate run` is one command for every platform, so a signer key left in + // `migrate import` is one command for every platform, so a signer key left in // .env.clerk-migrate after a Firebase migration is in scope for whatever runs // next unless the transformer says otherwise. test.each([["clerk"], ["supabase"], ["auth0"], ["authjs"], ["betterauth"]])( diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts index 836d374ea..10b06653d 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts @@ -2,7 +2,7 @@ * Firebase's four scrypt parameters: where they come from, and when they are * looked for at all. * - * **Only read when the transformer is `firebase`.** `migrate run` is one + * **Only read when the transformer is `firebase`.** `migrate import` is one * command serving every platform, so a `CLERK_FIREBASE_SIGNER_KEY` left in * `.env.clerk-migrate` after a Firebase migration is in scope for the Supabase * run that follows it unless something says otherwise. Nothing downstream would diff --git a/packages/cli-core/src/commands/migrate/lib/retry.ts b/packages/cli-core/src/commands/migrate/lib/retry.ts index f9d1fc158..a01de49a2 100644 --- a/packages/cli-core/src/commands/migrate/lib/retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/retry.ts @@ -1,5 +1,5 @@ /** - * Rate-limit backoff, shared by `migrate run` and `migrate delete`. + * Rate-limit backoff, shared by `migrate import` and `migrate delete`. * * Both walk the whole user set through BAPI and hit the same limits, so they * back off identically rather than approximately: extracting this is what diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index f41aec567..be0bc5028 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -63,10 +63,10 @@ function resolve(tokens: string[]): { command: Command; rest: string[] } { /** * The flags a command accepts, including those of a default subcommand. * - * `clerk migrate` carries no options of its own — `run` is registered - * `isDefault`, so Commander hands it everything after the group name. The - * documented spelling is `clerk migrate --transformer …`, and this has to see - * the same flags Commander does or every such example reads as unsupported. + * `migrate logs` and `migrate transformers` register their `list` `isDefault`, + * so Commander hands it everything after the group name. The documented + * spelling is `clerk migrate logs --json`, and this has to see the same flags + * Commander does or every such example reads as unsupported. */ function flagsOf(command: Command): string[] { const own = command.options.flatMap( diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 44198a6f7..9d29d32f1 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -1,5 +1,5 @@ /** - * The human-mode half of `migrate run`: the wizard fills in missing flags, the + * The human-mode half of `migrate import`: the wizard fills in missing flags, the * readiness report renders, and declining the confirmation writes nothing. * * Kept in its own file because `mock.module` registrations are process-lifetime, @@ -164,7 +164,7 @@ function stubInstanceSettings(settings: StubSettings) { const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); describe("the wizard fills in missing flags", () => { - test("bare `clerk migrate` prompts for the transformer and file, then imports", async () => { + test("bare `clerk migrate import` prompts for the transformer and file, then imports", async () => { await run({ secretKey: "sk_test_x" }); expect(mockSelect).toHaveBeenCalledTimes(1); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 9d9272f95..7fa811d46 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -183,13 +183,6 @@ describe("run", () => { expect(captured.err).toContain("1 user failed validation"); }); - test("warns that --clerk-secret-key is deprecated but still honours it", async () => { - await run({ ...baseOptions, secretKey: undefined, clerkSecretKey: "sk_test_x" }); - - expect(captured.err).toContain("--clerk-secret-key is deprecated"); - expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); - }); - test("refuses to exceed the development-instance user limit", async () => { fs.writeFileSync( path.join(workDir, "export.json"), diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index bdcd865e9..e37cde385 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -1,15 +1,15 @@ /** - * `clerk migrate` — the user import itself. + * `clerk migrate import` — the user import itself. * * Ported from the standalone migration-tool's `src/migrate/cli.ts` * (`runNonInteractive`), with auth moved onto the CLI's standard secret-key * resolution chain and every failure raised as a `CliError` instead of * `console.error` + `process.exit`. * - * Registered as the `run` subcommand and marked default, so `clerk migrate` and - * `clerk migrate run` both land here. Whatever the flags did not supply is - * filled in by `wizard.ts` for a human, or raised as a usage error naming the - * missing flags for an agent, which cannot answer a prompt. + * Registered as the `import` subcommand. The exported handler keeps the name + * `run` because `import` is a reserved word. Whatever the flags did not supply + * is filled in by `wizard.ts` for a human, or raised as a usage error naming + * the missing flags for an agent, which cannot answer a prompt. */ import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; @@ -64,8 +64,6 @@ export type MigrateRunOptions = { requirePassword?: boolean; yes?: boolean; secretKey?: string; - /** Deprecated alias for `--secret-key`, kept for existing prompts and docs. */ - clerkSecretKey?: string; app?: string; instance?: string; /** Path to a user-authored transformer, for a platform with no built-in. */ @@ -95,7 +93,8 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate -y --transformer-file ./my-transformer.ts --file users.json", + command: + "clerk migrate import -y --transformer-file ./my-transformer.ts --file users.json", description: "Import with a custom transformer", }, ], @@ -117,7 +116,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate -y --transformer clerk --file users.json", + command: "clerk migrate import -y --transformer clerk --file users.json", description: "Import a Clerk export", }, ], @@ -135,7 +134,7 @@ export function validateRunOptions(options: MigrateRunOptions): { ERROR_CODE.USAGE_ERROR, [ { - command: "clerk migrate -y --transformer clerk --file users.json", + command: "clerk migrate import -y --transformer clerk --file users.json", description: "Import a Clerk export", }, ], @@ -404,7 +403,7 @@ async function showReadinessReport( * to pass. * * Agent mode is the CLI's existing non-interactive signal, so an agent that - * runs bare `clerk migrate` gets a usage error naming the flags rather than a + * runs bare `clerk migrate import` gets a usage error naming the flags rather than a * prompt it cannot answer. */ async function resolveMissingOptions(options: MigrateRunOptions): Promise { @@ -454,11 +453,12 @@ async function applyCustomTransformer(options: MigrateRunOptions): Promise { - if (rawOptions.clerkSecretKey) { - log.warn("--clerk-secret-key is deprecated; use --secret-key instead."); - } - rawOptions = await applyCustomTransformer(rawOptions); const options = await resolveMissingOptions(rawOptions); - const secretKeyOption = options.secretKey ?? options.clerkSecretKey; const { transformer, file } = validateRunOptions(options); const firebaseHashConfig = await resolveFirebaseHashConfig(options, transformer); await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { - const target = await describeBapiTarget({ ...options, secretKey: secretKeyOption }); - const secretKey = await resolveBapiSecretKey({ ...options, secretKey: secretKeyOption }); + const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const limits = resolveLimits(secretKey); const dateTime = getDateTimeStamp(); const logFile = getLogFilePath("migration", dateTime); diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index e034ec60d..8ec0fe982 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -20,7 +20,7 @@ export interface SettingDef { /** * What the user types: `clerk migrate settings set `. * - * Kebab-case, and identical to the `migrate run` flag it backs. A setting and + * Kebab-case, and identical to the `migrate import` flag it backs. A setting and * its flag are the same knob reached two ways, so `firebase-signer-key` here * and `--firebase-signer-key` there must not drift into two spellings the * user has to learn separately. Sentence-case prose belongs in diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index 91d982076..42e4f7c1e 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -139,7 +139,7 @@ describe("list", () => { ); }); - // The names are kebab-case because they mirror the `migrate run` flags; the + // The names are kebab-case because they mirror the `migrate import` flags; the // description column is what makes the list readable. test("explains each setting in prose", async () => { await list(); diff --git a/packages/cli-core/src/commands/migrate/transformers/registry.ts b/packages/cli-core/src/commands/migrate/transformers/registry.ts index 6279577a0..df9448595 100644 --- a/packages/cli-core/src/commands/migrate/transformers/registry.ts +++ b/packages/cli-core/src/commands/migrate/transformers/registry.ts @@ -1,7 +1,7 @@ /** * Transformer registry. * - * `migrate run` reads this array to resolve `--transformer` and to list the + * `migrate import` reads this array to resolve `--transformer` and to list the * valid choices in help output and tab-completion. * * To add a platform: create `transformers/.ts` exporting a diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts index ff6e7173e..122173046 100644 --- a/packages/cli-core/src/commands/migrate/wizard.ts +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -1,5 +1,5 @@ /** - * The interactive path behind a bare `clerk migrate`. + * The interactive path behind a bare `clerk migrate import`. * * Ported from the standalone migration-tool's `src/migrate/cli.ts` interactive * flow. The platform and file are pre-filled from the previous run, so a repeat @@ -154,12 +154,12 @@ export function throwAgentFlagsRequired(missing: { transformer: boolean; file: b ].filter(Boolean); throwUsageError( - `\`clerk migrate\` is interactive and cannot prompt in agent mode. Pass ${flags.join(" and ")}.`, + `\`clerk migrate import\` is interactive and cannot prompt in agent mode. Pass ${flags.join(" and ")}.`, undefined, undefined, [ { - command: `clerk migrate -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, + command: `clerk migrate import -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, description: "Run non-interactively", }, ], diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index c93b359e1..08e0295d1 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -50,7 +50,7 @@ interface RelayEntry { token: string; } -/** What `clerk migrate` last imported for a project, and how. */ +/** What `clerk migrate import` last imported for a project, and how. */ interface MigrationEntry { transformer?: string; file?: string; diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index 41d415f90..a1b53e1d7 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -79,7 +79,7 @@ export const NEXT_STEPS = { // The only parameterized entry: a suggested import is worthless unless it // names the transformer that reads this export and the file just written. MIGRATE_EXPORT: (transformerKey: string, file: string) => [ - `Run \`clerk migrate --transformer ${transformerKey} --file ${file}\` to import them`, + `Run \`clerk migrate import --transformer ${transformerKey} --file ${file}\` to import them`, ], } as const; From 35c2b0974489acf8709765137f1a1ed527877a89 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:31:56 -0400 Subject: [PATCH 18/34] feat(migrate): timestamp export filenames and ask where to save MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two exports of the same platform used to write the same path, so the second silently overwrote the first. Filenames now carry a local `YYYYMMDD-HHmm` stamp, and every export settles its destination *before* it starts — one prompt, prefilled with the proposed path, so Enter accepts it. Asked up front on purpose: coming back to a long export stalled on a prompt, with every user held in memory and nothing on disk, is the worse half of that trade. `--output` is an answer already given, and agent mode takes the proposal without asking. --- .../cli-core/src/commands/migrate/README.md | 22 +++-- .../src/commands/migrate/export/auth0.test.ts | 25 ++++-- .../src/commands/migrate/export/auth0.ts | 6 +- .../src/commands/migrate/export/authjs.ts | 6 +- .../src/commands/migrate/export/betterauth.ts | 6 +- .../src/commands/migrate/export/clerk.test.ts | 35 +++++--- .../src/commands/migrate/export/clerk.ts | 6 +- .../commands/migrate/export/firebase.test.ts | 25 ++++-- .../src/commands/migrate/export/firebase.ts | 6 +- .../src/commands/migrate/export/index.ts | 18 ++-- .../commands/migrate/export/shared.test.ts | 82 +++++++++++++++++++ .../src/commands/migrate/export/shared.ts | 52 +++++++++++- .../src/commands/migrate/export/supabase.ts | 6 +- 13 files changed, 250 insertions(+), 45 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/export/shared.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 53dcc2b44..c448e8359 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -148,9 +148,21 @@ with what a database export needs. | `betterauth` | Better Auth database | `--transformer betterauth` | | `firebase` | Firebase Identity Toolkit | `--transformer firebase` | -Exports land at `./exports/-export.json` unless `--output` says -otherwise. `--output` resolves against the **current directory**, like every -other path flag here. +Every export asks where to save the file before it starts, proposing +`./exports/-export-.json`. Press enter to take it, +or type over it to save somewhere else — the proposal is prefilled, so it is +one prompt rather than a confirm and a path question. + +The stamp is ISO 8601 basic format in local time, to the minute: it goes in a +name people read off the screen and tab-complete, and it means a second export +never silently overwrites the first. + +`--output` answers that prompt up front and skips it, as does agent mode, which +takes the proposed path. `--output` resolves against the **current directory**, +like every other path flag here. + +The question comes before any users are fetched, so a long export can be left +unattended rather than stalling on a prompt with everything held in memory. | Flag | Platforms | Description | | -------------------------- | ---------------------------------- | -------------------------------------------- | @@ -175,9 +187,9 @@ Field coverage ! 1/3 have a username ! 2/3 have a password (not exportable — see below) -Exported 3 users to /project/exports/clerk-export.json +Exported 3 users to /project/exports/clerk-export-20260817-1432.json └ Next steps - → Run `clerk migrate --transformer clerk --file exports/clerk-export.json` to import them + → Run `clerk migrate import --transformer clerk --file exports/clerk-export-20260817-1432.json` to import them ``` Every export also writes `logs/export-.log`, so `migrate logs list` diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts index 9746f79cb..aae8e7906 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.test.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -42,6 +42,9 @@ afterAll(() => { }); beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); requests = []; fs.rmSync(getLogDir(), { recursive: true, force: true }); fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); @@ -276,15 +279,25 @@ describe("buildAuth0Export", () => { }); }); +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + describe("exportAuth0", () => { test("writes the default path and reports coverage", async () => { stubAuth0([[auth0User(0)], []]); await exportAuth0({ ...CREDENTIALS }); - const written = JSON.parse( - fs.readFileSync(path.join(workDir, "exports", "auth0-export.json"), "utf-8"), - ) as Record[]; + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^auth0-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; expect(written[0]?.user_id).toBe("auth0|a0"); expect(captured.err).toContain("Field coverage"); }); @@ -296,11 +309,13 @@ describe("exportAuth0", () => { const originalMode = getMode(); setMode("human"); try { - await exportAuth0({ ...CREDENTIALS }); + // --output answers the destination prompt, which human mode would + // otherwise stop on. + await exportAuth0({ ...CREDENTIALS, output: "exports/mine.json" }); } finally { setMode(originalMode); } - expect(captured.err).toContain("migrate --transformer auth0 --file exports/auth0-export.json"); + expect(captured.err).toContain("migrate import --transformer auth0 --file exports/mine.json"); }); test("--output controls the destination", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index bf1657267..478b944d7 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -23,7 +23,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spin import { isAgent, isHuman } from "../../../mode.ts"; import { findMigrateEnvValue } from "../lib/env-file.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; const PAGE_SIZE = 100; @@ -316,6 +316,8 @@ export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0Exp export async function exportAuth0(options: ExportAuth0Options): Promise { const credentials = await resolveAuth0Credentials(options); + const destination = await resolveOutputPath("auth0", options.output); + await withGutter("Exporting users from Auth0", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); log.info(`Exporting from ${credentials.domain}.`); @@ -329,7 +331,7 @@ export async function exportAuth0(options: ExportAuth0Options): Promise { ); const { users: exported, coverage } = buildAuth0Export(users, dateTime); - const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("auth0")); + const outputPath = writeExportOutput(exported, destination); setNextSteps( reportExport({ diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index a78df15de..1f74223a4 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -15,7 +15,7 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { log } from "../../../lib/log.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; /** Table names to try, in order. Prisma capitalizes; Drizzle does not. */ @@ -113,6 +113,8 @@ export async function exportAuthJs(options: DbExportOptions): Promise { hint: "Postgres, MySQL or a SQLite file — whichever your Auth.js adapter uses.", }); + const destination = await resolveOutputPath("authjs", options.output); + await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); @@ -122,7 +124,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); const { users, coverage } = buildAuthJsExport(rows, dateTime); - const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("authjs")); + const outputPath = writeExportOutput(users, destination); setNextSteps( reportExport({ diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 85ab190db..006af76f9 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -19,7 +19,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; /** Columns a Better Auth plugin adds to the user table. */ @@ -160,6 +160,8 @@ export async function exportBetterAuth(options: DbExportOptions): Promise hint: "Postgres, MySQL or a SQLite file — whichever your Better Auth install uses.", }); + const destination = await resolveOutputPath("betterauth", options.output); + await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); @@ -178,7 +180,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise ); const { users, coverage } = buildBetterAuthExport(rows, dateTime); - const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("betterauth")); + const outputPath = writeExportOutput(users, destination); setNextSteps( reportExport({ diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index b628a8408..75e1f0565 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -33,6 +33,9 @@ afterAll(() => { }); beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); requests = []; fs.rmSync(getLogDir(), { recursive: true, force: true }); fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); @@ -74,7 +77,7 @@ describe("mapClerkUserToExport", () => { }); }); - // `migrate run` puts the first entry on POST /v1/users and attaches the rest + // `migrate import` puts the first entry on POST /v1/users and attaches the rest // afterwards, so a reordered list would change which address signs the user in. test("keeps the primary identifier out of the additional list", () => { const mapped = mapClerkUserToExport( @@ -229,15 +232,25 @@ describe("buildClerkExport", () => { }); }); +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + describe("exportClerk", () => { test("writes the default path and reports coverage", async () => { stubPages([[user({ id: "u1", first_name: "Ada" })], []]); await exportClerk({ secretKey: "sk_test_x" }); - const written = JSON.parse( - fs.readFileSync(path.join(workDir, "exports", "clerk-export.json"), "utf-8"), - ) as Record[]; + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^clerk-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; expect(written).toHaveLength(1); expect(written[0]?.id).toBe("u1"); expect(captured.err).toContain("Field coverage"); @@ -251,11 +264,13 @@ describe("exportClerk", () => { const originalMode = getMode(); setMode("human"); try { - await exportClerk({ secretKey: "sk_test_x" }); + // --output answers the destination prompt, which human mode would + // otherwise stop on. + await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); } finally { setMode(originalMode); } - expect(captured.err).toContain("migrate --transformer clerk --file exports/clerk-export.json"); + expect(captured.err).toContain("migrate import --transformer clerk --file exports/mine.json"); }); test("--output controls the destination, relative to the working directory", async () => { @@ -264,7 +279,7 @@ describe("exportClerk", () => { await exportClerk({ secretKey: "sk_test_x", output: "somewhere/mine.json" }); expect(fs.existsSync(path.join(workDir, "somewhere", "mine.json"))).toBe(true); - expect(fs.existsSync(path.join(workDir, "exports", "clerk-export.json"))).toBe(false); + expect(fs.existsSync(path.join(workDir, "exports"))).toBe(false); }); // Silence here would be the worst outcome: the operator finds out when @@ -281,9 +296,7 @@ describe("exportClerk", () => { await exportClerk({ secretKey: "sk_test_x" }); expect(captured.err).toContain("No users found to export"); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, "exports", "clerk-export.json"), "utf-8")), - ).toEqual([]); + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toEqual([]); }); test("an empty export warns but does not suggest importing it", async () => { @@ -291,7 +304,7 @@ describe("exportClerk", () => { const originalMode = getMode(); setMode("human"); try { - await exportClerk({ secretKey: "sk_test_x" }); + await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); } finally { setMode(originalMode); } diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index a9ff2f062..fc3875339 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -20,7 +20,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { retryOn429 } from "../lib/retry.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; /** BAPI's maximum page size for `GET /v1/users`. */ const PAGE_SIZE = 500; @@ -223,6 +223,8 @@ export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExpo } export async function exportClerk(options: ExportClerkOptions): Promise { + const destination = await resolveOutputPath("clerk", options.output); + await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); @@ -235,7 +237,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { ); const { users: exported, coverage } = buildClerkExport(users, dateTime); - const outputPath = writeExportOutput(exported, options.output ?? defaultOutputPath("clerk")); + const outputPath = writeExportOutput(exported, destination); setNextSteps( reportExport({ diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 1de54a3c8..5754bf901 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -65,6 +65,9 @@ afterAll(() => { }); beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); requests = []; delete process.env.FIREBASE_AUTH_EMULATOR_HOST; fs.rmSync(getLogDir(), { recursive: true, force: true }); @@ -397,6 +400,13 @@ describe("formatHashConfigGuidance", () => { }); }); +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + describe("exportFirebase", () => { test("exports end to end and reports coverage", async () => { stubFirebase([[fbUser(0), fbUser(1)]], { @@ -405,9 +415,12 @@ describe("exportFirebase", () => { await exportFirebase({ serviceAccount: "./sa.json" }); - const written = JSON.parse( - fs.readFileSync(path.join(workDir, "exports", "firebase-export.json"), "utf-8"), - ) as Record[]; + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^firebase-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; expect(written).toHaveLength(2); expect(captured.err).toContain("Field coverage"); expect(captured.err).toContain("demo-fb project"); @@ -420,12 +433,14 @@ describe("exportFirebase", () => { const originalMode = getMode(); setMode("human"); try { - await exportFirebase({ serviceAccount: "./sa.json" }); + // --output answers the destination prompt, which human mode would + // otherwise stop on. + await exportFirebase({ serviceAccount: "./sa.json", output: "exports/mine.json" }); } finally { setMode(originalMode); } expect(captured.err).toContain( - "migrate --transformer firebase --file exports/firebase-export.json", + "migrate import --transformer firebase --file exports/mine.json", ); }); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 8106840f1..acb413e6d 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -30,7 +30,7 @@ import { loggedFetch } from "../../../lib/fetch.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; /** Identity Toolkit's maximum for `accounts:batchGet`. */ const PAGE_SIZE = 1000; @@ -424,6 +424,8 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { const dateTime = getDateTimeStamp(); log.info(`Exporting from the ${account.project_id} project.`); @@ -437,7 +439,7 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise.json)", + ) .option("-o, --output ", "Where to write the export, relative to the current directory") .option("--secret-key ", "Backend API secret key to use") .option("--app ", "Application ID to target (works from any directory)") @@ -109,7 +111,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json", }, { command: "clerk migrate export clerk --instance prod --output prod-users.json", @@ -122,7 +124,9 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)", + ) .option("--domain ", "Auth0 tenant domain, e.g. my-tenant.us.auth0.com") .option("--client-id ", "Machine-to-machine application client ID") .option("--client-secret ", "Machine-to-machine application client secret") @@ -144,7 +148,9 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)", + ) .option("--service-account ", "Path to a service account key JSON file") .option("-o, --output ", "Where to write the export, relative to the current directory") .setExamples([ @@ -162,7 +168,9 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)`, + ) .option("--db-url ", "Postgres, MySQL or SQLite connection string") .option("-o, --output ", "Where to write the export, relative to the current directory") .setExamples([ diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts new file mode 100644 index 000000000..23b387db1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -0,0 +1,82 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +const mockText = mock(); +mock.module("../../../lib/prompts.ts", () => ({ + text: (...args: unknown[]) => mockText(...args), +})); + +let human = true; +mock.module("../../../mode.ts", () => ({ + isHuman: () => human, + isAgent: () => !human, + getMode: () => (human ? "human" : "agent"), + setMode: () => {}, +})); + +const { defaultOutputPath, outputStamp, resolveOutputPath } = await import("./shared.ts"); + +beforeEach(() => { + human = true; + mockText.mockReset(); +}); + +describe("outputStamp", () => { + // Local time, and no seconds: this ends up in a filename someone reads off + // the screen and types back. + test("stamps to the minute", () => { + expect(outputStamp(new Date(2026, 7, 17, 14, 32, 59))).toBe("20260817-1432"); + }); + + test("pads single-digit months, days, hours and minutes", () => { + expect(outputStamp(new Date(2026, 0, 3, 9, 5, 0))).toBe("20260103-0905"); + }); +}); + +describe("defaultOutputPath", () => { + test("names the platform and the stamp, under exports/", () => { + expect(defaultOutputPath("clerk", new Date(2026, 7, 17, 14, 32))).toBe( + "exports/clerk-export-20260817-1432.json", + ); + }); + + // Two exports of the same platform an hour apart must not collide. + test("gives two runs different names", () => { + expect(defaultOutputPath("auth0", new Date(2026, 7, 17, 14, 32))).not.toBe( + defaultOutputPath("auth0", new Date(2026, 7, 17, 15, 32)), + ); + }); +}); + +describe("resolveOutputPath", () => { + test("--output is an answer already given", async () => { + expect(await resolveOutputPath("clerk", "somewhere/mine.json")).toBe("somewhere/mine.json"); + expect(mockText).not.toHaveBeenCalled(); + }); + + // One prompt, not a confirm plus a path question: the proposal is prefilled, + // so enter accepts it and typing replaces it. + test("prefills the proposed path so enter accepts it", async () => { + mockText.mockImplementation(async (config: { default: string }) => config.default); + + const chosen = await resolveOutputPath("clerk"); + + expect(chosen).toMatch(/^exports\/clerk-export-\d{8}-\d{4}\.json$/); + expect(mockText).toHaveBeenCalledTimes(1); + expect(mockText.mock.calls[0]?.[0]).toMatchObject({ message: "Save the export to:" }); + }); + + test("takes a path typed over the proposal, trimmed", async () => { + mockText.mockResolvedValue(" ../elsewhere/users.json "); + + expect(await resolveOutputPath("firebase")).toBe("../elsewhere/users.json"); + }); + + test("agent mode takes the proposed path without asking", async () => { + human = false; + + expect(await resolveOutputPath("supabase")).toMatch( + /^exports\/supabase-export-\d{8}-\d{4}\.json$/, + ); + expect(mockText).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index e880d90d3..7f968d845 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -14,10 +14,58 @@ import path from "node:path"; import { dim, green, yellow } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { NEXT_STEPS } from "../../../lib/next-steps.ts"; +import { text } from "../../../lib/prompts.ts"; +import { isHuman } from "../../../mode.ts"; + +/** + * `YYYYMMDD-HHmm`, local time — ISO 8601 basic format, minus seconds. + * + * Basic throughout rather than `2026-08-17-1954`, which mixes the extended + * date form with the basic time form and leaves the trailing group looking + * like a fourth date component. One separator, and it sorts lexically. + * + * Seconds are dropped on purpose. This lands in a filename people read off the + * screen, type back and tab-complete, and two exports of the same platform + * inside one minute is not an accident anyone has by surprise. + * + * Local rather than UTC because the only reader is the person who just ran the + * command, deciding which of two files is the one they meant. + */ +export function outputStamp(now: Date = new Date()): string { + const pad = (value: number) => String(value).padStart(2, "0"); + const date = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}`; + return `${date}-${pad(now.getHours())}${pad(now.getMinutes())}`; +} /** Where an export lands when `--output` is not given. */ -export function defaultOutputPath(platform: string): string { - return path.join("exports", `${platform}-export.json`); +export function defaultOutputPath(platform: string, now?: Date): string { + return path.join("exports", `${platform}-export-${outputStamp(now)}.json`); +} + +/** + * Settles where the file lands, before the export runs. + * + * Asked up front rather than at write time so a long export can be left + * unattended — coming back to a stalled prompt with every user held in memory + * and nothing on disk is the worse half of that trade. + * + * One prompt, not a confirm followed by a path prompt: the proposed path is + * prefilled, so Enter accepts it and typing replaces it. + * + * `--output` is an answer already given, and agent mode has nobody to ask. + */ +export async function resolveOutputPath(platform: string, output?: string): Promise { + if (output) return output; + + const proposed = defaultOutputPath(platform); + if (!isHuman()) return proposed; + + const chosen = await text({ + message: "Save the export to:", + default: proposed, + validate: (value) => (value?.trim() ? undefined : "A path is required"), + }); + return chosen.trim(); } /** diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 80ff3785e..2436c95c5 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -14,7 +14,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; -import { defaultOutputPath, reportExport, writeExportOutput } from "./shared.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; /** @@ -111,6 +111,8 @@ export async function exportSupabase(options: DbExportOptions): Promise { hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", }); + const destination = await resolveOutputPath("supabase", options.output); + await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { const dateTime = getDateTimeStamp(); @@ -119,7 +121,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { ); const { users, coverage } = buildSupabaseExport(rows, dateTime); - const outputPath = writeExportOutput(users, options.output ?? defaultOutputPath("supabase")); + const outputPath = writeExportOutput(users, destination); setNextSteps( reportExport({ From 5a1618981fcee9dc102ae324f6ee8631da83fbe7 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:32:07 -0400 Subject: [PATCH 19/34] feat(migrate): URL-encode credentials pasted into a connection string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dashboards hand out `postgres://user:[YOUR-PASSWORD]@host/db` and people paste their real password in verbatim. A `#`, `@` or `/` in it makes the whole string unparseable, here and later inside `Bun.SQL` — and the prompt is masked, so the paste that failed is not even visible to check. `normalizeConnectionString` percent-encodes the userinfo when the raw string will not parse, splitting on the LAST `@` so an unencoded one inside the password does not end the userinfo early. Strings that already parse are returned untouched, so a correctly encoded password is never double-encoded. --- .../migrate/export/db-exports.test.ts | 43 +++++++++++- .../src/commands/migrate/export/db-options.ts | 68 +++++++++++++++---- 2 files changed, 95 insertions(+), 16 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 08816b69a..62097f2b2 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -25,7 +25,11 @@ import { PLUGIN_COLUMNS, } from "./betterauth.ts"; import { buildSupabaseExport } from "./supabase.ts"; -import { looksLikeConnectionString, resolveDbUrl } from "./db-options.ts"; +import { + looksLikeConnectionString, + normalizeConnectionString, + resolveDbUrl, +} from "./db-options.ts"; /** A cwd with no `.env` files, so these tests exercise only the injected env. */ const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); @@ -104,6 +108,33 @@ describe("looksLikeConnectionString", () => { }); }); +describe("normalizeConnectionString", () => { + test("encodes a password pasted in raw", () => { + const raw = "postgres://postgres:aB#c%92^d@db.example.supabase.co:5432/postgres"; + const normalized = normalizeConnectionString(raw); + + expect(looksLikeConnectionString(normalized)).toBe(true); + expect(decodeURIComponent(new URL(normalized).password)).toBe("aB#c%92^d"); + expect(new URL(normalized).hostname).toBe("db.example.supabase.co"); + }); + + test("encodes an unencoded @ in the password", () => { + const normalized = normalizeConnectionString("postgres://u:p@ss@host:5432/db"); + + expect(decodeURIComponent(new URL(normalized).password)).toBe("p@ss"); + expect(new URL(normalized).hostname).toBe("host"); + }); + + test("leaves an already-valid string alone", () => { + const encoded = "postgres://u:p%40ss@host:5432/db"; + expect(normalizeConnectionString(encoded)).toBe(encoded); + }); + + test("leaves non-URL forms alone", () => { + expect(normalizeConnectionString(" ./db.sqlite ")).toBe("./db.sqlite"); + }); +}); + describe("resolveDbUrl", () => { const config = { platform: "authjs" as const, envVar: "AUTHJS_DB_URL", prompt: "url" }; @@ -120,6 +151,16 @@ describe("resolveDbUrl", () => { ).toBe("mysql://u:p@h/db"); }); + test("encodes a raw password passed to the flag", async () => { + const url = await resolveDbUrl( + { dbUrl: "postgres://u:p#ss@host:5432/db" }, + config, + NO_ENV_FILES, + {}, + ); + expect(decodeURIComponent(new URL(url).password)).toBe("p#ss"); + }); + test("rejects a flag that is not a connection string, naming the encoding trap", async () => { await expect(resolveDbUrl({ dbUrl: "not a url" }, config, NO_ENV_FILES, {})).rejects.toThrow( /URL-encode it/, diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 999e6d5be..e2957697e 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -27,22 +27,60 @@ type ResolveConfig = { hint?: string; }; +const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2):\/\//i; + +/** + * True when the string parses as a URL with a host. + * + * A hostname is required: `postgres://` alone parses as a valid URL, and + * accepting it only defers the failure into the driver. + */ +function parsesAsUrl(value: string): boolean { + try { + return new URL(value).hostname.length > 0; + } catch { + return false; + } +} + +/** + * Percent-encodes the credentials when the raw string will not parse as a URL. + * + * Dashboards hand out `postgres://user:[YOUR-PASSWORD]@host/db` and people + * paste their real password in verbatim. A `#`, `@`, `/` or `^` in it makes the + * whole string unparseable — here and later inside `Bun.SQL` — so encode it for + * them rather than bouncing a paste they cannot even see (the prompt is + * masked). Strings that already parse are returned untouched, so a password + * that was correctly encoded is never double-encoded. + */ +export function normalizeConnectionString(value: string): string { + const trimmed = value.trim(); + if (!URL_SCHEME.test(trimmed) || parsesAsUrl(trimmed)) return trimmed; + + // Greedy up to the LAST `@`: everything before it is userinfo, so an + // unencoded `@` inside the password does not split the string early. + const match = /^([a-z0-9+]+:\/\/)(.*)@([^@]*)$/i.exec(trimmed); + if (!match) return trimmed; + + const [, scheme = "", userinfo = "", rest = ""] = match; + const separator = userinfo.indexOf(":"); + const user = separator === -1 ? userinfo : userinfo.slice(0, separator); + const secret = separator === -1 ? undefined : userinfo.slice(separator + 1); + const credentials = + secret === undefined + ? encodeURIComponent(user) + : `${encodeURIComponent(user)}:${encodeURIComponent(secret)}`; + + const encoded = `${scheme}${credentials}@${rest}`; + return parsesAsUrl(encoded) ? encoded : trimmed; +} + /** True for something that could plausibly be a connection string. */ export function looksLikeConnectionString(value: string): boolean { const trimmed = value.trim(); if (!trimmed) return false; - if (/^(postgresql|postgres|mysql|mysql2):\/\//i.test(trimmed)) { - try { - // A hostname is required: `postgres://` alone parses as a valid URL, and - // accepting it only defers the failure into the driver. - return new URL(trimmed).hostname.length > 0; - } catch { - // A password with an unencoded `@` or `#` is the usual cause, and it is - // worth saying so rather than failing later inside the driver. - return false; - } - } + if (URL_SCHEME.test(trimmed)) return parsesAsUrl(trimmed); return ( trimmed.startsWith("file:") || /\.(sqlite3?|db)$/i.test(trimmed) || trimmed.startsWith("./") @@ -61,7 +99,7 @@ export async function resolveDbUrl( cwd: string = process.cwd(), env: Record = process.env, ): Promise { - const fromFlag = options.dbUrl?.trim(); + const fromFlag = options.dbUrl ? normalizeConnectionString(options.dbUrl) : undefined; if (fromFlag) { if (!looksLikeConnectionString(fromFlag)) { throwUsageError( @@ -73,7 +111,7 @@ export async function resolveDbUrl( } const located = await findMigrateEnvValue([config.envVar], cwd, env); - const fromEnv = located?.value.trim(); + const fromEnv = located ? normalizeConnectionString(located.value) : undefined; if (fromEnv) { if (looksLikeConnectionString(fromEnv)) return fromEnv; // Falling through silently would make the prompt look unexplained. @@ -100,12 +138,12 @@ export async function resolveDbUrl( const answer = await passwordPrompt({ message: config.prompt, validate: (value) => - looksLikeConnectionString(value ?? "") + looksLikeConnectionString(normalizeConnectionString(value ?? "")) ? undefined : "Expected postgres://…, mysql://… or a SQLite file path", }); - return answer.trim(); + return normalizeConnectionString(answer); } /** Describes the target for the run's opening line, credentials removed. */ From 8f374a90c76c191cf9cde5b80a68fe016257ba9a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:32:24 -0400 Subject: [PATCH 20/34] feat(migrate): choose the Clerk export source from a flat instance list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every other resolver in the CLI answers "where do I operate?" with the linked project, silently. For an export that default is actively dangerous: the linked instance is normally the migration's *destination*, so taking it without asking is how a run exports an instance and imports it straight back into itself. So a resolved instance is no longer taken silently — the account's instances are offered, one flat row each (`my-app - Production instance (ins_…)`) rather than an application picker followed by an instance picker. An application is not what an export reads from; an instance is, and dev and prod are different user pools. The resolved application's instances lead the list, so taking one is still a single Enter. `--secret-key` still names an instance outright and runs unquestioned. --- .../cli-core/src/commands/migrate/README.md | 23 +- .../migrate/export/clerk-source.test.ts | 211 ++++++++++++++++++ .../commands/migrate/export/clerk-source.ts | 173 ++++++++++++++ .../src/commands/migrate/export/clerk.ts | 17 +- .../src/commands/migrate/export/index.ts | 2 +- 5 files changed, 419 insertions(+), 7 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/export/clerk-source.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/clerk-source.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index c448e8359..b3c440bae 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -174,7 +174,28 @@ unattended rather than stalling on a prompt with everything held in memory. | `--client-secret ` | `auth0` | Machine-to-machine application client secret | `export clerk` also takes the targeting flags — it reads from a Clerk instance, -so it resolves a key exactly the way `clerk migrate` does. +so it resolves a key the same way `clerk migrate import` does, with one extra +step. The linked project is usually the migration's _destination_, so taking it +as the source without asking is how a run exports an instance and imports it +back into itself. Instead: + +- `--secret-key ` names the source instance outright and runs unquestioned. +- Anything resolved on your behalf — the linked project, a keyless app — is + never taken silently. A picker of every **instance** on your account opens + instead — one flat row each, `my-app - Production instance (ins_…)`, not an + application picker followed by an instance picker — with the resolved + application's instances listed **first** so taking one is still a single + Enter. Only when there are no instances to offer does it stop and list + `--secret-key`, `--app`/`--instance` and `clerk link` instead. +- With nothing to resolve at all (no link, no key, no flags), that same picker + opens directly, rather than an error about an unlinked directory. + +The picker has no "create a new application" choice, unlike `clerk link`'s — a +new application has no users to export. Rows are searchable by what they show, +so typing an application name, `production`, or an instance id all narrow it. + +In agent mode the resolved instance is used without a prompt; pass +`--secret-key` or `--app`/`--instance` to be explicit. After each export you get a field-coverage table — which Clerk-relevant fields were present on how many users — so you know the data is thin _before_ you diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts new file mode 100644 index 000000000..0b7fa6e46 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -0,0 +1,211 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +const mockDescribeBapiTarget = mock(); +const mockResolveBapiSecretKey = mock(); +mock.module("../../../lib/bapi-command.ts", () => ({ + describeBapiTarget: (...args: unknown[]) => mockDescribeBapiTarget(...args), + resolveBapiSecretKey: (...args: unknown[]) => mockResolveBapiSecretKey(...args), +})); + +const mockResolveProfile = mock(); +mock.module("../../../lib/config.ts", () => ({ + resolveProfile: (...args: unknown[]) => mockResolveProfile(...args), +})); + +const mockFetchApps = mock(); +mock.module("../../../lib/app-picker.ts", () => ({ + fetchAppsTolerantly: (...args: unknown[]) => mockFetchApps(...args), +})); + +const mockSearch = mock(); +mock.module("../../../lib/listage.ts", () => ({ + search: (...args: unknown[]) => mockSearch(...args), +})); + +const mockResolveUsersInstanceContext = mock(); +mock.module("../../users/interactive/instance-context.ts", () => ({ + resolveUsersInstanceContext: (...args: unknown[]) => mockResolveUsersInstanceContext(...args), +})); + +let human = true; +mock.module("../../../mode.ts", () => ({ + isHuman: () => human, + isAgent: () => !human, + getMode: () => (human ? "human" : "agent"), + setMode: () => {}, +})); + +const { resolveClerkSource } = await import("./clerk-source.ts"); + +const captured = useCaptureLog(); + +beforeEach(() => { + human = true; + mockDescribeBapiTarget.mockReset(); + mockResolveBapiSecretKey.mockReset(); + mockResolveProfile.mockReset(); + mockResolveProfile.mockResolvedValue(undefined); + mockResolveUsersInstanceContext.mockReset(); + mockFetchApps.mockReset(); + mockSearch.mockReset(); + delete process.env.CLERK_SECRET_KEY; +}); + +/** The linked-project case: something resolved, nobody asked for it. */ +function stubResolved(target: string | undefined, secretKey = "sk_test_resolved") { + mockDescribeBapiTarget.mockResolvedValue(target); + mockResolveBapiSecretKey.mockResolvedValue(secretKey); +} + +describe("resolveClerkSource", () => { + test("--secret-key names the instance outright and is never questioned", async () => { + stubResolved(undefined, "sk_test_explicit"); + + const source = await resolveClerkSource({ secretKey: "sk_test_explicit" }); + + expect(source).toEqual({ secretKey: "sk_test_explicit", target: undefined }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // Exporting the instance that is about to be imported *into* is the failure + // this whole module exists to prevent, so a resolved instance is offered as + // one choice among the account's applications rather than taken silently. + test("offers every instance, flat, with the linked application's first", async () => { + stubResolved("my-app (development)"); + mockResolveProfile.mockResolvedValue({ profile: { appId: "app_2" } }); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "my-app", + instances: [ + { instance_id: "ins_1d", environment_type: "development" }, + { instance_id: "ins_1p", environment_type: "production" }, + ], + }, + { + application_id: "app_2", + name: "other-app", + instances: [{ instance_id: "ins_2d", environment_type: "development" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + mockResolveUsersInstanceContext.mockResolvedValue({ + secretKey: "sk_live_other", + appLabel: "my-app", + instanceLabel: "production", + }); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_live_other", target: "my-app (production)" }); + // One row per instance, not per application: dev and prod are different + // user pools, and exporting the wrong one is silent. + const { message, source: listSource } = mockSearch.mock.calls[0]![0]; + expect(message).toBe("What Clerk instance do you want to export users from?"); + expect(listSource("")).toEqual([ + { + name: "other-app - Development instance (ins_2d)", + value: { app: "app_2", instance: "ins_2d" }, + }, + { + name: "my-app - Development instance (ins_1d)", + value: { app: "app_1", instance: "ins_1d" }, + }, + { + name: "my-app - Production instance (ins_1p)", + value: { app: "app_1", instance: "ins_1p" }, + }, + ]); + // Both halves are handed on, so the secret-key lookup runs against exactly + // the instance that was chosen and nothing prompts a second time. + expect(mockResolveUsersInstanceContext).toHaveBeenCalledWith({ + app: "app_1", + instance: "ins_1p", + }); + expect(captured.err).toBe(""); + }); + + // The list is searched by its rendered label, so an application id typed from + // a dashboard URL still finds its instances. + test("filters on the rendered label", async () => { + stubResolved("my-app (development)"); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "my-app", + instances: [{ instance_id: "ins_1p", environment_type: "production" }], + }, + { + application_id: "app_2", + name: "other-app", + instances: [{ instance_id: "ins_2d", environment_type: "development" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + mockResolveUsersInstanceContext.mockResolvedValue({ secretKey: "sk_live_other" }); + + await resolveClerkSource({}); + + const { source: listSource } = mockSearch.mock.calls[0]![0]; + expect(listSource("ins_2d")).toEqual([ + { + name: "other-app - Development instance (ins_2d)", + value: { app: "app_2", instance: "ins_2d" }, + }, + ]); + expect(listSource("production")).toHaveLength(1); + }); + + // An empty list is not a picker. PLAPI being degraded looks the same as an + // account with no applications, and neither one has an instance to offer. + test("no instances to offer falls back to the flags", async () => { + stubResolved("my-app (development)"); + // An application with no instances is not an offer either. + mockFetchApps.mockResolvedValue([{ application_id: "app_1", name: "my-app", instances: [] }]); + + await expect(resolveClerkSource({})).rejects.toBeInstanceOf(UserAbortError); + + expect(mockSearch).not.toHaveBeenCalled(); + expect(captured.err).toContain("--secret-key"); + expect(captured.err).toContain("--app"); + expect(captured.err).toContain("clerk link"); + }); + + test("agent mode takes the resolved instance without prompting", async () => { + human = false; + stubResolved("my-app (production)"); + + const source = await resolveClerkSource({}); + + expect(source.secretKey).toBe("sk_test_resolved"); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("an unlinked directory picks an application instead of failing", async () => { + mockDescribeBapiTarget.mockRejectedValue( + new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }), + ); + mockResolveUsersInstanceContext.mockResolvedValue({ + secretKey: "sk_test_picked", + appLabel: "other-app", + instanceLabel: "production", + }); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_test_picked", target: "other-app (production)" }); + // The picker just asked which application; asking again is noise. + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("an explicit --app that fails to resolve surfaces the error, not the picker", async () => { + const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); + mockDescribeBapiTarget.mockRejectedValue(failure); + + await expect(resolveClerkSource({ app: "app_123" })).rejects.toThrow(failure); + + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts new file mode 100644 index 000000000..0754114d5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -0,0 +1,173 @@ +/** + * Which Clerk instance `migrate export clerk` reads *from*. + * + * Every other resolver in the CLI answers "where do I operate?" with the linked + * project, silently. For an export that default is actively dangerous: the + * linked instance is normally the migration's *destination*, so taking it + * without asking is how a run ends up exporting an instance and importing it + * straight back into itself. + * + * So the source is resolved in three tiers: + * + * 1. `--secret-key` names an instance outright — it runs unquestioned. + * 2. Anything the CLI resolved on the user's behalf (the linked project, a + * keyless app) is not taken silently: every instance on the account is + * offered, with the resolved application's instances first so "yes, that + * one" is still a single Enter. + * 3. Nothing to resolve at all — no link, no key, no flags — offers those same + * instances, the trade `users list` makes, rather than failing on an + * unlinked directory. + */ + +import { fetchAppsTolerantly } from "../../../lib/app-picker.ts"; +import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; +import { resolveProfile } from "../../../lib/config.ts"; +import { CliError, ERROR_CODE, throwUserAbort } from "../../../lib/errors.ts"; +import { search } from "../../../lib/listage.ts"; +import type { ApplicationInstance } from "../../../lib/plapi.ts"; +import { log } from "../../../lib/log.ts"; +import { isHuman } from "../../../mode.ts"; +import { resolveUsersInstanceContext } from "../../users/interactive/instance-context.ts"; + +/** e.g. `Development instance`. Unknown environment types print as-is. */ +function instanceLabel(instance: ApplicationInstance): string { + const type = instance.environment_type; + if (!type) return "instance"; + return `${type.charAt(0).toUpperCase()}${type.slice(1)} instance`; +} + +export type ResolveClerkSourceOptions = { + secretKey?: string; + app?: string; + instance?: string; + cwd?: string; +}; + +export type ClerkExportSource = { + secretKey: string; + /** + * Human-readable target, e.g. `my-app (production)`. Absent when + * `--secret-key` (or `CLERK_SECRET_KEY`) named the instance directly, since + * a bare key carries no application context to describe. + */ + target?: string; +}; + +/** {@link ClerkExportSource} plus whether the user already chose it out loud. */ +type ResolvedSource = ClerkExportSource & { chosen: boolean }; + +async function resolveSource(options: ResolveClerkSourceOptions): Promise { + try { + return { + target: await describeBapiTarget(options), + secretKey: await resolveBapiSecretKey(options), + // Flags and env keys are a choice the user typed; the linked project is + // one they made for some other purpose, possibly months ago. + chosen: Boolean(options.secretKey), + }; + } catch (error) { + const hasExplicitTarget = + Boolean(options.secretKey) || + Boolean(options.app) || + Boolean(options.instance) || + Boolean(process.env.CLERK_SECRET_KEY); + + if ( + !isHuman() || + hasExplicitTarget || + !(error instanceof CliError) || + error.code !== ERROR_CODE.NO_SECRET_KEY + ) { + throw error; + } + + const ctx = await resolveUsersInstanceContext({}); + return { + secretKey: ctx.secretKey, + target: ctx.appLabel ? `${ctx.appLabel} (${ctx.instanceLabel})` : undefined, + // The picker just asked. Confirming the answer to a question the user + // answered one prompt ago is noise. + chosen: true, + }; + } +} + +/** + * Offers every instance on the account, flat — one row per instance rather than + * an application picker followed by an instance picker. + * + * An application is not what an export reads from; an instance is. Picking + * "Migration Test" and then "development" is two questions with one answer, and + * it hides the thing that actually matters — dev and prod are different user + * pools, and exporting the wrong one is silent. + * + * Deliberately not `pickOrCreateApp`: its "+ Create a new application" choice + * makes sense when you are choosing somewhere to *write*, and no sense at all + * as an export source — a brand-new application has no users in it. + * + * @param currentAppId the application the CLI resolved on the user's behalf. + * Its instances lead the list, because they are the likeliest answer. + * @returns undefined when there is nothing to offer, so the caller can fall + * back to telling the user which flags to pass instead of showing an empty + * list. `fetchAppsTolerantly` returns empty on a degraded PLAPI, not just on + * an account with no applications. + */ +async function pickInstance(currentAppId?: string): Promise { + const apps = await fetchAppsTolerantly(); + + const ordered = currentAppId + ? [ + ...apps.filter((app) => app.application_id === currentAppId), + ...apps.filter((app) => app.application_id !== currentAppId), + ] + : apps; + + const choices = ordered.flatMap((app) => + (app.instances ?? []).map((instance) => ({ + name: `${app.name || app.application_id} - ${instanceLabel(instance)} (${instance.instance_id})`, + value: { app: app.application_id, instance: instance.instance_id }, + })), + ); + if (choices.length === 0) return undefined; + + const picked = await search<{ app: string; instance: string }>({ + message: "What Clerk instance do you want to export users from?", + source: (term) => + term + ? choices.filter((choice) => choice.name.toLowerCase().includes(term.toLowerCase())) + : choices, + }); + + // Both halves are passed on, so the secret-key lookup runs against exactly + // the instance that was chosen and nothing prompts a second time. + const ctx = await resolveUsersInstanceContext(picked); + return { + secretKey: ctx.secretKey, + target: ctx.appLabel ? `${ctx.appLabel} (${ctx.instanceLabel})` : undefined, + }; +} + +/** The application the CLI resolved on the user's behalf, if it knows one. */ +async function currentAppId(options: ResolveClerkSourceOptions): Promise { + if (options.app) return options.app; + const resolved = await resolveProfile(options.cwd ?? process.cwd()).catch(() => undefined); + return resolved?.profile.appId; +} + +export async function resolveClerkSource( + options: ResolveClerkSourceOptions, +): Promise { + const { chosen, ...source } = await resolveSource(options); + if (chosen || !source.target || !isHuman()) return source; + + const picked = await pickInstance(await currentAppId(options)); + if (picked) return picked; + + log.info( + "Export from a different instance with one of:\n" + + " `--secret-key ` — the source instance's secret key\n" + + " `--app --instance ` — another application on your account\n" + + " `clerk link` — link this directory to a different application first", + ); + throwUserAbort(); +} diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index fc3875339..ec4a633be 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -15,11 +15,11 @@ */ import { bapiRequest } from "../../../lib/bapi.ts"; -import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { retryOn429 } from "../lib/retry.ts"; +import { resolveClerkSource } from "./clerk-source.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; /** BAPI's maximum page size for `GET /v1/users`. */ @@ -223,17 +223,24 @@ export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExpo } export async function exportClerk(options: ExportClerkOptions): Promise { + // Resolved before the gutter opens, the way `export auth0` resolves its + // credentials: confirming the source is a question about whether to run at + // all, not a step of the run. + const source = await resolveClerkSource({ + secretKey: options.secretKey, + app: options.app, + instance: options.instance, + }); + const destination = await resolveOutputPath("clerk", options.output); await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { - const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); - const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const dateTime = getDateTimeStamp(); - log.info(`Exporting from ${target ?? "the resolved instance"}.`); + log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); const users = await withSpinner("Fetching users from Clerk...", (spinner) => - fetchAllClerkUsers({ secretKey, spinner }), + fetchAllClerkUsers({ secretKey: source.secretKey, spinner }), ); const { users: exported, coverage } = buildClerkExport(users, dateTime); diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index f9f99dea6..b11ea8814 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -111,7 +111,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json", + description: "Prompts for the source instance and where to save the file", }, { command: "clerk migrate export clerk --instance prod --output prod-users.json", From f0777861ab7aadbf12734284b69dcfd6a538b44f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:32:36 -0400 Subject: [PATCH 21/34] feat(migrate): prompt for the Firebase service account key `export firebase` without `--service-account` exited with a usage error, which is a dead end in the interactive picker: choose Firebase, get told to re-run with a flag. It now prompts, the way `export supabase` prompts for its connection string. The answer can be a path to the downloaded file *or* the key's JSON pasted whole, so a key kept in a password manager or a CI secret never has to be written to disk. Prompted as a password, since the JSON carries a private key. Agent mode has nobody to ask, so it still names the flag. --- .../cli-core/src/commands/migrate/README.md | 16 ++- .../commands/migrate/export/firebase.test.ts | 27 +++- .../src/commands/migrate/export/firebase.ts | 128 +++++++++++++----- 3 files changed, 132 insertions(+), 39 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index b3c440bae..51dcd957d 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -284,10 +284,18 @@ clerk migrate export firebase --service-account ./service-account.json ``` Needs a service account key from **Project settings → Service accounts → -Generate new private key**, with the Firebase Authentication Admin role. The -file is validated before anything reaches the network, so downloading the web -app config by mistake fails in a second with the right console page named -rather than after an auth round-trip. Key material never appears in output. +Generate new private key**, with the Firebase Authentication Admin role. + +Without `--service-account` you are prompted for it, the way `export supabase` +prompts for its connection string. The answer can be a path to the downloaded +file _or_ the key's JSON pasted whole, so a key kept in a password manager or a +CI secret never has to be written to disk. The prompt is masked, since the key +carries a private key. Agent mode cannot prompt, so it names the flag instead. + +Either way the key is validated before anything reaches the network, so +downloading the web app config by mistake fails in a second with the right +console page named rather than after an auth round-trip. Key material never +appears in output. Firebase's scrypt is a modified variant, so a digest is worthless without the project's four hash parameters. The export **reads them from the project** and diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 5754bf901..4f9c4ac25 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -13,6 +13,7 @@ import { fetchAllFirebaseUsers, fetchHashConfig, formatHashConfigGuidance, + loadServiceAccount, mapFirebaseUserToExport, readServiceAccount, signServiceAccountJwt, @@ -111,6 +112,28 @@ const fbUser = (i: number, overrides: Record = {}) => ({ ...overrides, }); +describe("loadServiceAccount", () => { + // The prompt takes either, so a key pasted out of a password manager never + // has to be written to disk first. + test("accepts the key JSON pasted whole", () => { + expect(loadServiceAccount(` ${JSON.stringify(account)} `).project_id).toBe("demo-fb"); + }); + + test("accepts a path to the key file", () => { + expect(loadServiceAccount("./sa.json").project_id).toBe("demo-fb"); + }); + + test("rejects a paste that is not valid JSON", () => { + expect(() => loadServiceAccount('{"project_id":')).toThrow(/pasted key is not valid JSON/); + }); + + test("rejects a paste missing a required field", () => { + expect(() => loadServiceAccount('{"type":"service_account"}')).toThrow( + /The pasted key is not a usable service account key: "project_id" is missing/, + ); + }); +}); + describe("readServiceAccount", () => { test("reads a valid key file", () => { expect(readServiceAccount("./sa.json").project_id).toBe("demo-fb"); @@ -450,7 +473,9 @@ describe("exportFirebase", () => { expect(fs.existsSync(path.join(workDir, "fb.json"))).toBe(true); }); - test("requires --service-account, before anything is read", async () => { + // Human runs get a prompt instead; an agent has nobody to ask, so it is told + // which flag to pass. + test("agent mode names the flag rather than prompting", async () => { await expect(exportFirebase({})).rejects.toThrow(/needs a service account key file/); }); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index acb413e6d..6facdd770 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -28,6 +28,8 @@ import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; import { bold, dim } from "../../../lib/color.ts"; import { loggedFetch } from "../../../lib/fetch.ts"; import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt } from "../../../lib/prompts.ts"; +import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; @@ -55,12 +57,43 @@ export type ServiceAccount = { }; /** - * Reads and validates a service-account key file. + * Validates already-parsed JSON as a service-account key. * * Every failure names the field, because the usual causes are downloading the * wrong JSON from the console (a web app config rather than a service account) * or pasting a key with its newlines mangled. + * + * @param label how to refer to the source in an error — a file name, or + * "the pasted key" when it came from the prompt. */ +function validateServiceAccount(parsed: unknown, label: string): ServiceAccount { + const account = parsed as Partial & { type?: string }; + const invalid = (problem: string): never => { + throw new CliError(`${label} is not a usable service account key: ${problem}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + }; + + if (account.type && account.type !== "service_account") { + invalid( + `its "type" is "${account.type}", not "service_account". Download a private key from ` + + "Project settings → Service accounts → Generate new private key.", + ); + } + for (const field of ["project_id", "client_email", "private_key"] as const) { + if (typeof account[field] !== "string" || account[field].length === 0) { + invalid(`"${field}" is missing`); + } + } + if (!account.private_key?.includes("PRIVATE KEY")) { + invalid('"private_key" does not look like a PEM key — check its newlines survived copying'); + } + + return account as ServiceAccount; +} + +/** Reads and validates a service-account key file. */ export function readServiceAccount(file: string): ServiceAccount { const resolved = path.resolve(process.cwd(), file); @@ -81,30 +114,71 @@ export function readServiceAccount(file: string): ServiceAccount { }); } - const account = parsed as Partial & { type?: string }; - const invalid = (problem: string): never => { - throw new CliError(`${file} is not a usable service account key: ${problem}`, { - code: ERROR_CODE.USAGE_ERROR, + return validateServiceAccount(parsed, file); +} + +/** + * Accepts what the prompt accepts: a path to the downloaded key file, or the + * key's JSON pasted in whole. Console downloads land as a file, but a key + * copied out of a password manager or CI secret never touches disk. + */ +export function loadServiceAccount(source: string): ServiceAccount { + const trimmed = source.trim(); + if (!trimmed.startsWith("{")) return readServiceAccount(trimmed); + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch (error) { + throw new CliError(`The pasted key is not valid JSON: ${(error as Error).message}`, { + code: ERROR_CODE.INVALID_JSON, docsUrl: DOCS_URL, }); - }; + } - if (account.type && account.type !== "service_account") { - invalid( - `its "type" is "${account.type}", not "service_account". Download a private key from ` + - "Project settings → Service accounts → Generate new private key.", + return validateServiceAccount(parsed, "The pasted key"); +} + +/** + * Resolves the key: the flag, then a prompt — the shape `export supabase` uses + * for its connection string. Prompted as a password: the JSON carries a private + * key, and a path typed blind is short enough to survive being masked. + */ +async function resolveServiceAccount(options: ExportFirebaseOptions): Promise { + if (options.serviceAccount) return loadServiceAccount(options.serviceAccount); + + if (!isHuman()) { + throwUsageError( + "`clerk migrate export firebase` needs a service account key file and cannot prompt here. " + + "Pass --service-account .", + DOCS_URL, + undefined, + [ + { + command: "clerk migrate export firebase --service-account ./service-account.json", + description: "Export using a downloaded service account key", + }, + ], ); } - for (const field of ["project_id", "client_email", "private_key"] as const) { - if (typeof account[field] !== "string" || account[field].length === 0) { - invalid(`"${field}" is missing`); - } - } - if (!account.private_key?.includes("PRIVATE KEY")) { - invalid('"private_key" does not look like a PEM key — check its newlines survived copying'); - } - return account as ServiceAccount; + log.info( + dim("Firebase console → Project settings → Service accounts → Generate new private key."), + ); + + const answer = await passwordPrompt({ + message: "Path to the service account key file, or paste the key JSON", + validate: (value) => { + try { + loadServiceAccount(value ?? ""); + return undefined; + } catch (error) { + return error instanceof CliError ? error.message : String(error); + } + }, + }); + + return loadServiceAccount(answer); } function base64Url(input: string | Uint8Array): string { @@ -406,23 +480,9 @@ export function formatHashConfigGuidance( } export async function exportFirebase(options: ExportFirebaseOptions): Promise { - if (!options.serviceAccount) { - throwUsageError( - "`clerk migrate export firebase` needs a service account key file. Pass --service-account .", - DOCS_URL, - undefined, - [ - { - command: "clerk migrate export firebase --service-account ./service-account.json", - description: "Export using a downloaded service account key", - }, - ], - ); - } - // Read and validate before anything reaches the network, so a wrong file // fails in a second rather than after an auth round-trip. - const account = readServiceAccount(options.serviceAccount); + const account = await resolveServiceAccount(options); const destination = await resolveOutputPath("firebase", options.output); From f0ba767c55a3ab189b531cbd4f83b3871e932a94 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 18 Aug 2026 17:32:36 -0400 Subject: [PATCH 22/34] refactor(migrate): restyle `clerk migrate transformers list` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A section heading and a leading sentence, matching how `--help` lays out its own sections, with each description wrapped rather than run off the edge. Width is capped at 80 columns, not merely measured, so two runs of the same command lay out the same way on different terminals. A backticked span is never broken across lines: `log.info` pairs backticks per line, so a split span leaves an unmatched backtick on each and colours the wrong half of both. No gutter — this reads a static registry, it does not run anything — and no dimmed text, which the descriptions are the whole point of. --- .../cli-core/src/commands/migrate/README.md | 22 +++++ .../migrate/transformers/list.test.ts | 48 +++++++--- .../src/commands/migrate/transformers/list.ts | 94 ++++++++++++++----- 3 files changed, 131 insertions(+), 33 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 51dcd957d..8d8e97e4c 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -485,6 +485,28 @@ clerk migrate transformers list --transformer-file ./my-transformer.ts | `--json` | Output as JSON | | `--transformer-file ` | Also list a transformer you wrote | +Each entry prints its key, the platform label, and what the transformer assumes +about the export — wrapped to the terminal, capped at 80 columns so two runs of +the same command lay out the same way. A backticked span is never broken across +lines. There is no intro/outro gutter: this reads a static registry rather than +running anything. + +``` +A transformer maps one platform's export onto the fields Clerk imports. Pass the +one your export came from as `--transformer `. + +Transformers: + clerk Clerk + Migrate between Clerk instances (e.g. development to production, or to + another Clerk application). Export your users from the Clerk Dashboard + first. + + … + +6 built-in transformers +Migrating from something else? Write a transformer and pass --transformer-file. +``` + `--json` gives an agent the same data, including which source field each transformer maps to `userId`. diff --git a/packages/cli-core/src/commands/migrate/transformers/list.test.ts b/packages/cli-core/src/commands/migrate/transformers/list.test.ts index 26179ee9f..51c57eca2 100644 --- a/packages/cli-core/src/commands/migrate/transformers/list.test.ts +++ b/packages/cli-core/src/commands/migrate/transformers/list.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import { getMode, setMode, type Mode } from "../../../mode.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { list } from "./list.ts"; +import { list, wrapText } from "./list.ts"; import { transformers } from "./registry.ts"; const captured = useCaptureLog(); @@ -43,11 +43,15 @@ describe("human output", () => { expect(captured.err).toContain(transformer.label); }); - // `log.info` auto-highlights backticked spans, so the rendered description - // carries colour codes the source string does not. + // Two normalizations: `log.info` auto-highlights backticked spans, so the + // rendered description carries colour codes the source string does not, and + // descriptions are wrapped to the terminal width across several indented + // lines. Collapsing whitespace compares the words, not the layout. + const collapse = (value: string) => stripAnsi(value).replace(/\s+/g, " "); + test.each([...transformers])("includes the $key description", async (transformer) => { await list(); - expect(stripAnsi(captured.err)).toContain(transformer.description); + expect(collapse(captured.err)).toContain(collapse(transformer.description)); }); test("counts the built-ins", async () => { @@ -128,19 +132,41 @@ describe("human-mode frame", () => { setMode(originalMode); }); - test("wraps its output in an intro/outro gutter", async () => { + // Reading a static registry is not a run: there is no progress to bracket, + // and the gutter's `│` would sit in front of every wrapped line. + test("prints no intro/outro gutter", async () => { await list(); - expect(captured.err).toContain("┌"); - expect(captured.err).toContain("Listing transformers"); - expect(captured.err).toContain("└"); - expect(captured.err).toContain("Done"); + expect(captured.err).not.toContain("┌"); + expect(captured.err).not.toContain("└"); + expect(stripAnsi(captured.err)).toContain("Transformers:"); }); - test("--json stays outside the gutter, on stdout only", async () => { + test("--json stays on stdout only", async () => { await list({ json: true }); expect(() => JSON.parse(captured.out)).not.toThrow(); - expect(captured.err).not.toContain("┌"); + expect(captured.err).toBe(""); + }); +}); + +describe("wrapText", () => { + test("breaks on whitespace within the width", () => { + expect(wrapText("one two three four", 9)).toEqual(["one two", "three", "four"]); + }); + + // `log.info` pairs backticks per line, so a span split across two lines + // leaves one unmatched backtick on each and colours the wrong half of both. + test("never breaks inside a backticked span", () => { + const lines = wrapText("Assumes an export of `SELECT id, name FROM users`.", 30); + + expect(lines).toContain("`SELECT id, name FROM users`."); + for (const line of lines) { + expect((line.match(/`/g) ?? []).length % 2).toBe(0); + } + }); + + test("gives an over-long word its own line rather than dropping it", () => { + expect(wrapText("short supercalifragilistic", 8)).toEqual(["short", "supercalifragilistic"]); }); }); diff --git a/packages/cli-core/src/commands/migrate/transformers/list.ts b/packages/cli-core/src/commands/migrate/transformers/list.ts index 37da1e0a5..993ee4838 100644 --- a/packages/cli-core/src/commands/migrate/transformers/list.ts +++ b/packages/cli-core/src/commands/migrate/transformers/list.ts @@ -6,9 +6,8 @@ * A compiled binary's users have neither, so the list is a command. */ -import { bold, cyan, dim } from "../../../lib/color.ts"; +import { bold, cyan } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; -import { withGutter } from "../../../lib/spinner.ts"; import type { TransformerRegistryEntry } from "../types.ts"; import { loadCustomTransformer } from "./load-custom.ts"; import { transformers } from "./registry.ts"; @@ -32,6 +31,47 @@ function toJson(entries: Listed[]) { })); } +/** + * Capped, not just measured: a description that rewrapped differently on every + * terminal makes two runs of the same command look like different output. 80 is + * the same width `--help` lays itself out at. + */ +const MAX_WIDTH = 80; + +function outputWidth(): number { + return Math.min(process.stderr.columns || MAX_WIDTH, MAX_WIDTH); +} + +/** + * A run of non-space characters, except that a backticked span counts as one + * character run even when it contains spaces. Keeps `SELECT a, b FROM users` + * whole: `log.info` pairs backticks per line, so a span broken across two lines + * leaves an unmatched backtick on each and colours the wrong half of both. + */ +const WORD = /(?:`[^`]*`|\S)+/g; + +/** + * Wraps on whitespace. Safe to measure raw because the backtick spans + * `log.info` highlights keep their backticks — the colour it adds is invisible + * to width, and nothing here is coloured before wrapping. + */ +export function wrapText(text: string, width: number): string[] { + const lines: string[] = []; + let line = ""; + + for (const word of text.match(WORD) ?? []) { + if (!line) line = word; + else if (line.length + 1 + word.length <= width) line += ` ${word}`; + else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + + return lines; +} + export async function list(options: TransformersListOptions = {}): Promise { const entries: Listed[] = transformers.map((entry) => ({ ...entry, builtIn: true })); @@ -45,26 +85,36 @@ export async function list(options: TransformersListOptions = {}): Promise return; } - await withGutter("Listing transformers", async () => { - for (const entry of entries) { - const suffix = entry.builtIn ? "" : ` ${dim(`(custom — ${entry.source})`)}`; - log.info(`${cyan(bold(entry.key))} ${entry.label}${suffix}`); - log.info(` ${dim(entry.description)}`); - log.info(""); - } + const width = outputWidth(); - const custom = entries.length - transformers.length; - log.info( - dim( - `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + - (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), - ), - ); - - if (custom === 0) { - log.info( - dim("Migrating from something else? Write a transformer and pass --transformer-file."), - ); + // No gutter: this reads a static registry, it does not run anything. The + // frame belongs on `migrate import`, where there is progress to bracket. + for (const line of wrapText( + "A transformer maps one platform's export onto the fields Clerk imports. " + + "Pass the one your export came from as `--transformer `.", + width, + )) { + log.info(line); + } + log.blank(); + + log.info(bold("Transformers:")); + for (const entry of entries) { + const suffix = entry.builtIn ? "" : ` (custom — ${entry.source})`; + log.info(` ${cyan(bold(entry.key))} ${entry.label}${suffix}`); + for (const line of wrapText(entry.description, width - 4)) { + log.info(` ${line}`); } - }); + log.blank(); + } + + const custom = entries.length - transformers.length; + log.info( + `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + + (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), + ); + + if (custom === 0) { + log.info("Migrating from something else? Write a transformer and pass --transformer-file."); + } } From aa62b62162b0e572351955c75fc67b2ad363ea24 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 8 Sep 2026 11:31:29 -0400 Subject: [PATCH 23/34] refactor(migrate): name log files after the commands that write them `migrate import` now writes `import-.log` and `migrate delete` writes `delete-.log`, so a listing points at the command behind each line. The old `migration-` and `user-deletion-` names, written by the standalone tool and earlier CLI builds, still classify and convert. `migrate logs list` leads with the filename (what `logs convert` and `logs clean` talk about), renders the UTC stamp in the reader's own zone, prints the log directory relative to the cwd, and closes with a fixed legend of every kind rather than only the ones present. --- .../cli-core/src/commands/migrate/README.md | 51 ++++++-- .../src/commands/migrate/delete.test.ts | 2 +- .../cli-core/src/commands/migrate/delete.ts | 2 +- .../src/commands/migrate/import-users.test.ts | 2 +- .../commands/migrate/lib/log-files.test.ts | 45 +++---- .../src/commands/migrate/lib/log-files.ts | 24 +++- .../src/commands/migrate/lib/logger.test.ts | 6 +- .../src/commands/migrate/lib/logger.ts | 34 +++-- .../src/commands/migrate/logs/list.ts | 91 +++++++++++--- .../migrate/logs/logs-interactive.test.ts | 46 +++---- .../src/commands/migrate/logs/logs.test.ts | 119 +++++++++++------- .../cli-core/src/commands/migrate/run.test.ts | 2 +- packages/cli-core/src/commands/migrate/run.ts | 2 +- 13 files changed, 283 insertions(+), 143 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 8d8e97e4c..bd601ca00 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -380,7 +380,7 @@ Rate limiting and 429 retries are literally the same code path as the import A failure on one user is logged and the rest continue: a half-undone migration with no record of which half is far worse than a reported failure. Every -attempt lands in a timestamped `logs/user-deletion-.log`, carrying +attempt lands in a timestamped `logs/delete-.log`, carrying both the source ID and the Clerk ID. The command exits non-zero if any deletion failed. @@ -399,12 +399,12 @@ clerk migrate logs # defaults to list clerk migrate logs list --json clerk migrate logs clean -y clerk migrate logs convert --all -clerk migrate logs convert migration-2026-01-01T12-00-00.log +clerk migrate logs convert import-2026-01-01T12-00-00.log ``` | Subcommand | Takes | Description | | -------------- | ------------------ | ----------------------------------------------- | -| `logs list` | `--json` | Type, timestamp, size and entry count per file | +| `logs list` | `--json` | File, type, date, size and entry count per file | | `logs clean` | `-y, --yes` | Delete the `.log` files in `./logs/` | | `logs convert` | `[file…]`, `--all` | NDJSON → a JSON array, written as `.json` | @@ -414,15 +414,42 @@ All three read the directory through one shared enumerator, which is what makes #### `logs list` The default, because listing is read-only and therefore safe to run by -accident. Reports each file's type, timestamp, size and entry count, newest +accident. Reports each file's name, type, date, size and entry count, newest first; `--json` gives an agent the same data without parsing NDJSON. ``` -TYPE TIMESTAMP SIZE ENTRIES -migration 2026-02-01T09-14-22 4.1 KB 120 -deletion 2026-01-30T17-02-51 612 B 18 +Each log represents a user export, user import, or a user delete run. +Each log consists of a single NDJSON entry per user. + +FILE TYPE DATE SIZE ENTRIES +import-2026-02-01T09-14-22.log import Feb 1, 2026 at 4:14 AM 4.1 KB 120 +delete-2026-01-30T17-02-51.log delete Jan 30, 2026 at 12:02 PM 612 B 18 + +2 log files in ./logs + +Log types: + export One entry per user pulled from the source platform. + import One entry per user created in Clerk, with any error. + delete One entry per user removed from Clerk, with any error. ``` +A kind is the name of the command that wrote it — `migrate import` writes +`import-.log` — so a listing points straight at the run behind each +line. The legend is fixed rather than derived from what happens to be present, +because "what else could be here" is the other half of the question. + +The older `migration-` and `user-deletion-` names, written by the standalone +tool and by earlier CLI builds, still classify as `import` and `delete`, so a +directory of old logs lists and converts unchanged. + +The filename leads, because it is what `logs convert` and `logs clean` talk +about. The date column renders the filename's UTC stamp in the reader's own +zone — "which run was that" is a question about local time; `--json` keeps the +raw stamp. + +The directory is printed relative (`./logs`) when it sits under the current +directory and absolute when it does not, so the path can be pasted either way. + Says so plainly when `./logs/` is empty or absent. #### `logs clean` @@ -444,7 +471,7 @@ A malformed line is reported with its line number and skipped, and the remaining entries still convert: ``` -migration-2026-01-01T12-00-00.log:2 is not valid JSON and was skipped — … +import-2026-01-01T12-00-00.log:2 is not valid JSON and was skipped — … 1 malformed line skipped. ``` @@ -928,9 +955,9 @@ rather than "which project is linked here". | Path | Contents | | ------------------------------------------ | --------------------------------------------------------------------- | -| `./logs/migration-.log` | NDJSON: one line per user, plus validation failures and retry notices | -| `./logs/user-deletion-.log` | NDJSON: one line per `migrate delete` attempt | | `./logs/export-.log` | NDJSON: one line per exported user | +| `./logs/import-.log` | NDJSON: one line per user, plus validation failures and retry notices | +| `./logs/delete-.log` | NDJSON: one line per `migrate delete` attempt | | `./exports/-export-.json` | The export itself, unless `--output` says otherwise | | `./.env.clerk-migrate` | Migration credentials, written by `settings set` and gitignored | @@ -959,8 +986,8 @@ long append-only stream, and that format is the one that survives it: Which is also why it greps usefully without any tooling: ```sh -grep '"status":"success"' logs/migration-2026-01-01T12-00-00.log | wc -l -grep '"userId":"user_123"' logs/migration-2026-01-01T12-00-00.log +grep '"status":"success"' logs/import-2026-01-01T12-00-00.log | wc -l +grep '"userId":"user_123"' logs/import-2026-01-01T12-00-00.log ``` The trade-off is that spreadsheets, databases and most JSON tooling want an diff --git a/packages/cli-core/src/commands/migrate/delete.test.ts b/packages/cli-core/src/commands/migrate/delete.test.ts index 01711ec97..3ae3131ac 100644 --- a/packages/cli-core/src/commands/migrate/delete.test.ts +++ b/packages/cli-core/src/commands/migrate/delete.test.ts @@ -373,7 +373,7 @@ describe("deleteMigration", () => { const logs = fs.readdirSync(getLogDir()); expect(logs).toHaveLength(1); - expect(logs[0]).toMatch(/^user-deletion-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + expect(logs[0]).toMatch(/^delete-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); }); test("leaves users the migration did not create alone", async () => { diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 06781d3cd..32e7d4c41 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -264,7 +264,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise { const logEntries = () => fs - .readFileSync(getLogFilePath("migration", DATE_TIME), "utf-8") + .readFileSync(getLogFilePath("import", DATE_TIME), "utf-8") .trim() .split("\n") .map((line) => JSON.parse(line) as Record); diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.test.ts b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts index 1af83c1f0..cc7587453 100644 --- a/packages/cli-core/src/commands/migrate/lib/log-files.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts @@ -33,14 +33,17 @@ function writeLog(name: string, entries: unknown[]): string { describe("classifyLogFile", () => { test.each([ - ["migration-2026-01-01T12-00-00.log", "migration", "2026-01-01T12-00-00"], - ["user-deletion-2026-01-01T12-00-00.log", "deletion", "2026-01-01T12-00-00"], + ["import-2026-01-01T12-00-00.log", "import", "2026-01-01T12-00-00"], + ["delete-2026-01-01T12-00-00.log", "delete", "2026-01-01T12-00-00"], ["export-2026-01-01T12-00-00.log", "export", "2026-01-01T12-00-00"], + // Written by the standalone tool and by earlier CLI builds. + ["migration-2026-01-01T12-00-00.log", "import", "2026-01-01T12-00-00"], + ["user-deletion-2026-01-01T12-00-00.log", "delete", "2026-01-01T12-00-00"], ])("%s is a %s log from %s", (name, kind, timestamp) => { expect(classifyLogFile(name)).toEqual({ kind: kind as never, timestamp }); }); - test.each([["random.log"], ["migration.log"], ["notes.txt"]])( + test.each([["random.log"], ["import.log"], ["notes.txt"]])( "%s is unrecognized rather than a parse failure", (name) => { expect(classifyLogFile(name)).toEqual({ kind: "unknown", timestamp: "" }); @@ -60,12 +63,12 @@ describe("listLogFiles", () => { }); test("reports kind, timestamp, size and entry count per file", () => { - writeLog("migration-2026-01-01T12-00-00.log", [{ userId: "u1" }, { userId: "u2" }]); + writeLog("import-2026-01-01T12-00-00.log", [{ userId: "u1" }, { userId: "u2" }]); const [file] = listLogFiles(); expect(file).toMatchObject({ - name: "migration-2026-01-01T12-00-00.log", - kind: "migration", + name: "import-2026-01-01T12-00-00.log", + kind: "import", timestamp: "2026-01-01T12-00-00", entryCount: 2, }); @@ -73,11 +76,11 @@ describe("listLogFiles", () => { }); test("ignores files that are not logs", () => { - writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); - fs.writeFileSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"), "[]"); + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); + fs.writeFileSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"), "[]"); fs.writeFileSync(path.join(getLogDir(), "notes.txt"), "hi"); - expect(listLogFiles().map((file) => file.name)).toEqual(["migration-2026-01-01T12-00-00.log"]); + expect(listLogFiles().map((file) => file.name)).toEqual(["import-2026-01-01T12-00-00.log"]); }); test("ignores subdirectories", () => { @@ -86,9 +89,9 @@ describe("listLogFiles", () => { }); test("returns the newest run first", () => { - writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); - writeLog("migration-2026-03-01T12-00-00.log", [{ a: 1 }]); - writeLog("migration-2026-02-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-03-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-02-01T12-00-00.log", [{ a: 1 }]); expect(listLogFiles().map((file) => file.timestamp)).toEqual([ "2026-03-01T12-00-00", @@ -97,21 +100,21 @@ describe("listLogFiles", () => { ]); }); - // Sorting on the filename would put every "user-deletion-" ahead of every + // Sorting on the filename would put every "import-" ahead of every // "migration-", regardless of when the runs actually happened. test("orders by timestamp across log kinds, not by the name's prefix", () => { - writeLog("user-deletion-2026-01-30T17-02-51.log", [{ a: 1 }]); - writeLog("migration-2026-02-01T09-14-22.log", [{ a: 1 }]); + writeLog("import-2026-01-30T17-02-51.log", [{ a: 1 }]); + writeLog("export-2026-02-01T09-14-22.log", [{ a: 1 }]); - expect(listLogFiles().map((file) => file.kind)).toEqual(["migration", "deletion"]); + expect(listLogFiles().map((file) => file.kind)).toEqual(["export", "import"]); }); test("sorts unrecognized names last", () => { writeLog("something-else.log", [{ a: 1 }]); - writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); expect(listLogFiles().map((file) => file.name)).toEqual([ - "migration-2026-01-01T12-00-00.log", + "import-2026-01-01T12-00-00.log", "something-else.log", ]); }); @@ -130,15 +133,15 @@ describe("listLogFiles", () => { describe("findLogFile", () => { beforeEach(() => { - writeLog("migration-2026-01-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); }); test("finds a log by name", () => { - expect(findLogFile("migration-2026-01-01T12-00-00.log")?.entryCount).toBe(1); + expect(findLogFile("import-2026-01-01T12-00-00.log")?.entryCount).toBe(1); }); test("accepts a path and matches on the basename", () => { - expect(findLogFile("./logs/migration-2026-01-01T12-00-00.log")?.entryCount).toBe(1); + expect(findLogFile("./logs/import-2026-01-01T12-00-00.log")?.entryCount).toBe(1); }); test("returns nothing for a name that is not there", () => { diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.ts b/packages/cli-core/src/commands/migrate/lib/log-files.ts index 1498ddd36..af87c6712 100644 --- a/packages/cli-core/src/commands/migrate/lib/log-files.ts +++ b/packages/cli-core/src/commands/migrate/lib/log-files.ts @@ -10,15 +10,27 @@ import fs from "node:fs"; import path from "node:path"; import { getLogDir } from "./logger.ts"; -/** The run that produced a log file, read from its filename prefix. */ -export type LogKind = "migration" | "deletion" | "export" | "unknown"; +/** + * The run that produced a log file, read from its filename prefix. + * + * The kinds are the command names — `migrate import` writes `import-*.log` — + * so a listing points straight at the command that produced each line. + */ +export type LogKind = "export" | "import" | "delete" | "unknown"; -const FILENAME_PATTERN = /^(migration|user-deletion|export)-(.+)\.log$/; +const FILENAME_PATTERN = /^(export|import|delete|migration|user-deletion)-(.+)\.log$/; +/** + * `migration-` and `user-deletion-` are the names the standalone tool and + * earlier CLI builds wrote. They still classify, so a directory of older logs + * lists and converts rather than reading as "unknown". + */ const KIND_BY_PREFIX: Record = { - migration: "migration", - "user-deletion": "deletion", export: "export", + import: "import", + delete: "delete", + migration: "import", + "user-deletion": "delete", }; export type LogFile = { @@ -84,7 +96,7 @@ export function listLogFiles(): LogFile[] { // Sort on the timestamp, not the filename: the kind prefix sorts first in a // filename comparison, which would interleave a run from January ahead of one - // from March purely because "user-deletion" > "migration". Timestamps are + // from March purely because "import" > "export". Timestamps are // ISO-ish and zero-padded, so lexical order is chronological. Names without // one sort last, then alphabetically. return files.sort( diff --git a/packages/cli-core/src/commands/migrate/lib/logger.test.ts b/packages/cli-core/src/commands/migrate/lib/logger.test.ts index db973f00a..eb153218d 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.test.ts @@ -35,7 +35,7 @@ beforeEach(() => { function readEntries(): Record[] { return fs - .readFileSync(getLogFilePath("migration", DATE_TIME), "utf-8") + .readFileSync(getLogFilePath("import", DATE_TIME), "utf-8") .trim() .split("\n") .map((line) => JSON.parse(line) as Record); @@ -47,8 +47,8 @@ describe("log file paths", () => { }); test("replaces the timestamp's colons so the name is valid on Windows", () => { - expect(path.basename(getLogFilePath("migration", DATE_TIME))).toBe( - "migration-2026-01-01T12-00-00.log", + expect(path.basename(getLogFilePath("import", DATE_TIME))).toBe( + "import-2026-01-01T12-00-00.log", ); }); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.ts b/packages/cli-core/src/commands/migrate/lib/logger.ts index 72cd98372..54d0d54a3 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.ts @@ -28,6 +28,20 @@ export function getLogDir(): string { return path.join(process.cwd(), "logs"); } +/** + * The log directory the way the user would type it from here. + * + * Relative (`./logs`) when it sits under the current directory, absolute when + * it does not — a path the reader can paste either way, without a home + * directory's worth of prefix on the common case. + */ +export function displayLogDir(): string { + const dir = getLogDir(); + const relative = path.relative(process.cwd(), dir); + if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) return dir; + return `.${path.sep}${relative}`; +} + /** Absolute path of the log file a run with this timestamp writes to. */ export function getLogFilePath(logFile: string, dateTime: string): string { // Colons are illegal in Windows filenames, and the timestamp is an ISO string. @@ -59,13 +73,13 @@ export function errorLogger(payload: ErrorPayload, dateTime: string): void { status: payload.status, error: err.longMessage ?? err.message, }; - appendToLogFile(getLogFilePath("migration", dateTime), entry); + appendToLogFile(getLogFilePath("import", dateTime), entry); } } /** Writes a user that failed schema validation before any API call. */ export function validationLogger(payload: ValidationErrorPayload, dateTime: string): void { - appendToLogFile(getLogFilePath("migration", dateTime), { + appendToLogFile(getLogFilePath("import", dateTime), { userId: payload.userId, status: "fail" as const, error: payload.error, @@ -76,25 +90,25 @@ export function validationLogger(payload: ValidationErrorPayload, dateTime: stri /** Writes the outcome of one import attempt. */ export function importLogger(entry: ImportLogEntry, dateTime: string): void { - appendToLogFile(getLogFilePath("migration", dateTime), entry); + appendToLogFile(getLogFilePath("import", dateTime), entry); } /** * Writes the outcome of one deletion attempt. * - * A separate `user-deletion-` file rather than another line in the migration - * log: undoing a migration is its own run, and mixing the two would make - * "what did this import do" unanswerable after an undo. + * A separate `delete-` file rather than another line in the import log: undoing + * a migration is its own run, and mixing the two would make "what did this + * import do" unanswerable after an undo. */ export function deleteLogger(entry: DeleteLogEntry, dateTime: string): void { - appendToLogFile(getLogFilePath("user-deletion", dateTime), entry); + appendToLogFile(getLogFilePath("delete", dateTime), entry); } /** * Writes the outcome of exporting one user. * - * Its own `export-` file for the same reason deletions get theirs: an export - * is a distinct run, and `migrate logs list` reports each kind separately. + * Its own `export-` file for the same reason deletes get theirs: an export is a + * distinct run, and `migrate logs list` reports each kind separately. */ export function exportLogger(entry: ExportLogEntry, dateTime: string): void { appendToLogFile(getLogFilePath("export", dateTime), entry); @@ -109,6 +123,6 @@ export function deleteErrorLogger(payload: ErrorPayload, dateTime: string): void status: payload.status, error: err.longMessage ?? err.message, }; - appendToLogFile(getLogFilePath("user-deletion", dateTime), entry); + appendToLogFile(getLogFilePath("delete", dateTime), entry); } } diff --git a/packages/cli-core/src/commands/migrate/logs/list.ts b/packages/cli-core/src/commands/migrate/logs/list.ts index 1ef7f9405..ae0f53928 100644 --- a/packages/cli-core/src/commands/migrate/logs/list.ts +++ b/packages/cli-core/src/commands/migrate/logs/list.ts @@ -6,11 +6,20 @@ * an agent a read-only way to inspect a migration without parsing NDJSON. */ -import { cyan, dim } from "../../../lib/color.ts"; +import { bold, cyan, dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { withGutter } from "../../../lib/spinner.ts"; -import { formatSize, listLogFiles, type LogFile } from "../lib/log-files.ts"; -import { getLogDir } from "../lib/logger.ts"; +import { formatSize, listLogFiles, type LogFile, type LogKind } from "../lib/log-files.ts"; +import { displayLogDir } from "../lib/logger.ts"; + +/** Every kind a log file can be, and what one entry in it records. */ +const KIND_LEGEND: Record, string> = { + export: "One entry per user pulled from the source platform.", + import: "One entry per user created in Clerk, with any error.", + delete: "One entry per user removed from Clerk, with any error.", +}; + +const legendWidth = Math.max(...Object.keys(KIND_LEGEND).map((kind) => kind.length)) + 2; export type LogsListOptions = { json?: boolean; @@ -27,6 +36,22 @@ function toJson(files: LogFile[]) { })); } +/** + * The filename stamp as a date a human reads at a glance. + * + * The stamp is UTC (`getDateTimeStamp` is an ISO string with its colons swapped + * for filename-legal dashes), so it is parsed as UTC and rendered in the + * viewer's own zone — "which run was that" is a question about local time. + * Returns the raw stamp for anything unparseable rather than printing + * "Invalid Date". + */ +export function formatTimestamp(stamp: string): string { + if (!stamp) return ""; + const date = new Date(`${stamp.replace(/T(\d{2})-(\d{2})-(\d{2})$/, "T$1:$2:$3")}Z`); + if (Number.isNaN(date.getTime())) return stamp; + return date.toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short" }); +} + export async function list(options: LogsListOptions = {}): Promise { const files = listLogFiles(); @@ -37,32 +62,64 @@ export async function list(options: LogsListOptions = {}): Promise { await withGutter("Listing migration logs", async () => { if (files.length === 0) { - log.info(`No migration logs in ${getLogDir()}.`); + log.info(`No migration logs in ${displayLogDir()}.`); return; } - const kindWidth = Math.max(...files.map((file) => file.kind.length), "TYPE".length) + 2; - const timeWidth = - Math.max(...files.map((file) => file.timestamp.length), "TIMESTAMP".length) + 2; - const sizeWidth = Math.max(...files.map((file) => formatSize(file.sizeBytes).length), 4) + 2; + const rows = files.map((file) => ({ + name: file.name, + kind: file.kind, + when: formatTimestamp(file.timestamp), + size: formatSize(file.sizeBytes), + entries: String(file.entryCount), + })); + + const width = (header: string, pick: (row: (typeof rows)[number]) => string) => + Math.max(header.length, ...rows.map((row) => pick(row).length)) + 2; + + /** + * Pads to the visible width, then colours. Colouring first would count the + * ANSI escape bytes towards the width and pull every later column left. + */ + const column = (text: string, size: number, paint: (value: string) => string) => + paint(text) + " ".repeat(Math.max(0, size - text.length)); + + const nameWidth = width("FILE", (row) => row.name); + const kindWidth = width("TYPE", (row) => row.kind); + const whenWidth = width("DATE", (row) => row.when); + const sizeWidth = width("SIZE", (row) => row.size); + + log.info("Each log represents a user export, user import, or a user delete run."); + log.info("Each log consists of a single NDJSON entry per user."); + log.blank(); log.info( - dim("TYPE".padEnd(kindWidth)) + - dim("TIMESTAMP".padEnd(timeWidth)) + + dim("FILE".padEnd(nameWidth)) + + dim("TYPE".padEnd(kindWidth)) + + dim("DATE".padEnd(whenWidth)) + dim("SIZE".padEnd(sizeWidth)) + dim("ENTRIES"), ); - for (const file of files) { + for (const row of rows) { log.info( - cyan(file.kind.padEnd(kindWidth)) + - (file.timestamp || dim("—")).padEnd(timeWidth) + - dim(formatSize(file.sizeBytes).padEnd(sizeWidth)) + - String(file.entryCount), + column(row.name, nameWidth, cyan) + + row.kind.padEnd(kindWidth) + + column(row.when || "—", whenWidth, row.when ? (value) => value : dim) + + column(row.size, sizeWidth, dim) + + row.entries, ); } - log.info(""); - log.info(dim(`${files.length} log file${files.length === 1 ? "" : "s"} in ${getLogDir()}`)); + log.blank(); + log.info(`${files.length} log file${files.length === 1 ? "" : "s"} in ${displayLogDir()}`); + log.blank(); + + // A listing only shows the kinds that happen to be present, so the legend + // is fixed: it also answers "what else could be here". + log.info(bold("Log types:")); + for (const [kind, description] of Object.entries(KIND_LEGEND)) { + log.info(` ${cyan(bold(kind))}${" ".repeat(legendWidth - kind.length)}${description}`); + } }); } diff --git a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts index 4a7cf9eb1..72da1cffc 100644 --- a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts @@ -44,8 +44,8 @@ const captured = useCaptureLog(); let workDir: string; let originalCwd: string; -const MIGRATION = "migration-2026-01-01T12-00-00.log"; -const DELETION = "user-deletion-2026-02-01T12-00-00.log"; +const IMPORT = "import-2026-01-01T12-00-00.log"; +const DELETE = "delete-2026-02-01T12-00-00.log"; beforeAll(() => { originalMode = getMode(); @@ -80,8 +80,8 @@ function writeLog(name: string, entries: unknown[]): void { describe("logs clean", () => { test("prompts before deleting anything", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); await clean(); @@ -93,7 +93,7 @@ describe("logs clean", () => { // Deleting on a stray enter would be the wrong default for a destructive // command sitting next to `clerk migrate delete`. test("defaults the prompt to no", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await clean(); @@ -101,16 +101,16 @@ describe("logs clean", () => { }); test("declining leaves every file in place", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); mockConfirm.mockResolvedValue(false); await expect(clean()).rejects.toThrow(UserAbortError); - expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); }); test("-y skips the prompt entirely", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await clean({ yes: true }); @@ -126,40 +126,40 @@ describe("logs clean", () => { describe("logs convert", () => { test("offers a multiselect when given neither files nor --all", async () => { - writeLog(MIGRATION, [{ a: 1 }, { b: 2 }]); - writeLog(DELETION, [{ a: 1 }]); - mockMultiselect.mockResolvedValue([MIGRATION]); + writeLog(IMPORT, [{ a: 1 }, { b: 2 }]); + writeLog(DELETE, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([IMPORT]); await convert(); const options = mockMultiselect.mock.calls[0]?.[0]?.options; - expect(options?.map((option) => option.value)).toEqual([DELETION, MIGRATION]); + expect(options?.map((option) => option.value)).toEqual([DELETE, IMPORT]); expect(options?.[1]?.hint).toBe("2 entries"); }); test("converts only what was selected", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ a: 1 }]); - mockMultiselect.mockResolvedValue([MIGRATION]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([IMPORT]); await convert(); expect(fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json"))).toEqual([ - "migration-2026-01-01T12-00-00.json", + "import-2026-01-01T12-00-00.json", ]); }); test("selecting nothing aborts without writing", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); mockMultiselect.mockResolvedValue([]); await expect(convert()).rejects.toThrow(UserAbortError); - expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); }); test("does not prompt when --all was passed", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await convert({ all: true }); @@ -168,9 +168,9 @@ describe("logs convert", () => { }); test("does not prompt when files were named", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); - await convert({ files: [MIGRATION] }); + await convert({ files: [IMPORT] }); expect(mockMultiselect).not.toHaveBeenCalled(); }); @@ -180,7 +180,7 @@ describe("logs convert", () => { // with `└ Failed`. Declining a prompt is not a failure, so the two must not swap. describe("cancelling inside the gutter", () => { test("declining the logs clean confirm closes with Paused, not Failed", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); mockConfirm.mockResolvedValue(false); await expect(clean()).rejects.toThrow(UserAbortError); @@ -190,7 +190,7 @@ describe("cancelling inside the gutter", () => { }); test("selecting nothing in the logs convert multiselect closes with Paused", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); mockMultiselect.mockResolvedValue([]); await expect(convert()).rejects.toThrow(UserAbortError); diff --git a/packages/cli-core/src/commands/migrate/logs/logs.test.ts b/packages/cli-core/src/commands/migrate/logs/logs.test.ts index d0db18fb6..2e80ce5d5 100644 --- a/packages/cli-core/src/commands/migrate/logs/logs.test.ts +++ b/packages/cli-core/src/commands/migrate/logs/logs.test.ts @@ -8,10 +8,13 @@ import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { clean } from "./clean.ts"; import { convert } from "./convert.ts"; -import { list } from "./list.ts"; +import { formatTimestamp, list } from "./list.ts"; const captured = useCaptureLog(); +const ANSI_ESCAPE_PATTERN = new RegExp(String.raw`\u001b\[[0-9;]*m`, "g"); +const stripAnsi = (value: string) => value.replace(ANSI_ESCAPE_PATTERN, ""); + let workDir: string; let originalCwd: string; @@ -39,8 +42,8 @@ function writeLog(name: string, entries: unknown[]): void { ); } -const MIGRATION = "migration-2026-01-01T12-00-00.log"; -const DELETION = "user-deletion-2026-02-01T12-00-00.log"; +const IMPORT = "import-2026-01-01T12-00-00.log"; +const DELETE = "delete-2026-02-01T12-00-00.log"; describe("logs list", () => { test("says so plainly when there is no logs directory", async () => { @@ -54,42 +57,66 @@ describe("logs list", () => { expect(captured.err).toContain("No migration logs in"); }); - test("reports type, timestamp, size and entry count", async () => { - writeLog(MIGRATION, [{ userId: "u1" }, { userId: "u2" }, { userId: "u3" }]); + test("reports file, type, date, size and entry count", async () => { + writeLog(IMPORT, [{ userId: "u1" }, { userId: "u2" }, { userId: "u3" }]); await list(); + expect(captured.err).toContain("FILE"); expect(captured.err).toContain("TYPE"); - expect(captured.err).toContain("TIMESTAMP"); + expect(captured.err).toContain("DATE"); expect(captured.err).toContain("SIZE"); expect(captured.err).toContain("ENTRIES"); - expect(captured.err).toContain("migration"); - expect(captured.err).toContain("2026-01-01T12-00-00"); + expect(captured.err).toContain(IMPORT); + expect(captured.err).toContain("import"); + expect(captured.err).toContain(formatTimestamp("2026-01-01T12-00-00")); expect(captured.err).toMatch(/\bB\b/); expect(captured.err).toContain("3"); }); + // The filename stamp is for sorting and for `logs convert`; the column a + // human scans should read like a date. + test("shows the date rendered, not the raw filename stamp", async () => { + writeLog(IMPORT, [{ userId: "u1" }]); + + await list(); + + const dateColumn = stripAnsi(captured.err) + .split("\n") + .find((line) => line.includes(IMPORT)); + expect(dateColumn?.replace(IMPORT, "")).not.toContain("2026-01-01T12-00-00"); + }); + + test("reports the log directory relative to the current directory", async () => { + writeLog(IMPORT, [{ userId: "u1" }]); + + await list(); + + expect(stripAnsi(captured.err)).toContain(`1 log file in .${path.sep}logs`); + expect(captured.err).not.toContain(getLogDir()); + }); + test("lists every log kind", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); await list(); - expect(captured.err).toContain("migration"); - expect(captured.err).toContain("deletion"); + expect(captured.err).toContain("import"); + expect(captured.err).toContain("delete"); expect(captured.err).toContain("2 log files"); }); test("--json emits a machine-readable listing on stdout", async () => { - writeLog(MIGRATION, [{ userId: "u1" }]); + writeLog(IMPORT, [{ userId: "u1" }]); await list({ json: true }); const parsed = JSON.parse(captured.out) as Record[]; expect(parsed).toHaveLength(1); expect(parsed[0]).toMatchObject({ - name: MIGRATION, - kind: "migration", + name: IMPORT, + kind: "import", timestamp: "2026-01-01T12-00-00", entry_count: 1, }); @@ -109,22 +136,22 @@ describe("logs clean", () => { // Tests run non-TTY, which is the same signal an agent gives. test("refuses without -y when it cannot prompt, and explains", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await expect(clean()).rejects.toThrow(/cannot prompt here.*Pass -y/s); - expect(fs.existsSync(path.join(getLogDir(), MIGRATION))).toBe(true); + expect(fs.existsSync(path.join(getLogDir(), IMPORT))).toBe(true); }); test("names how many files are at stake when it refuses", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); await expect(clean()).rejects.toThrow(/2 log files/); }); test("-y deletes the log files and reports the count", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); await clean({ yes: true }); @@ -133,12 +160,12 @@ describe("logs clean", () => { }); test("leaves converted JSON output alone", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - fs.writeFileSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"), "[]"); + writeLog(IMPORT, [{ a: 1 }]); + fs.writeFileSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"), "[]"); await clean({ yes: true }); - expect(fs.readdirSync(getLogDir())).toEqual(["migration-2026-01-01T12-00-00.json"]); + expect(fs.readdirSync(getLogDir())).toEqual(["import-2026-01-01T12-00-00.json"]); }); }); @@ -149,42 +176,42 @@ describe("logs convert", () => { }); test("writes a JSON array alongside the original, leaving it intact", async () => { - writeLog(MIGRATION, [{ userId: "u1" }, { userId: "u2" }]); + writeLog(IMPORT, [{ userId: "u1" }, { userId: "u2" }]); - await convert({ files: [MIGRATION] }); + await convert({ files: [IMPORT] }); - const output = path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"); + const output = path.join(getLogDir(), "import-2026-01-01T12-00-00.json"); expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([ { userId: "u1" }, { userId: "u2" }, ]); - expect(fs.existsSync(path.join(getLogDir(), MIGRATION))).toBe(true); + expect(fs.existsSync(path.join(getLogDir(), IMPORT))).toBe(true); expect(captured.err).toContain("Originals left in place"); }); test("--all converts every log file", async () => { - writeLog(MIGRATION, [{ a: 1 }]); - writeLog(DELETION, [{ b: 2 }]); + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ b: 2 }]); await convert({ all: true }); const written = fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json")); expect(written.sort()).toEqual([ - "migration-2026-01-01T12-00-00.json", - "user-deletion-2026-02-01T12-00-00.json", + "delete-2026-02-01T12-00-00.json", + "import-2026-01-01T12-00-00.json", ]); }); test("accepts a path and resolves it against ./logs/", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); - await convert({ files: [`./logs/${MIGRATION}`] }); + await convert({ files: [`./logs/${IMPORT}`] }); - expect(fs.existsSync(path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"))).toBe(true); + expect(fs.existsSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"))).toBe(true); }); test("fails clearly on a file that is not there", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await expect(convert({ files: ["migration-nope.log"] })).rejects.toThrow(CliError); }); @@ -192,26 +219,26 @@ describe("logs convert", () => { // Silently dropping the line would leave a JSON array that looks complete. test("reports a malformed line by number and converts the rest", async () => { fs.mkdirSync(getLogDir(), { recursive: true }); - fs.writeFileSync(path.join(getLogDir(), MIGRATION), '{"a":1}\n{"b":\n{"c":3}\n'); + fs.writeFileSync(path.join(getLogDir(), IMPORT), '{"a":1}\n{"b":\n{"c":3}\n'); - await convert({ files: [MIGRATION] }); + await convert({ files: [IMPORT] }); - expect(captured.err).toContain(`${MIGRATION}:2`); + expect(captured.err).toContain(`${IMPORT}:2`); expect(captured.err).toContain("1 malformed line skipped"); - const output = path.join(getLogDir(), "migration-2026-01-01T12-00-00.json"); + const output = path.join(getLogDir(), "import-2026-01-01T12-00-00.json"); expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([{ a: 1 }, { c: 3 }]); }); test("refuses without a target when it cannot prompt, naming the alternatives", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await expect(convert()).rejects.toThrow(/cannot prompt here/); - expect(fs.readdirSync(getLogDir())).toEqual([MIGRATION]); + expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); }); test("reports the entry count per converted file", async () => { - writeLog(MIGRATION, [{ a: 1 }, { b: 2 }, { c: 3 }]); + writeLog(IMPORT, [{ a: 1 }, { b: 2 }, { c: 3 }]); await convert({ all: true }); @@ -232,7 +259,7 @@ describe("human-mode frame", () => { }); test("logs list wraps its output in an intro/outro gutter", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await list(); @@ -243,7 +270,7 @@ describe("human-mode frame", () => { }); test("--json stays outside the gutter, on stdout only", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await list({ json: true }); @@ -252,7 +279,7 @@ describe("human-mode frame", () => { }); test("a failure inside logs convert closes with Failed and still throws", async () => { - writeLog(MIGRATION, [{ a: 1 }]); + writeLog(IMPORT, [{ a: 1 }]); await expect(convert({ files: ["nope.log"] })).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 7fa811d46..049a553ca 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -144,7 +144,7 @@ describe("run", () => { const logs = fs.readdirSync(getLogDir()); expect(logs).toHaveLength(1); - expect(logs[0]).toMatch(/^migration-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + expect(logs[0]).toMatch(/^import-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); const entries = fs .readFileSync(path.join(getLogDir(), logs[0] as string), "utf-8") diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index e37cde385..ca0576022 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -484,7 +484,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const limits = resolveLimits(secretKey); const dateTime = getDateTimeStamp(); - const logFile = getLogFilePath("migration", dateTime); + const logFile = getLogFilePath("import", dateTime); const { users: loaded, validationFailed } = await withSpinner( `Loading users from ${file}...`, From b1fec90e5f400a86e789fd100bde706f60f69a92 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 8 Sep 2026 11:31:37 -0400 Subject: [PATCH 24/34] refactor: move the redaction placeholder into constants `[REDACTED]` was local to `lib/users.ts`; `clerk migrate settings` needs the same string so a withheld credential reads identically wherever the CLI declines to show one. --- packages/cli-core/src/lib/constants.ts | 12 ++++++++++++ packages/cli-core/src/lib/users.ts | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/lib/constants.ts b/packages/cli-core/src/lib/constants.ts index f88439df2..6505ab035 100644 --- a/packages/cli-core/src/lib/constants.ts +++ b/packages/cli-core/src/lib/constants.ts @@ -55,3 +55,15 @@ export const NPM_REGISTRY_URL = "https://registry.npmjs.org/"; /** Event ingestion endpoint (telemetry-service worker → BigQuery). */ export const DEFAULT_TELEMETRY_ENDPOINT = "https://clerk-telemetry.com/v1/event"; export const TELEMETRY_TIMEOUT_MS = 1000; + +// ── Redaction ───────────────────────────────────────────────────────────── + +/** + * What a withheld secret displays as, everywhere the CLI shows one. + * + * Square brackets rather than a mask or a truncation: a row of dots or a + * head-and-tail (`aVer…3456`) reads as a value, and the reader has to work out + * that it is not one. Used by `clerk users create --dry-run` and by + * `clerk migrate settings`. + */ +export const REDACTED = "[REDACTED]"; diff --git a/packages/cli-core/src/lib/users.ts b/packages/cli-core/src/lib/users.ts index d43a767e6..5d9345f18 100644 --- a/packages/cli-core/src/lib/users.ts +++ b/packages/cli-core/src/lib/users.ts @@ -1,8 +1,8 @@ import { bapiRequest } from "./bapi.ts"; +import { REDACTED } from "./constants.ts"; import { ERROR_CODE, throwUsageError } from "./errors.ts"; const USERS_INVALID_JSON_MESSAGE = "User payload must be a JSON object."; -const REDACTED = "[REDACTED]"; const DIRECT_REDACT_KEYS = new Set(["password", "code"]); const OBJECT_REDACT_KEYS = new Set(["private_metadata", "unsafe_metadata"]); From f9c0b0826b6e0e46a4e980e615d7d394c4b663d7 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 8 Sep 2026 11:31:50 -0400 Subject: [PATCH 25/34] feat(migrate): accept Firebase's own variable names and restyle settings list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Firebase hands its scrypt parameters over as `base64_signer_key`, `rounds` and friends, and every guide — Clerk's own standalone script included — tells you to paste them into `.env` under those names. Those spellings, and their `FIREBASE_` prefixed forms, now resolve as aliases behind the `CLERK_FIREBASE_*` variables, read from one registry shared by the listing and the import. `migrate settings list` gains the orientation lines, count and next-steps block the CLI's other listings carry, attributes an environment value to the env file it actually came from (Bun loads `.env.local` before the CLI runs, so "`ROUNDS` env var" named nothing the reader could edit), names the alias alongside the file, and withholds credentials as `[REDACTED]` rather than a head-and-tail truncation. --- .../cli-core/src/commands/migrate/README.md | 95 ++++++++++++++----- .../src/commands/migrate/lib/env-file.test.ts | 54 ++++++++++- .../src/commands/migrate/lib/env-file.ts | 43 ++++++++- .../migrate/lib/firebase-hash.test.ts | 39 ++++++++ .../src/commands/migrate/lib/firebase-hash.ts | 23 +++-- .../src/commands/migrate/readme.test.ts | 13 ++- .../src/commands/migrate/settings/list.ts | 47 +++++++-- .../src/commands/migrate/settings/registry.ts | 45 +++++++-- .../migrate/settings/settings.test.ts | 62 ++++++++++-- packages/cli-core/src/lib/dotenv.ts | 6 +- packages/cli-core/src/lib/next-steps.ts | 4 + 11 files changed, 363 insertions(+), 68 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index bd601ca00..018cd76d6 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -540,36 +540,65 @@ transformer maps to `userId`. ### `clerk migrate settings` What a run in this directory would pick up, and where each value comes from. +Listing is the default, because it is the read-only one: a bare `clerk migrate +settings` shows, never changes. ```sh clerk migrate settings # list +clerk migrate settings list --json clerk migrate settings set transformer firebase -clerk migrate settings set firebase-signer-key abc123… +clerk migrate settings set firebase-signer-key abc123 clerk migrate settings clear -y ``` -``` -SETTING VALUE SOURCE DESCRIPTION -transformer firebase clerk config Source platform the export came from -file users.json clerk config Export file to import users from -firebase-signer-key aVer…3456 .env.clerk-migrate Firebase base64 signer key -firebase-rounds — not set Firebase scrypt rounds -``` +| Subcommand | Takes | Description | +| ----------------------------- | ---------------- | --------------------------------------------------------- | +| `settings list` | `--json` | Every setting, its value and the source it resolved from | +| `settings set ` | ` ` | Change one setting | +| `settings clear` | `-y, --yes` | Forget this project's settings and delete its credentials | -Setting names are kebab-case and identical to the `clerk migrate import` flag each one -backs, so `firebase-signer-key` here is `--firebase-signer-key` there rather -than a second spelling to learn. The description column carries the prose. +Setting names are kebab-case and identical to the `clerk migrate import` flag +each one backs, so `firebase-signer-key` here is `--firebase-signer-key` there +rather than a second spelling to learn. The description column carries the +prose. The source column is the point. A migration reads from flags, the environment, two of the app's env files and the CLI's config, so when a run picks up a stale -value the question is never "what is it" but "which of those won". +value the question is never "what is it" but "which of those won". A value that +arrived under one of the accepted aliases names the variable alongside the file. + +It names a **file** wherever there is one to name. Bun loads `.env`/`.env.local` +into the environment before the CLI runs, so a value a developer typed into +`.env.local` would otherwise be reported as "`ROUNDS` env var" — true, and no +help to someone asking which file to edit. Attribution is by value: a file +holding the same key with a _different_ value lost to something exported in the +shell, and that row keeps saying `ROUNDS env var`, because that is exactly the +case this column exists to catch. + +A setting with no value leaves the column empty rather than filling it with a +placeholder — the source column already reads `not set` on that row, and the +blank is what makes the settings that do have a value stand out. + +It closes on next steps naming the two commands that change what it just +showed — the same block `clerk mcp list` and `clerk whoami` end on, and human +only. The full command surface stays in `--help`. -| Command | Description | -| ----------------------------- | ----------------------------------------------------- | -| `settings` / `settings list` | Show every setting, its value and its source | -| `settings list --json` | The same, machine-readable | -| `settings set ` | Change one setting | -| `settings clear [-y]` | Forget this project's settings and delete its secrets | +``` +A migration run in this directory picks these up unless a flag overrides them. +Each setting is named after the `clerk migrate import` flag it stands in for. + +SETTING VALUE SOURCE DESCRIPTION +transformer firebase clerk config Source platform the export came from +file users.json clerk config Export file to import users from +firebase-signer-key [REDACTED] .env.clerk-migrate Firebase base64 signer key +firebase-rounds 8 .env.local (ROUNDS) Firebase scrypt rounds +firebase-mem-cost 14 MEM_COST env var Firebase scrypt memory cost + +4 of 7 settings set. Credentials are shown redacted. + + → Run `clerk migrate settings set ` to change one + → Run `clerk migrate settings clear` to forget them all, credentials included +``` #### Where each setting is kept @@ -586,8 +615,11 @@ being migrated and does not belong in the file its developers read daily. The CLI adds it to `.gitignore` the first time it writes it, and deletes it when `settings clear` removes the last value. -Credentials are redacted wherever they are displayed, including under `--json`, -so the output is safe to paste into an issue. +Credentials are withheld wherever they are displayed, including under `--json`, +so the output is safe to paste into an issue. They display as `[REDACTED]` — +the same thing `clerk users create --dry-run` prints for a password — rather +than a truncation like `aVer…3456`: the source column already says which value +is in play, and a partial secret is one the reader has to recognise as partial. ### Custom transformers (`--transformer-file`) @@ -681,12 +713,23 @@ that file is not a secret store. To avoid re-passing all four on every run, set them once with [`clerk migrate settings`](#clerk-migrate-settings), or export them yourself: -| Variable | Flag | -| ------------------------------- | --------------------------- | -| `CLERK_FIREBASE_SIGNER_KEY` | `--firebase-signer-key` | -| `CLERK_FIREBASE_SALT_SEPARATOR` | `--firebase-salt-separator` | -| `CLERK_FIREBASE_ROUNDS` | `--firebase-rounds` | -| `CLERK_FIREBASE_MEM_COST` | `--firebase-mem-cost` | +| Flag | Variable | Also accepted | +| --------------------------- | ------------------------------- | --------------------------------------------------------- | +| `--firebase-signer-key` | `CLERK_FIREBASE_SIGNER_KEY` | `FIREBASE_BASE64_SIGNER_KEY`, `BASE64_SIGNER_KEY` | +| `--firebase-salt-separator` | `CLERK_FIREBASE_SALT_SEPARATOR` | `FIREBASE_BASE64_SALT_SEPARATOR`, `BASE64_SALT_SEPARATOR` | +| `--firebase-rounds` | `CLERK_FIREBASE_ROUNDS` | `FIREBASE_ROUNDS`, `ROUNDS` | +| `--firebase-mem-cost` | `CLERK_FIREBASE_MEM_COST` | `FIREBASE_MEM_COST`, `MEM_COST` | + +The unprefixed names are what Firebase itself calls these (`base64_signer_key`, +`rounds`) and what every guide, Clerk's own standalone migration script +included, tells you to paste into `.env`. Someone who followed one has the +values the import needs, spelled the way the source platform spells them, so +they are read rather than reported as missing. + +They are a fallback, not a synonym: a `CLERK_FIREBASE_*` variable wins wherever +both exist, and `clerk migrate settings` names the variable it read alongside +the file — `ROUNDS` is generic enough to mean something else in an app that was +never a Firebase project, and that should be visible rather than silent. Resolution order is flag, then exported variable, then `.env.clerk-migrate`, then the app's `.env.local`/`.env`. The sources can be mixed as long as all four diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.test.ts b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts index 7c190a9d7..855cc9274 100644 --- a/packages/cli-core/src/commands/migrate/lib/env-file.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts @@ -71,7 +71,11 @@ describe("findMigrateEnvValue", () => { await writeMigrateEnvValues({ CLERK_FIREBASE_SIGNER_KEY: "from-file" }, workDir); const located = await findMigrateEnvValue(["CLERK_FIREBASE_SIGNER_KEY"], workDir, {}); - expect(located).toEqual({ value: "from-file", source: MIGRATE_ENV_FILE }); + expect(located).toEqual({ + value: "from-file", + name: "CLERK_FIREBASE_SIGNER_KEY", + source: MIGRATE_ENV_FILE, + }); }); test("beats the app's own .env.local", async () => { @@ -89,12 +93,58 @@ describe("findMigrateEnvValue", () => { const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { CLERK_FIREBASE_ROUNDS: "99", }); - expect(located).toEqual({ value: "99", source: "CLERK_FIREBASE_ROUNDS env var" }); + expect(located).toEqual({ + value: "99", + name: "CLERK_FIREBASE_ROUNDS", + source: "CLERK_FIREBASE_ROUNDS env var", + }); }); test("returns nothing when the setting is absent everywhere", async () => { expect(await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, {})).toBeUndefined(); }); + + // Bun loads `.env.local` into process.env before the CLI runs, so a value a + // developer put in a file arrives looking like an exported variable. Naming + // the variable answers nothing — the question is which file to edit. + describe("attributing an environment value to the file it came from", () => { + test("names the file when it holds the same value", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "8", + }); + expect(located?.source).toBe(".env.local"); + }); + + // The one case the source column exists for: the file lost, so naming it + // would point at the value that is not being used. + test("keeps the variable when the file holds a different value", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "99", + }); + expect(located?.source).toBe("CLERK_FIREBASE_ROUNDS env var"); + }); + + test("prefers the file the runtime would have loaded last", async () => { + fs.writeFileSync(path.join(workDir, ".env"), "CLERK_FIREBASE_ROUNDS=8\n"); + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "8", + }); + expect(located?.source).toBe(".env.local"); + }); + + test("keeps the variable when no file holds it at all", async () => { + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "8", + }); + expect(located?.source).toBe("CLERK_FIREBASE_ROUNDS env var"); + }); + }); }); describe("clearMigrateEnvValues", () => { diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.ts b/packages/cli-core/src/commands/migrate/lib/env-file.ts index 4d7a61f0a..2c15c9163 100644 --- a/packages/cli-core/src/commands/migrate/lib/env-file.ts +++ b/packages/cli-core/src/commands/migrate/lib/env-file.ts @@ -31,6 +31,36 @@ export const MIGRATE_ENV_FILE = ".env.clerk-migrate"; /** Lowest priority first: the migration's own file overrides the app's. */ const MIGRATE_ENV_FILES = [".env", ".env.local", MIGRATE_ENV_FILE] as const; +/** + * The project env file a value in the environment actually came from, if any. + * + * Bun loads `.env`, `.env.local` and friends into `process.env` before the CLI + * runs, so a variable a developer wrote into `.env.local` reaches + * {@link findEnvValue} as an environment variable and gets reported as one. + * That is true but useless: "`ROUNDS` env var" does not tell anyone which of + * their files to edit. + * + * Attribution is by value, not by presence. A file that holds the same key with + * a *different* value lost to something exported in the shell, and saying + * `.env.local` there would name the file that is not winning — the one case + * this column exists to catch. Highest-priority file first, matching the order + * the runtime loaded them in. + */ +async function fileHolding( + cwd: string, + { name, value }: LocatedEnvValue, +): Promise { + for (const envFile of [...MIGRATE_ENV_FILES].reverse()) { + const file = Bun.file(join(cwd, envFile)); + if (!(await file.exists())) continue; + + for (const line of parseEnvFile(await file.text())) { + if (line.type === "entry" && line.key === name && line.value === value) return envFile; + } + } + return undefined; +} + /** Resolves a migration setting: environment first, then the project's env files. */ export async function findMigrateEnvValue( names: string[], @@ -38,8 +68,17 @@ export async function findMigrateEnvValue( env: Record = process.env, ): Promise { const located = await findEnvValue(cwd, names, { env, files: MIGRATE_ENV_FILES }); - if (located) log.debug(`migrate: ${names[0]} from ${located.source}`); - return located; + if (!located) return undefined; + + // `findEnvValue` reports the environment before it reads a file, so a value + // the runtime loaded out of `.env.local` is credited to the variable rather + // than to the file the user would edit. Put the file back. + const source = located.source.endsWith(" env var") + ? ((await fileHolding(cwd, located)) ?? located.source) + : located.source; + + log.debug(`migrate: ${names[0]} from ${source}`); + return { ...located, source }; } /** diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts index 45c65c452..ec5d89fb2 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts @@ -170,3 +170,42 @@ describe("on a firebase run", () => { expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); }); }); + +// Firebase names these `base64_signer_key`, `rounds` and friends, and that is +// how every guide — Clerk's own standalone script included — tells you to write +// them into `.env`. A project that followed one has the values already. +describe("the names Firebase itself uses", () => { + const writeEnvLocal = (contents: string) => + fs.writeFileSync(path.join(workDir, ".env.local"), contents); + + test("reads a set written under the unprefixed names", async () => { + writeEnvLocal("BASE64_SIGNER_KEY=SIGNER\nBASE64_SALT_SEPARATOR=Bw==\nROUNDS=8\nMEM_COST=14\n"); + + expect(await resolveFirebaseHashConfig({}, "firebase")).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + test("reads a set written under the FIREBASE_ prefix", async () => { + writeEnvLocal( + "FIREBASE_BASE64_SIGNER_KEY=SIGNER\nFIREBASE_BASE64_SALT_SEPARATOR=Bw==\n" + + "FIREBASE_ROUNDS=8\nFIREBASE_MEM_COST=14\n", + ); + + expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); + }); + + // The alias is a fallback, not a synonym: `ROUNDS` in an app's own env file + // is not necessarily about Firebase at all. + test("prefers the prefixed variable in the same file", async () => { + writeEnvLocal( + "ROUNDS=99\nCLERK_FIREBASE_ROUNDS=8\nBASE64_SIGNER_KEY=SIGNER\n" + + "BASE64_SALT_SEPARATOR=Bw==\nMEM_COST=14\n", + ); + + expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts index 10b06653d..d32fc3c58 100644 --- a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts @@ -18,15 +18,16 @@ import { throwUsageError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; +import { envNames, findSetting } from "../settings/registry.ts"; import { findMigrateEnvValue } from "./env-file.ts"; import type { FirebaseHashConfig } from "../types.ts"; -/** The `--firebase-*` flags, and the variable each falls back to. */ +/** The `--firebase-*` flags, and the setting each falls back to. */ export const FIREBASE_FLAGS = [ - ["firebaseSignerKey", "--firebase-signer-key", "CLERK_FIREBASE_SIGNER_KEY"], - ["firebaseSaltSeparator", "--firebase-salt-separator", "CLERK_FIREBASE_SALT_SEPARATOR"], - ["firebaseRounds", "--firebase-rounds", "CLERK_FIREBASE_ROUNDS"], - ["firebaseMemCost", "--firebase-mem-cost", "CLERK_FIREBASE_MEM_COST"], + ["firebaseSignerKey", "--firebase-signer-key", "firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator", "firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds", "firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost", "firebase-mem-cost"], ] as const; const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); @@ -39,18 +40,22 @@ export type FirebaseHashFlags = { }; /** - * Overlays the `CLERK_FIREBASE_*` values onto whichever flags were not passed. + * Overlays the saved environment values onto whichever flags were not passed. * - * Resolved through {@link findMigrateEnvValue}: the environment first, then + * The variables come from the settings registry — `CLERK_FIREBASE_*` and the + * unprefixed names Firebase itself uses — so `clerk migrate settings` and the + * import read exactly the same set. Resolved through + * {@link findMigrateEnvValue}: the environment first, then * `.env.clerk-migrate`, then the app's own `.env` files. The signer key is a * Firebase secret, so it is never written to the CLI's config — * `.env.clerk-migrate` is gitignored on creation. */ async function withFirebaseEnv(flags: FirebaseHashFlags): Promise { const merged: FirebaseHashFlags = { ...flags }; - for (const [key, , envVar] of FIREBASE_FLAGS) { + for (const [key, , settingName] of FIREBASE_FLAGS) { if (merged[key] !== undefined) continue; - const located = await findMigrateEnvValue([envVar]); + const setting = findSetting(settingName); + const located = setting && (await findMigrateEnvValue(envNames(setting))); if (!located || located.value.trim() === "") continue; // A non-numeric round count is left to fail the flag's own validation // rather than silently becoming NaN. diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts index be0bc5028..4887b0b64 100644 --- a/packages/cli-core/src/commands/migrate/readme.test.ts +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -32,9 +32,16 @@ function documentedCommands(markdown: string): string[] { // Line continuations first: the Firebase example spans three lines. for (const line of block.replace(/\\\n\s*/g, " ").split("\n")) { const start = line.indexOf("clerk migrate"); - // A command never contains a backtick or a `#`; the sample error output - // that quotes `clerk migrate` mid-sentence does. - if (start !== -1) found.add(line.slice(start).split(/[`#]/)[0]!.trim()); + // A command never contains a backtick, a `#`, or a run of two spaces; + // the sample error output that quotes `clerk migrate` mid-sentence does, + // and so does a pasted listing whose descriptions sit in a padded column. + if (start !== -1) + found.add( + line + .slice(start) + .split(/[`#]|\s{2,}/)[0]! + .trim(), + ); } } diff --git a/packages/cli-core/src/commands/migrate/settings/list.ts b/packages/cli-core/src/commands/migrate/settings/list.ts index f5e9e43ec..049264958 100644 --- a/packages/cli-core/src/commands/migrate/settings/list.ts +++ b/packages/cli-core/src/commands/migrate/settings/list.ts @@ -6,13 +6,20 @@ * environment, two of the app's env files and the CLI's config; when a run uses * a stale value, the question is never "what is it" but "which of those is * winning". Credentials are redacted, so this is safe to paste into an issue. + * + * Laid out like the CLI's other listings — `migrate logs list` and `migrate + * transformers list`: a line or two of orientation, the table, then a count. + * It closes with next steps, the way `mcp list` and `whoami` do, because a + * listing is where someone lands before they know what to type. Those are for + * humans; the full command surface stays in `--help`. */ import { cyan, dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; +import { NEXT_STEPS, printNextSteps } from "../../../lib/next-steps.ts"; import { findMigrateEnvValue } from "../lib/env-file.ts"; import { loadSettings } from "../lib/settings.ts"; -import { displayValue, SETTINGS, type SettingDef } from "./registry.ts"; +import { displayValue, envNames, SETTINGS, type SettingDef } from "./registry.ts"; export type SettingsListOptions = { json?: boolean; @@ -24,6 +31,21 @@ interface ResolvedSetting { source?: string; } +/** + * Names the variable as well as the file when an alias supplied the value. + * + * `.env.local` alone would be a half-answer for a setting that has four + * accepted spellings: the reader has to know *which* line in that file the run + * is reading before they can change it. An exported variable already carries + * its name in `source`. + */ +function describeSource(setting: SettingDef, located: { name: string; source: string }): string { + if (located.name === setting.envVar || located.source.startsWith(located.name)) { + return located.source; + } + return `${located.source} (${located.name})`; +} + async function resolveAll(): Promise { const saved = await loadSettings(); @@ -36,8 +58,10 @@ async function resolveAll(): Promise { : { setting, value: String(value), source: "clerk config" }; } - const located = await findMigrateEnvValue([setting.envVar as string]); - return located ? { setting, value: located.value, source: located.source } : { setting }; + const located = await findMigrateEnvValue(envNames(setting)); + return located + ? { setting, value: located.value, source: describeSource(setting, located) } + : { setting }; }), ); } @@ -73,10 +97,13 @@ export async function list(options: SettingsListOptions = {}): Promise { return; } + // An unset value leaves the column empty rather than filling it with a + // placeholder: the source column already reads "not set" on the same row, and + // an empty cell is what makes the settings that do have a value stand out. const cells = resolved.map(({ setting, value, source }) => ({ setting, name: setting.name, - value: value === undefined ? "—" : displayValue(setting, value), + value: value === undefined ? "" : displayValue(setting, value), unset: value === undefined, source: source ?? "not set", })); @@ -88,6 +115,10 @@ export async function list(options: SettingsListOptions = {}): Promise { const valueWidth = width("VALUE", (c) => c.value); const sourceWidth = width("SOURCE", (c) => c.source); + log.info("A migration run in this directory picks these up unless a flag overrides them."); + log.info("Each setting is named after the `clerk migrate import` flag it stands in for."); + log.blank(); + log.info( column("SETTING", nameWidth, dim) + column("VALUE", valueWidth, dim) + @@ -98,12 +129,16 @@ export async function list(options: SettingsListOptions = {}): Promise { for (const cell of cells) { log.info( column(cell.name, nameWidth, cyan) + - column(cell.value, valueWidth, cell.unset ? dim : (value) => value) + + column(cell.value, valueWidth, (value) => value) + column(cell.source, sourceWidth, dim) + dim(cell.setting.description), ); } + const set = cells.filter((cell) => !cell.unset).length; log.blank(); - log.info(dim("Credentials are shown redacted. `clerk migrate settings set `.")); + log.info(`${set} of ${cells.length} settings set. Credentials are shown redacted.`); + log.blank(); + + printNextSteps(NEXT_STEPS.MIGRATE_SETTINGS); } diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index 8ec0fe982..2b403da46 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -14,6 +14,8 @@ * this table rather than each keeping their own idea of what exists. */ +import { REDACTED } from "../../../lib/constants.ts"; + export type SettingStore = "config" | "env"; export interface SettingDef { @@ -31,6 +33,22 @@ export interface SettingDef { description: string; /** For `env` settings, the variable read at run time. */ envVar?: string; + /** + * Other variables accepted for the same setting, read only when + * {@link envVar} is absent. + * + * Firebase hands its four scrypt parameters over as `base64_signer_key`, + * `rounds` and friends, and every guide — including Clerk's own standalone + * migration script — tells the reader to paste them into `.env` under those + * names. Someone who did that has the values the CLI needs, spelled the way + * the source platform spells them, and a listing that reports "not set" is + * wrong about the project rather than strict about it. + * + * Prefixed names still win, and the listing names the variable it read, so a + * generic `ROUNDS` that means something else in the app is visible rather + * than silent. + */ + envAliases?: string[]; /** For `config` settings, the key on the saved migration entry. */ configKey?: "transformer" | "file" | "skipUnsupportedProviders"; /** Redact when displaying — the value is a credential. */ @@ -71,6 +89,7 @@ export const SETTINGS: SettingDef[] = [ name: "firebase-signer-key", store: "env", envVar: "CLERK_FIREBASE_SIGNER_KEY", + envAliases: ["FIREBASE_BASE64_SIGNER_KEY", "BASE64_SIGNER_KEY"], description: "Firebase base64 signer key", secret: true, }, @@ -78,12 +97,14 @@ export const SETTINGS: SettingDef[] = [ name: "firebase-salt-separator", store: "env", envVar: "CLERK_FIREBASE_SALT_SEPARATOR", + envAliases: ["FIREBASE_BASE64_SALT_SEPARATOR", "BASE64_SALT_SEPARATOR"], description: "Firebase base64 salt separator", }, { name: "firebase-rounds", store: "env", envVar: "CLERK_FIREBASE_ROUNDS", + envAliases: ["FIREBASE_ROUNDS", "ROUNDS"], description: "Firebase scrypt rounds", validate: positiveInteger, }, @@ -91,6 +112,7 @@ export const SETTINGS: SettingDef[] = [ name: "firebase-mem-cost", store: "env", envVar: "CLERK_FIREBASE_MEM_COST", + envAliases: ["FIREBASE_MEM_COST", "MEM_COST"], description: "Firebase scrypt memory cost", validate: positiveInteger, }, @@ -103,17 +125,24 @@ export function findSetting(name: string): SettingDef | undefined { } /** - * Shows enough of a credential to recognise it, never enough to use it. + * Every variable an `env` setting answers to, highest priority first. * - * Anything short enough that head-and-tail would leak most of it is masked - * whole: a 10-character key shown as `abcd…wxyz` has given away 8 of them. + * One list, read by both the listing and the run, so `clerk migrate settings` + * can never show a value the import would ignore. */ -export function redact(value: string): string { - if (value.length < 16) return "•".repeat(8); - return `${value.slice(0, 4)}…${value.slice(-4)}`; +export function envNames(setting: SettingDef): string[] { + return [setting.envVar as string, ...(setting.envAliases ?? [])]; } -/** The display value for a setting: redacted when it is a credential. */ +/** + * The display value for a setting: withheld entirely when it is a credential. + * + * {@link REDACTED} is what `clerk users create --dry-run` already prints for a + * password, so a credential reads the same wherever the CLI declines to show + * one. Head-and-tail (`aVer…3456`) would say *which* key is set, but the source + * column answers that, and a partial value is one the reader has to recognise + * as partial. + */ export function displayValue(setting: SettingDef, value: string): string { - return setting.secret ? redact(value) : value; + return setting.secret ? REDACTED : value; } diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index 42e4f7c1e..f27b84f8e 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -3,12 +3,13 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { _setConfigDir } from "../../../lib/config.ts"; +import { setMode } from "../../../mode.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { MIGRATE_ENV_FILE } from "../lib/env-file.ts"; import { loadSettings, saveSettings } from "../lib/settings.ts"; import { clear } from "./clear.ts"; import { list } from "./list.ts"; -import { redact } from "./registry.ts"; +import { displayValue, findSetting } from "./registry.ts"; import { set } from "./set.ts"; const captured = useCaptureLog(); @@ -44,14 +45,20 @@ afterEach(() => { process.exitCode = 0; }); -describe("redact", () => { - test("shows head and tail of a long value", () => { - expect(redact("aVeryLongSignerKeyValue123456")).toBe("aVer…3456"); - }); +describe("displayValue", () => { + const signerKey = findSetting("firebase-signer-key")!; + + // No part of the value, at any length — the same `[REDACTED]` that + // `clerk users create --dry-run` prints for a password. + test.each([["short"], ["0123456789"], ["aVeryLongSignerKeyValue123456"]])( + "withholds the credential %p entirely", + (value) => { + expect(displayValue(signerKey, value)).toBe("[REDACTED]"); + }, + ); - // Head-and-tail on a short value gives away most of it. - test.each([["short"], ["0123456789"], ["123456789012345"]])("masks %p whole", (value) => { - expect(redact(value)).toBe("••••••••"); + test("shows a setting that is not a credential", () => { + expect(displayValue(findSetting("transformer")!, "firebase")).toBe("firebase"); }); }); @@ -121,7 +128,7 @@ describe("list", () => { await list(); - expect(captured.err).toContain("aVer…3456"); + expect(captured.err).toContain("[REDACTED]"); expect(captured.err).not.toContain("aVeryLongSignerKeyValue123456"); expect(captured.err).toContain("firebase"); }); @@ -135,7 +142,7 @@ describe("list", () => { expect(captured.out).not.toContain("aVeryLongSignerKeyValue123456"); expect(JSON.parse(captured.out)).toContainEqual( - expect.objectContaining({ name: "firebase-signer-key", value: "aVer…3456", secret: true }), + expect.objectContaining({ name: "firebase-signer-key", value: "[REDACTED]", secret: true }), ); }); @@ -181,6 +188,41 @@ describe("list", () => { await list({ json: true }); expect(JSON.parse(captured.out).every((entry: { set: boolean }) => !entry.set)).toBe(true); }); + + // A listing is where someone lands before they know what to type, so it + // closes by naming the two commands that change what it just showed — + // the same next-steps block `mcp list` and `whoami` end on. + test("closes with next steps", async () => { + setMode("human"); + await list(); + setMode("agent"); + + expect(captured.err).toContain("clerk migrate settings set "); + expect(captured.err).toContain("clerk migrate settings clear"); + }); + + test("counts how many are set", async () => { + await set("transformer", "firebase"); + captured.clear(); + + await list(); + + expect(captured.err).toContain("1 of 7 settings set"); + }); + + // Firebase's own names for these, and what every guide tells you to paste + // into `.env`. Reporting "not set" for a value the import would read is the + // listing being wrong about the project rather than strict about it. + test("reads a credential written under the name Firebase uses", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "ROUNDS=8\n"); + captured.clear(); + + await list(); + + fs.rmSync(path.join(workDir, ".env.local")); + // Named alongside the file: `ROUNDS` may mean something else in this app. + expect(captured.err).toContain(".env.local (ROUNDS)"); + }); }); describe("clear", () => { diff --git a/packages/cli-core/src/lib/dotenv.ts b/packages/cli-core/src/lib/dotenv.ts index 16bc28187..a2e0c2ec7 100644 --- a/packages/cli-core/src/lib/dotenv.ts +++ b/packages/cli-core/src/lib/dotenv.ts @@ -48,6 +48,8 @@ export interface FindEnvValueOptions { export interface LocatedEnvValue { value: string; + /** Which of `names` supplied it — the caller may have passed several aliases. */ + name: string; /** Where it came from, for `--verbose` (`CLERK_SECRET_KEY env var`, `.env.local`). */ source: string; } @@ -70,7 +72,7 @@ export async function findEnvValue( for (const name of new Set(names)) { const value = env[name]; - if (value) return { value, source: `${name} env var` }; + if (value) return { value, name, source: `${name} env var` }; } // Priority is by name, not by position: the framework-specific name beats @@ -84,7 +86,7 @@ export async function findEnvValue( for (const line of parseEnvFile(await file.text())) { if (line.type !== "entry" || !line.value) continue; if (names.includes(line.key)) { - foundByName.set(line.key, { value: line.value, source: envFile }); + foundByName.set(line.key, { value: line.value, name: line.key, source: envFile }); } } } diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index a1b53e1d7..fbcf6fc64 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -76,6 +76,10 @@ export const NEXT_STEPS = { "Run `clerk migrate delete` to undo this migration", ], MIGRATE_DELETE: ["Run `clerk migrate logs list` to inspect the deletion log"], + MIGRATE_SETTINGS: [ + "Run `clerk migrate settings set ` to change one", + "Run `clerk migrate settings clear` to forget them all, credentials included", + ], // The only parameterized entry: a suggested import is worthless unless it // names the transformer that reads this export and the file just written. MIGRATE_EXPORT: (transformerKey: string, file: string) => [ From bf4d6ed24f8f797db1e1f5a2738e1155ca159d17 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 10 Sep 2026 23:59:24 -0400 Subject: [PATCH 26/34] fix(cli): name the host when a request cannot connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bun reports every connection-level failure — DNS, refused, no route — as a bare `Error` reading "Unable to connect. Is the computer able to access the url?". It names neither the host nor what wanted it, and the global handler could only render it as `unexpected_error`. Connection failures now surface as a `CliError` naming the host, under the new `network_unreachable` code. Everything else, an aborted request included, is left exactly as thrown. --- packages/cli-core/src/lib/errors.ts | 2 ++ packages/cli-core/src/lib/fetch.test.ts | 26 +++++++++++++++++++++++++ packages/cli-core/src/lib/fetch.ts | 22 ++++++++++++++++++++- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/lib/errors.ts b/packages/cli-core/src/lib/errors.ts index 247e02776..c984ba4d0 100644 --- a/packages/cli-core/src/lib/errors.ts +++ b/packages/cli-core/src/lib/errors.ts @@ -96,6 +96,8 @@ export const ERROR_CODE = { INSTALLER_NOT_FOUND: "installer_not_found", /** The npm registry was unreachable. */ REGISTRY_UNREACHABLE: "registry_unreachable", + /** A request never reached the server — DNS, refused connection, no route. */ + NETWORK_UNREACHABLE: "network_unreachable", /** Production instance was created but came back without a domain. */ DEPLOY_DOMAIN_MISSING: "deploy_domain_missing", /** Local publishable key and secret key address different applications. */ diff --git a/packages/cli-core/src/lib/fetch.test.ts b/packages/cli-core/src/lib/fetch.test.ts index 505131b9e..9ad2432d6 100644 --- a/packages/cli-core/src/lib/fetch.test.ts +++ b/packages/cli-core/src/lib/fetch.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { _resetUserAgentCache, loggedFetch } from "./fetch.ts"; import { _resetInterruptState, abortInFlight, beginInterrupt, interruptSignal } from "./signals.ts"; import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from "./config.ts"; +import { CliError } from "./errors.ts"; const originalFetch = globalThis.fetch; @@ -34,6 +35,31 @@ describe("loggedFetch", () => { expect(init.headers.get("User-Agent")).toBe("Custom/1.0"); }); + test("reports a connection failure as a CliError naming the host", async () => { + globalThis.fetch = mock(async () => { + // Bun's own shape for DNS failures, refused connections and no-route. + const error: NodeJS.ErrnoException = new Error( + "Unable to connect. Is the computer able to access the url?", + ); + error.code = "ConnectionRefused"; + throw error; + }) as unknown as typeof fetch; + + const failure = loggedFetch("https://example.test/x", { tag: "test" }); + await expect(failure).rejects.toThrow(/Could not reach example\.test/); + await expect(failure).rejects.toBeInstanceOf(CliError); + }); + + test("leaves a non-connection failure alone", async () => { + globalThis.fetch = mock(async () => { + throw new DOMException("The operation was aborted.", "AbortError"); + }) as unknown as typeof fetch; + + await expect(loggedFetch("https://example.test/x", { tag: "test" })).rejects.toThrow( + /operation was aborted/, + ); + }); + test("preserves other caller-provided headers", async () => { globalThis.fetch = mock( async () => new Response("ok", { status: 200 }), diff --git a/packages/cli-core/src/lib/fetch.ts b/packages/cli-core/src/lib/fetch.ts index 1d73ea9ff..36289e164 100644 --- a/packages/cli-core/src/lib/fetch.ts +++ b/packages/cli-core/src/lib/fetch.ts @@ -8,6 +8,7 @@ * every network error. See `.claude/rules/debug-logging.md`. */ +import { CliError, ERROR_CODE } from "./errors.ts"; import { log } from "./log.ts"; import { interruptSignal } from "./signals.ts"; import { withNetworkAccess } from "./host-execution.ts"; @@ -74,6 +75,23 @@ function interruptSignalFor( return own ? AbortSignal.any([own, interruptSignal()]) : interruptSignal(); } +/** + * Bun reports every connection-level failure — DNS, refused, no route — as a + * bare `Error` reading "Unable to connect. Is the computer able to access the + * url?", which names neither the host nor what wanted it, and which the global + * handler can only render as `unexpected_error`. Everything else, an aborted + * request included, is left exactly as thrown. + */ +function asConnectionError(error: unknown, url: string): unknown { + if ((error as NodeJS.ErrnoException | null)?.code !== "ConnectionRefused") return error; + + const host = URL.parse(url)?.host ?? url; + return new CliError( + `Could not reach ${host}. Check your network connection (or VPN) and try again.`, + { code: ERROR_CODE.NETWORK_UNREACHABLE }, + ); +} + export async function loggedFetch(url: URL | string, options: LoggedFetchInit): Promise { const { tag, bestEffort, ignoreInterrupt, ...init } = options; const method = init.method ?? "GET"; @@ -85,7 +103,9 @@ export async function loggedFetch(url: URL | string, options: LoggedFetchInit): const response = await withNetworkAccess( { operation: "connect", target: urlStr, label: tag, bestEffort }, async () => fetch(url, { ...init, headers, signal }), - ); + ).catch((error: unknown) => { + throw asConnectionError(error, urlStr); + }); if (!response.ok) { // Clone so the caller can still consume the body for error construction. const body = await response.clone().text(); From 1c029c8c317367f79ec1711abd77265f67fc152c Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 11 Sep 2026 00:01:17 -0400 Subject: [PATCH 27/34] feat(migrate): ask once where migration logs go, and remember it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration logs are the only record of which users landed and which failed, and `migrate delete` reads them to undo a run — so where they go is worth one question, asked before the first log file is written. `import`, `export` and `delete` now start at `startLogging()`, which settles the directory (`CLERK_MIGRATE_LOG_DIR`, then the saved `log-dir`, then `./logs`) and asks a human who has chosen neither. The answer is saved under the new `log-dir` setting, so the question is asked once per project and never again; `-y`, agent mode and a non-TTY take `./logs` and save nothing, leaving the question open for the first interactive run. `logs list|clean|convert` resolve the directory without ever asking: they are read-only, and "where should logs go?" is not a question to put in front of someone who asked to see the logs they already have. `log-dir` is the first setting kept in the config that also answers to an environment variable, so `settings list` checks the environment for a config setting too — a listing that showed the remembered path while the run read another is the one thing the source column exists to prevent. --- .../cli-core/src/commands/migrate/README.md | 48 ++++++-- .../cli-core/src/commands/migrate/delete.ts | 4 +- .../src/commands/migrate/export/auth0.test.ts | 3 +- .../src/commands/migrate/export/auth0.ts | 4 +- .../src/commands/migrate/export/authjs.ts | 4 +- .../src/commands/migrate/export/betterauth.ts | 4 +- .../src/commands/migrate/export/clerk.test.ts | 3 +- .../src/commands/migrate/export/clerk.ts | 4 +- .../commands/migrate/export/firebase.test.ts | 3 +- .../src/commands/migrate/export/firebase.ts | 4 +- .../src/commands/migrate/export/supabase.ts | 4 +- .../migrate/lib/log-dir-prompt.test.ts | 113 ++++++++++++++++++ .../src/commands/migrate/lib/logger.test.ts | 95 +++++++++++++++ .../src/commands/migrate/lib/logger.ts | 110 ++++++++++++++++- .../src/commands/migrate/logs/clean.ts | 3 +- .../src/commands/migrate/logs/convert.ts | 3 +- .../src/commands/migrate/logs/index.ts | 4 +- .../src/commands/migrate/logs/list.ts | 5 +- .../commands/migrate/run-interactive.test.ts | 10 +- packages/cli-core/src/commands/migrate/run.ts | 4 +- .../src/commands/migrate/settings/list.ts | 11 ++ .../src/commands/migrate/settings/registry.ts | 22 +++- .../migrate/settings/settings.test.ts | 2 +- packages/cli-core/src/lib/config.ts | 2 + packages/cli-core/src/test/lib/stubs.ts | 32 ++++- 25 files changed, 462 insertions(+), 39 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 018cd76d6..91a06eebb 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -386,7 +386,8 @@ failed. ### `clerk migrate logs` -Everything that touches the local `./logs/` directory. Noun-verb like every +Everything that touches the local log directory — `./logs` unless the project +says otherwise; see [Where logs go](#where-logs-go). Noun-verb like every other group in the CLI (`config pull`, `users list`), rather than the standalone tool's `clean-logs`/`convert-logs`, which were npm script names. @@ -405,11 +406,36 @@ clerk migrate logs convert import-2026-01-01T12-00-00.log | Subcommand | Takes | Description | | -------------- | ------------------ | ----------------------------------------------- | | `logs list` | `--json` | File, type, date, size and entry count per file | -| `logs clean` | `-y, --yes` | Delete the `.log` files in `./logs/` | +| `logs clean` | `-y, --yes` | Delete the `.log` files in the log directory | | `logs convert` | `[file…]`, `--all` | NDJSON → a JSON array, written as `.json` | All three read the directory through one shared enumerator, which is what makes -`logs list` nearly free. +`logs list` nearly free. All three **resolve** the directory without ever asking +for one: they are read-only, and "where should logs go?" is not a question to +put in front of someone who asked to see the logs they already have. + +#### Where logs go + +`./logs`, relative to the current directory, until the project says otherwise. +Resolution order, highest first: + +| Source | Set by | +| --------------------------- | ----------------------------------------------------- | +| `CLERK_MIGRATE_LOG_DIR` | The shell, `.env`, `.env.local`, `.env.clerk-migrate` | +| `log-dir` in the CLI config | The first-run prompt, or `settings set` | +| `./logs` | The fallback | + +The first time `migrate import`, `migrate export` or `migrate delete` runs +interactively in a project with none of those set, it asks where logs should be +saved and offers `./logs`. The answer is saved under `log-dir`, so it is asked +once per project and never again. `-y`, agent mode and a non-TTY take `./logs` +without asking **and without saving it** — landing on a default is not a choice, +and recording one would retire the question for a human who never saw it. + +Logs are the only record of which users landed and which failed, and +`migrate delete` reads them to undo a run, so where they go is worth the one +question. Change it later with `clerk migrate settings set log-dir `, or +clear it with `clerk migrate settings clear log-dir` to be asked again. #### `logs list` @@ -450,7 +476,7 @@ raw stamp. The directory is printed relative (`./logs`) when it sits under the current directory and absolute when it does not, so the path can be pasted either way. -Says so plainly when `./logs/` is empty or absent. +Says so plainly when the log directory is empty or absent. #### `logs clean` @@ -604,10 +630,16 @@ firebase-mem-cost 14 MEM_COST env var Firebase scrypt mem Two stores, split by what the value **is** rather than by which command wrote it: -| Store | Holds | Why | -| -------------------- | --------------------------------------------------- | ---------------------------------------------------------------- | -| CLI config | `transformer`, `file`, `skip-unsupported-providers` | Project state, not secret, useless outside the CLI | -| `.env.clerk-migrate` | `firebase-*` | Credentials: gitignored on write, and hand-editable for rotation | +| Store | Holds | Why | +| -------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------- | +| CLI config | `transformer`, `file`, `skip-unsupported-providers`, `log-dir` | Project state, not secret, useless outside the CLI | +| `.env.clerk-migrate` | `firebase-*` | Credentials: gitignored on write, and hand-editable for rotation | + +`log-dir` is the one setting that answers to both: it is remembered in the CLI +config, and `CLERK_MIGRATE_LOG_DIR` outranks what is remembered, so a directory +can be pinned for one shell without disturbing the project. The listing's source +column says which is winning, and `settings clear log-dir` clears both — half a +clear would report the setting gone while the next run still read it. `.env.clerk-migrate` is the migration's own file rather than the app's `.env.local`, because a Firebase signer key is of no use to the application diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts index 32e7d4c41..f80133f76 100644 --- a/packages/cli-core/src/commands/migrate/delete.ts +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -37,7 +37,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../lib/spinner import { isAgent, isHuman } from "../../mode.ts"; import { normalizeErrorMessage } from "./import-users.ts"; import { resolveLimits, type ResolvedLimits } from "./lib/instance.ts"; -import { deleteErrorLogger, deleteLogger, getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; +import { deleteErrorLogger, deleteLogger, startLogging, getLogFilePath } from "./lib/logger.ts"; import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; import { createApiScheduler } from "./lib/scheduler.ts"; import { loadSettings } from "./lib/settings.ts"; @@ -263,7 +263,7 @@ export async function deleteMigration(options: MigrateDeleteOptions): Promise { const destination = await resolveOutputPath("auth0", options.output); await withGutter("Exporting users from Auth0", async ({ setNextSteps }) => { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); log.info(`Exporting from ${credentials.domain}.`); const token = await withSpinner("Authenticating with Auth0...", () => diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 1f74223a4..d378c93c1 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -13,7 +13,7 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { log } from "../../../lib/log.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; @@ -116,7 +116,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { const destination = await resolveOutputPath("authjs", options.output); await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); const { rows, table } = await withSpinner("Reading the user table...", () => withDbClient(dbUrl, "authjs", fetchAuthJsUsers), diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 006af76f9..f2d26c6c6 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -17,7 +17,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; @@ -163,7 +163,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise const destination = await resolveOutputPath("betterauth", options.output); await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); const { rows, plugins } = await withSpinner("Reading the user table...", () => withDbClient(dbUrl, "betterauth", async (client) => { diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 75e1f0565..67f825172 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -3,7 +3,7 @@ import { getMode, setMode } from "../../../mode.ts"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { buildClerkExport, @@ -13,6 +13,7 @@ import { } from "./clerk.ts"; const captured = useCaptureLog(); +useMigrateLogDir(); let workDir: string; let originalCwd: string; diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index ec4a633be..28d5a1e4d 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -17,7 +17,7 @@ import { bapiRequest } from "../../../lib/bapi.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; import { retryOn429 } from "../lib/retry.ts"; import { resolveClerkSource } from "./clerk-source.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; @@ -235,7 +235,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { const destination = await resolveOutputPath("clerk", options.output); await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index 4f9c4ac25..d4a440b7e 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; import { getLogDir } from "../lib/logger.ts"; import { buildFirebaseExport, @@ -21,6 +21,7 @@ import { } from "./firebase.ts"; const captured = useCaptureLog(); +useMigrateLogDir(); let workDir: string; let originalCwd: string; diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 6facdd770..93a6f53af 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -31,7 +31,7 @@ import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; /** Identity Toolkit's maximum for `accounts:batchGet`. */ @@ -487,7 +487,7 @@ export async function exportFirebase(options: ExportFirebaseOptions): Promise { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); log.info(`Exporting from the ${account.project_id} project.`); const token = await withSpinner("Authenticating with Google...", () => diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 2436c95c5..cf257f35f 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -12,7 +12,7 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; -import { exportLogger, getDateTimeStamp } from "../lib/logger.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; @@ -114,7 +114,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { const destination = await resolveOutputPath("supabase", options.output); await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); const rows = await withSpinner("Reading auth.users...", () => withDbClient(dbUrl, "supabase", fetchSupabaseUsers), diff --git a/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts b/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts new file mode 100644 index 000000000..766145d24 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts @@ -0,0 +1,113 @@ +/** + * The one path `logger.test.ts` cannot cover: `ensureLogDir` actually asking. + * + * Its own file because `mock.module` registrations last for the process, and + * `bun test --parallel` puts several files in each worker — a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +type TextConfig = { message: string; default?: string; placeholder?: string }; + +let answer = ""; +const mockText = mock(async (_config: TextConfig) => answer); + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../../lib/prompts.ts", () => ({ + text: (...args: unknown[]) => mockText(...(args as [TextConfig])), + confirm: async () => true, + multiselect: async () => [], + password: async () => "", + editor: async () => "{}", +})); + +const { _resetLogDir, ensureLogDir } = await import("./logger.ts"); +const { loadSettings, saveSettings } = await import("./settings.ts"); + +useCaptureLog(); + +let workDir: string; +let configDir: string; +let originalCwd: string; +let originalMode: Mode; +let originalEnv: string | undefined; + +beforeAll(() => { + originalCwd = process.cwd(); + originalMode = getMode(); + originalEnv = process.env.CLERK_MIGRATE_LOG_DIR; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-cfg-")); + _setConfigDir(configDir); + process.chdir(workDir); +}); + +afterAll(() => { + setMode(originalMode); + if (originalEnv === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; + else process.env.CLERK_MIGRATE_LOG_DIR = originalEnv; + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + _resetLogDir(); + delete process.env.CLERK_MIGRATE_LOG_DIR; + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + mockText.mockClear(); + answer = ""; + setMode("human"); +}); + +afterEach(() => _resetLogDir()); + +describe("ensureLogDir asks once", () => { + test("saves the answer, so the next run does not ask", async () => { + answer = "./migration-logs"; + + expect(await ensureLogDir()).toBe(path.join(workDir, "migration-logs")); + expect(await loadSettings()).toMatchObject({ logDir: "./migration-logs" }); + + _resetLogDir(); + expect(await ensureLogDir()).toBe(path.join(workDir, "migration-logs")); + expect(mockText).toHaveBeenCalledTimes(1); + }); + + test("offers ./logs as the default", async () => { + await ensureLogDir(); + expect(mockText.mock.calls[0]?.[0]).toMatchObject({ default: "./logs" }); + }); + + // Enter on the prompt is an answer, not a skip: it settles the question so + // the next run goes straight to importing. + test("treats an empty answer as ./logs and remembers it", async () => { + answer = " "; + + expect(await ensureLogDir()).toBe(path.join(workDir, "logs")); + expect(await loadSettings()).toMatchObject({ logDir: "./logs" }); + }); + + test("leaves the project's other settings alone", async () => { + await saveSettings({ transformer: "firebase", file: "users.json" }); + answer = "./audit"; + + await ensureLogDir(); + + expect(await loadSettings()).toEqual({ + transformer: "firebase", + file: "users.json", + logDir: "./audit", + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.test.ts b/packages/cli-core/src/commands/migrate/lib/logger.test.ts index eb153218d..e9176df9a 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.test.ts @@ -3,13 +3,21 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { + _resetLogDir, + DEFAULT_LOG_DIR, + ensureLogDir, errorLogger, getDateTimeStamp, getLogDir, getLogFilePath, importLogger, + resolveLogDir, validationLogger, } from "./logger.ts"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { MIGRATE_ENV_FILE } from "./env-file.ts"; +import { loadSettings, saveSettings } from "./settings.ts"; const DATE_TIME = "2026-01-01T12:00:00"; @@ -30,6 +38,7 @@ afterAll(() => { }); beforeEach(() => { + _resetLogDir(); fs.rmSync(getLogDir(), { recursive: true, force: true }); }); @@ -108,3 +117,89 @@ describe("log writers", () => { }); }); }); + +describe("resolving the log directory", () => { + let configDir: string; + let originalMode: Mode; + let originalEnv: string | undefined; + + beforeAll(() => { + originalMode = getMode(); + originalEnv = process.env.CLERK_MIGRATE_LOG_DIR; + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-cfg-")); + _setConfigDir(configDir); + }); + + afterAll(() => { + setMode(originalMode); + if (originalEnv === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; + else process.env.CLERK_MIGRATE_LOG_DIR = originalEnv; + _setConfigDir(undefined); + fs.rmSync(configDir, { recursive: true, force: true }); + }); + + beforeEach(() => { + delete process.env.CLERK_MIGRATE_LOG_DIR; + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.rmSync(path.join(workDir, MIGRATE_ENV_FILE), { force: true }); + setMode("agent"); + }); + + test("falls back to ./logs when nothing has chosen one", async () => { + expect(await resolveLogDir()).toBe(path.join(workDir, "logs")); + }); + + test("prefers the saved setting over the default", async () => { + await saveSettings({ logDir: "./audit" }); + expect(await resolveLogDir()).toBe(path.join(workDir, "audit")); + }); + + // A variable exported for one shell is the narrower statement of the two. + test("prefers the environment over the saved setting", async () => { + await saveSettings({ logDir: "./audit" }); + process.env.CLERK_MIGRATE_LOG_DIR = "./from-env"; + + expect(await resolveLogDir()).toBe(path.join(workDir, "from-env")); + }); + + test("reads the migration's own env file", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "CLERK_MIGRATE_LOG_DIR=./from-file\n"); + expect(await resolveLogDir()).toBe(path.join(workDir, "from-file")); + }); + + // Every synchronous write reads the settled value, so resolving is what makes + // the log files land anywhere but ./logs. + test("settles the directory the log writers use", async () => { + await saveSettings({ logDir: "./audit" }); + await resolveLogDir(); + + expect(getLogFilePath("import", DATE_TIME)).toBe( + path.join(workDir, "audit", `import-${DATE_TIME.replace(/:/g, "-")}.log`), + ); + }); + + describe("ensureLogDir", () => { + test("takes the default without saving it when nobody can be asked", async () => { + expect(await ensureLogDir()).toBe(path.join(workDir, DEFAULT_LOG_DIR)); + // Nothing saved: the question stays open for the first interactive run. + expect(await loadSettings()).toEqual({}); + }); + + test("does not ask once the setting is saved", async () => { + await saveSettings({ logDir: "./audit" }); + setMode("human"); + + // Reaching the prompt in a test without a TTY throws, so returning is the + // assertion. + expect(await ensureLogDir()).toBe(path.join(workDir, "audit")); + }); + + test("does not ask when the environment already answers", async () => { + process.env.CLERK_MIGRATE_LOG_DIR = "./from-env"; + setMode("human"); + + expect(await ensureLogDir()).toBe(path.join(workDir, "from-env")); + expect(await loadSettings()).toEqual({}); + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.ts b/packages/cli-core/src/commands/migrate/lib/logger.ts index 54d0d54a3..814fe4914 100644 --- a/packages/cli-core/src/commands/migrate/lib/logger.ts +++ b/packages/cli-core/src/commands/migrate/lib/logger.ts @@ -14,6 +14,11 @@ import fs from "node:fs"; import path from "node:path"; import { log } from "../../../lib/log.ts"; +import { text } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { envNames, findSetting } from "../settings/registry.ts"; +import { findMigrateEnvValue } from "./env-file.ts"; +import { loadSettings, saveSettings } from "./settings.ts"; import type { DeleteLogEntry, ErrorLog, @@ -23,9 +28,110 @@ import type { ValidationErrorPayload, } from "../types.ts"; -/** Absolute path of the cwd-relative `logs/` directory. */ +/** Where logs go when nobody has said otherwise. */ +export const DEFAULT_LOG_DIR = "./logs"; + +/** + * The directory settled for this process, once something has settled it. + * + * The log writers are synchronous — a run interrupted with Ctrl-C has to leave + * a complete record of what it already processed — but resolving the directory + * reads the config, the env files and possibly the operator. So resolution + * happens once, up front, and every synchronous write reads the answer from + * here. {@link resolveLogDir} and {@link ensureLogDir} are the only writers. + */ +let settled: string | undefined; + +function remember(dir: string): string { + settled = path.resolve(process.cwd(), dir); + return settled; +} + +/** Forgets the settled directory. Tests only — each one resolves its own. */ +export function _resetLogDir(): void { + settled = undefined; +} + +/** + * Absolute path of the log directory. + * + * Falls back to `./logs` when nothing has resolved yet, so a caller that + * forgets to is wrong about *where*, never broken. + */ export function getLogDir(): string { - return path.join(process.cwd(), "logs"); + return settled ?? path.resolve(process.cwd(), DEFAULT_LOG_DIR); +} + +/** The `log-dir` setting, which owns both the env var and the config key. */ +const LOG_DIR = findSetting("log-dir") as NonNullable>; + +/** + * The directory the operator has already chosen, by either route. + * + * The environment wins over the remembered value, matching every other setting + * the CLI resolves: a variable exported for one shell is the narrower, more + * deliberate statement of the two. + */ +async function chosenLogDir(): Promise { + const located = await findMigrateEnvValue(envNames(LOG_DIR)); + if (located?.value) return located.value; + return (await loadSettings()).logDir; +} + +/** + * Settles the log directory without asking: environment, then the saved + * setting, then `./logs`. + * + * For the read-only log commands. Landing on the default here does not save it + * — an operator who has only ever *listed* logs has still made no choice, and + * recording one on their behalf would skip the question forever. + */ +export async function resolveLogDir(): Promise { + return remember((await chosenLogDir()) ?? DEFAULT_LOG_DIR); +} + +/** + * Settles the log directory, asking a human who has not chosen yet. + * + * Migration logs are the only record of which users landed and which failed, + * and `migrate delete` reads them to undo a run — so where they go is worth one + * question, once per project, before the first thing is written. The answer is + * saved, so it is asked once and never again. + * + * `-y`, agent mode and a non-TTY take the default rather than a prompt they + * cannot answer, and save nothing: the question stays open for the first + * interactive run. + */ +export async function ensureLogDir(): Promise { + const chosen = await chosenLogDir(); + if (chosen) return remember(chosen); + if (!isHuman() || isAgent()) return remember(DEFAULT_LOG_DIR); + + const answer = await text({ + message: "Where should migration logs be saved?", + default: DEFAULT_LOG_DIR, + placeholder: DEFAULT_LOG_DIR, + }); + const dir = answer.trim() || DEFAULT_LOG_DIR; + + await saveSettings({ ...(await loadSettings()), logDir: dir }); + log.info( + `Saving migration logs to ${dir}. Change it with \`clerk migrate settings set log-dir \`.`, + ); + + return remember(dir); +} + +/** + * Settles where this run's logs go, and stamps it. + * + * Every command that writes a log starts here rather than calling + * {@link getDateTimeStamp} directly, so there is no path on which a log file is + * named before its directory has been resolved. + */ +export async function startLogging(): Promise { + await ensureLogDir(); + return getDateTimeStamp(); } /** diff --git a/packages/cli-core/src/commands/migrate/logs/clean.ts b/packages/cli-core/src/commands/migrate/logs/clean.ts index 3c609f5ad..4d17d684d 100644 --- a/packages/cli-core/src/commands/migrate/logs/clean.ts +++ b/packages/cli-core/src/commands/migrate/logs/clean.ts @@ -16,7 +16,7 @@ import { confirm } from "../../../lib/prompts.ts"; import { withGutter } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { listLogFiles } from "../lib/log-files.ts"; -import { getLogDir } from "../lib/logger.ts"; +import { getLogDir, resolveLogDir } from "../lib/logger.ts"; export type LogsCleanOptions = { yes?: boolean; @@ -24,6 +24,7 @@ export type LogsCleanOptions = { export async function clean(options: LogsCleanOptions = {}): Promise { await withGutter("Cleaning migration logs", async () => { + await resolveLogDir(); const files = listLogFiles(); if (files.length === 0) { diff --git a/packages/cli-core/src/commands/migrate/logs/convert.ts b/packages/cli-core/src/commands/migrate/logs/convert.ts index 33adf2bae..4536e70a5 100644 --- a/packages/cli-core/src/commands/migrate/logs/convert.ts +++ b/packages/cli-core/src/commands/migrate/logs/convert.ts @@ -15,7 +15,7 @@ import { multiselect } from "../../../lib/prompts.ts"; import { withGutter } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { findLogFile, listLogFiles, readNdjson, type LogFile } from "../lib/log-files.ts"; -import { getLogDir } from "../lib/logger.ts"; +import { getLogDir, resolveLogDir } from "../lib/logger.ts"; export type LogsConvertOptions = { all?: boolean; @@ -85,6 +85,7 @@ export async function convert(options: LogsConvertOptions = {}): Promise { // The multiselect lives inside the gutter so cancelling it closes with // `└ Paused` rather than leaving a half-drawn frame. await withGutter("Converting migration logs", async () => { + await resolveLogDir(); const targets = await resolveTargets(options); if (targets.length === 0) return; diff --git a/packages/cli-core/src/commands/migrate/logs/index.ts b/packages/cli-core/src/commands/migrate/logs/index.ts index 50279c8ff..d12a5d29b 100644 --- a/packages/cli-core/src/commands/migrate/logs/index.ts +++ b/packages/cli-core/src/commands/migrate/logs/index.ts @@ -32,7 +32,7 @@ export function registerMigrateLogs(migrateCommand: Command<[], Record, string> = { @@ -53,6 +53,9 @@ export function formatTimestamp(stamp: string): string { } export async function list(options: LogsListOptions = {}): Promise { + // Resolve, never ask: listing is read-only, and "where should logs go?" is + // not a question to answer before showing someone the ones they have. + await resolveLogDir(); const files = listLogFiles(); if (options.json) { diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 9d29d32f1..a7e6909c8 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -14,7 +14,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { getMode, setMode, type Mode } from "../../mode.ts"; -import { keylessTargetStubs, listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; +import { + keylessTargetStubs, + listageStubs, + useCaptureLog, + useMigrateLogDir, +} from "../../test/lib/stubs.ts"; import type { InstanceTarget } from "../../lib/keyless-target.ts"; const mockSelect = mock(async () => "clerk" as unknown); @@ -22,6 +27,8 @@ const mockText = mock(async () => "export.json" as unknown); type MultiselectConfig = { options: { value: string; label: string; hint?: string }[] }; const mockMultiselect = mock(async (_config: MultiselectConfig) => [] as unknown[]); let confirmAnswer = true; +/** Every confirmation the run put up, in order — the wording is the assertion. */ +let confirmMessages: string[] = []; let originalMode: Mode; const ACCOUNT_TARGET: InstanceTarget = { @@ -66,6 +73,7 @@ const { loadSettings, saveSettings } = await import("./lib/settings.ts"); const { _setConfigDir } = await import("../../lib/config.ts"); const captured = useCaptureLog(); +useMigrateLogDir(); let workDir: string; let configDir: string; diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index ca0576022..dd8b6b467 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -43,7 +43,7 @@ import { type SettingChange, } from "./lib/modify-settings.ts"; import { DEV_USER_LIMIT, resolveLimits } from "./lib/instance.ts"; -import { getDateTimeStamp, getLogFilePath } from "./lib/logger.ts"; +import { startLogging, getLogFilePath } from "./lib/logger.ts"; import { saveSettings } from "./lib/settings.ts"; import { countSocialProviders, @@ -483,7 +483,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); const limits = resolveLimits(secretKey); - const dateTime = getDateTimeStamp(); + const dateTime = await startLogging(); const logFile = getLogFilePath("import", dateTime); const { users: loaded, validationFailed } = await withSpinner( diff --git a/packages/cli-core/src/commands/migrate/settings/list.ts b/packages/cli-core/src/commands/migrate/settings/list.ts index 049264958..747a179f5 100644 --- a/packages/cli-core/src/commands/migrate/settings/list.ts +++ b/packages/cli-core/src/commands/migrate/settings/list.ts @@ -52,6 +52,17 @@ async function resolveAll(): Promise { return Promise.all( SETTINGS.map(async (setting): Promise => { if (setting.store === "config") { + // `log-dir` is remembered in the config but yields to an environment + // value, so the environment has to be checked first here too — a + // listing that shows the remembered path while the run reads another + // is the one thing the source column exists to prevent. + if (setting.envVar) { + const located = await findMigrateEnvValue(envNames(setting)); + if (located) { + return { setting, value: located.value, source: describeSource(setting, located) }; + } + } + const value = saved[setting.configKey as keyof typeof saved]; return value === undefined ? { setting } diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index 2b403da46..7a471e855 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -31,7 +31,14 @@ export interface SettingDef { name: string; store: SettingStore; description: string; - /** For `env` settings, the variable read at run time. */ + /** + * The environment variable this setting is read from at run time. + * + * Required for an `env` setting, which lives nowhere else. A `config` + * setting may also declare one, meaning "remembered here, but an environment + * value wins" — `log-dir` is that shape, so an operator can pin a directory + * per shell without disturbing what the project remembers. + */ envVar?: string; /** * Other variables accepted for the same setting, read only when @@ -50,7 +57,7 @@ export interface SettingDef { */ envAliases?: string[]; /** For `config` settings, the key on the saved migration entry. */ - configKey?: "transformer" | "file" | "skipUnsupportedProviders"; + configKey?: "transformer" | "file" | "skipUnsupportedProviders" | "logDir"; /** Redact when displaying — the value is a credential. */ secret?: boolean; /** Reject a value the run would only fail on later. */ @@ -65,6 +72,9 @@ const positiveInteger = (value: string): string | undefined => { const boolean = (value: string): string | undefined => ["true", "false"].includes(value) ? undefined : "Expected true or false"; +const path = (value: string): string | undefined => + value.trim().length > 0 ? undefined : "Expected a directory path"; + export const SETTINGS: SettingDef[] = [ { name: "transformer", @@ -85,6 +95,14 @@ export const SETTINGS: SettingDef[] = [ description: "Skip users with no provider enabled in Clerk (Supabase)", validate: boolean, }, + { + name: "log-dir", + store: "config", + configKey: "logDir", + envVar: "CLERK_MIGRATE_LOG_DIR", + description: "Directory migration logs are written to", + validate: path, + }, { name: "firebase-signer-key", store: "env", diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index f27b84f8e..946d475c6 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -207,7 +207,7 @@ describe("list", () => { await list(); - expect(captured.err).toContain("1 of 7 settings set"); + expect(captured.err).toContain("1 of 8 settings set"); }); // Firebase's own names for these, and what every guide tells you to paste diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index 08e0295d1..4d84ed82d 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -55,6 +55,8 @@ interface MigrationEntry { transformer?: string; file?: string; skipUnsupportedProviders?: boolean; + /** Where this project's migration logs are written. Absent until chosen. */ + logDir?: string; } interface ClerkConfig { diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index e16e7f131..7cb412560 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -1,7 +1,8 @@ import { Writable } from "node:stream"; -import { afterEach, beforeEach, type spyOn } from "bun:test"; +import { afterAll, afterEach, beforeAll, beforeEach, type spyOn } from "bun:test"; import { type CapturedLogs, setActiveCapture } from "../../lib/log.ts"; import { setUiOutput } from "../../lib/ui.ts"; +import { _resetLogDir } from "../../commands/migrate/lib/logger.ts"; export function capturedOutput(spy: ReturnType): string { return spy.mock.calls.map((c: unknown[]) => c[0]).join("\n"); @@ -251,3 +252,32 @@ type FetchImpl = (input: string | URL | Request, init?: RequestInit) => Promise< export function stubFetch(impl: FetchImpl): void { globalThis.fetch = impl as typeof fetch; } + +/** + * Settles the migration log directory for a whole test file. + * + * `migrate import`, `export` and `delete` ask a human where logs should go the + * first time a project runs one. A test that flips to human mode to exercise + * something else — next steps, a wizard — would stop on that question and, + * where `prompts.ts` is mocked, silently eat the answer meant for another + * prompt. Pinning the environment variable answers it before it is asked, the + * same way an operator who exported one never sees it. + */ +export function useMigrateLogDir(dir = "./logs"): void { + let original: string | undefined; + + beforeAll(() => { + original = process.env.CLERK_MIGRATE_LOG_DIR; + process.env.CLERK_MIGRATE_LOG_DIR = dir; + }); + + // Resolution is cached per process, and each test file runs under its own + // temporary cwd, so the cached absolute path has to go with it. + beforeEach(() => _resetLogDir()); + + afterAll(() => { + if (original === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; + else process.env.CLERK_MIGRATE_LOG_DIR = original; + _resetLogDir(); + }); +} From 9900266d94ccc0e92d9ea664118509eca8ef2929 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 11 Sep 2026 00:01:54 -0400 Subject: [PATCH 28/34] feat(migrate): clear one setting by name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clerk migrate settings clear ` forgets a single setting and leaves the rest of the project alone; with no name it still clears both stores, as before. Both stores are cleared either way, because a setting can sit in either and `log-dir` can sit in both — clearing half of one is worse than clearing none, since the command would report the setting gone while the next run still read it. An `env` value goes under every spelling the setting answers to, so dropping `CLERK_FIREBASE_ROUNDS` no longer leaves a bare `ROUNDS` behind to win the next resolution. `.choices()` rejects an unknown name before the action runs, so the friendly "Unknown setting" errors inside `set.ts` and `clear.ts` were unreachable from the CLI and a one-character miss got back only the list of eight names. The argument's parser now names the near miss first — `logs-dir` suggests `log-dir` — while leaving whether a value is allowed to Commander. --- .../cli-core/src/commands/migrate/README.md | 30 ++++- .../src/commands/migrate/settings/clear.ts | 109 +++++++++++++++--- .../src/commands/migrate/settings/index.ts | 59 ++++++++-- .../src/commands/migrate/settings/registry.ts | 45 ++++++++ .../migrate/settings/settings.test.ts | 77 ++++++++++++- packages/cli-core/src/lib/next-steps.ts | 1 + 6 files changed, 292 insertions(+), 29 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 91a06eebb..1fbf3e1ea 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -574,14 +574,31 @@ clerk migrate settings # list clerk migrate settings list --json clerk migrate settings set transformer firebase clerk migrate settings set firebase-signer-key abc123 -clerk migrate settings clear -y +clerk migrate settings clear firebase-signer-key # forget one +clerk migrate settings clear -y # forget them all ``` -| Subcommand | Takes | Description | -| ----------------------------- | ---------------- | --------------------------------------------------------- | -| `settings list` | `--json` | Every setting, its value and the source it resolved from | -| `settings set ` | ` ` | Change one setting | -| `settings clear` | `-y, --yes` | Forget this project's settings and delete its credentials | +| Subcommand | Takes | Description | +| ----------------------------- | --------------------- | -------------------------------------------------------- | +| `settings list` | `--json` | Every setting, its value and the source it resolved from | +| `settings set ` | ` ` | Change one setting | +| `settings clear [name]` | `[name]`, `-y, --yes` | Forget one setting, or every setting and its credentials | + +`settings clear ` leaves the rest of the project's settings alone. For a +credential it drops every variable the setting answers to, aliases included — +clearing `firebase-rounds` while a bare `ROUNDS` stayed behind in the same file +would report the setting cleared and leave the next run reading the old value. +It only ever edits `.env.clerk-migrate`; a value coming from the app's own env +file or the shell is named in the listing's source column and has to be removed +there. + +A misspelled name gets the closest match back, not just the list: + +``` +$ clerk migrate settings clear logs-dir +error: command-argument value 'logs-dir' is invalid for argument 'name'. + Did you mean "log-dir"? Allowed choices are transformer, file, … +``` Setting names are kebab-case and identical to the `clerk migrate import` flag each one backs, so `firebase-signer-key` here is `--firebase-signer-key` there @@ -623,6 +640,7 @@ firebase-mem-cost 14 MEM_COST env var Firebase scrypt mem 4 of 7 settings set. Credentials are shown redacted. → Run `clerk migrate settings set ` to change one + → Run `clerk migrate settings clear ` to forget one → Run `clerk migrate settings clear` to forget them all, credentials included ``` diff --git a/packages/cli-core/src/commands/migrate/settings/clear.ts b/packages/cli-core/src/commands/migrate/settings/clear.ts index cb588e0eb..23871d90e 100644 --- a/packages/cli-core/src/commands/migrate/settings/clear.ts +++ b/packages/cli-core/src/commands/migrate/settings/clear.ts @@ -1,41 +1,120 @@ /** - * `clerk migrate settings clear` — forget this project's migration settings. + * `clerk migrate settings clear [name]` — forget this project's migration + * settings, or just one of them. * - * Clears both stores by default. The credentials half is the reason this - * command exists: after a migration finishes, a Firebase signer key sitting in - * the repo has no further use, and "delete the file yourself" is a step people - * skip. + * Clears both stores when given no name. The credentials half is the reason + * this command exists: after a migration finishes, a Firebase signer key + * sitting in the repo has no further use, and "delete the file yourself" is a + * step people skip. * * `migrate delete` reads the saved transformer and file to know what to undo, * so clearing is confirmed unless `-y` — an operator who clears and then wants * to undo has no record left to undo from. */ -import { throwUserAbort } from "../../../lib/errors.ts"; +import { throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { confirm } from "../../../lib/prompts.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { clearMigrateEnvValues, MIGRATE_ENV_FILE } from "../lib/env-file.ts"; import { loadSettings, saveSettings } from "../lib/settings.ts"; -import { SETTINGS } from "./registry.ts"; +import type { MigrationEntry } from "../../../lib/config.ts"; +import { envNames, findSetting, SETTING_NAMES, SETTINGS } from "./registry.ts"; export type SettingsClearOptions = { yes?: boolean; }; -const ENV_VARS = SETTINGS.filter((s) => s.store === "env").map((s) => s.envVar as string); +/** + * Every variable the migration settings own, `log-dir`'s included. + * + * Keyed on declaring an `envVar` rather than on `store === "env"`: `log-dir` is + * remembered in the config but still answers to a variable, and a full clear + * that left that variable behind would not have cleared the setting. + */ +const ENV_VARS = SETTINGS.filter((s) => s.envVar).map((s) => s.envVar as string); + +/** + * Warns that clearing this is what `migrate delete` reads to find the users the + * last run created. + */ +function warnAboutUndo(saved: MigrationEntry): void { + if (!saved.file) return; + log.warn( + `\`clerk migrate delete\` uses the saved file (${saved.file}) to identify the users the last run created. ` + + "Clearing it leaves nothing to undo from.", + ); +} + +/** + * Clears one named setting, leaving the rest of the project's settings alone. + * + * Both stores are cleared, because a setting can sit in either and `log-dir` + * can sit in both. Clearing half of one is worse than clearing none: the + * command reports the setting gone while the next run still reads it. + * + * An `env` value goes under every spelling the setting answers to, not just the + * prefixed one — dropping `CLERK_FIREBASE_ROUNDS` while `ROUNDS` stayed in the + * same file would leave the old value winning. + */ +async function clearOne(name: string, options: SettingsClearOptions): Promise { + const setting = findSetting(name); + if (!setting) { + throwUsageError( + `Unknown setting "${name}". Valid names: ${SETTING_NAMES.join(", ")}.`, + undefined, + undefined, + [ + { + command: "clerk migrate settings", + description: "List the settings and their current values", + }, + ], + ); + } + + const saved = await loadSettings(); + + if (!options.yes && isHuman() && !isAgent()) { + // Only the file itself is what `migrate delete` cannot do without; the + // transformer it can be told again. + if (setting.configKey === "file") warnAboutUndo(saved); + const proceed = await confirm({ message: `Clear \`${name}\`?`, default: false }); + if (!proceed) throwUserAbort(); + } + + const cleared: string[] = []; + + if (setting.envVar && (await clearMigrateEnvValues(envNames(setting))).length > 0) { + cleared.push(MIGRATE_ENV_FILE); + } + + const key = setting.configKey as keyof MigrationEntry | undefined; + if (key && saved[key] !== undefined) { + const { [key]: _cleared, ...rest } = saved; + await saveSettings(rest); + cleared.push("this project's settings"); + } + + if (cleared.length === 0) { + log.info( + `\`${name}\` is not set here. A value coming from the app's own env files or the shell has ` + + "to be removed there — run `clerk migrate settings` to see which is supplying it.", + ); + return; + } + + log.success(`Cleared \`${name}\` from ${cleared.join(" and ")}.`); +} + +export async function clear(options: SettingsClearOptions = {}, name?: string): Promise { + if (name !== undefined) return clearOne(name, options); -export async function clear(options: SettingsClearOptions = {}): Promise { const saved = await loadSettings(); const hadConfig = Object.keys(saved).length > 0; if (!options.yes && isHuman() && !isAgent()) { - if (hadConfig && saved.file) { - log.warn( - `\`clerk migrate delete\` uses the saved file (${saved.file}) to identify the users the last run created. ` + - "Clearing it leaves nothing to undo from.", - ); - } + if (hadConfig) warnAboutUndo(saved); const proceed = await confirm({ message: "Clear this project's migration settings?", default: false, diff --git a/packages/cli-core/src/commands/migrate/settings/index.ts b/packages/cli-core/src/commands/migrate/settings/index.ts index 7ce5b6f44..2a9aa0fdf 100644 --- a/packages/cli-core/src/commands/migrate/settings/index.ts +++ b/packages/cli-core/src/commands/migrate/settings/index.ts @@ -1,12 +1,48 @@ -import { createArgument } from "@commander-js/extra-typings"; +import { createArgument, InvalidArgumentError } from "@commander-js/extra-typings"; import type { Command } from "@commander-js/extra-typings"; import { clear } from "./clear.ts"; import { list } from "./list.ts"; -import { SETTING_NAMES } from "./registry.ts"; +import { SETTING_NAMES, suggestSettingName } from "./registry.ts"; import { set } from "./set.ts"; const settings = { clear, list, set }; +/** + * The `` argument both `set` and `clear` take. + * + * `.choices()` is what drives tab-completion and the help output's choice list, + * but it is implemented as a `parseArg` that throws before the action runs — so + * the friendlier "Unknown setting" errors inside `set.ts` and `clear.ts` are + * unreachable from the CLI, and a one-character miss like `logs-dir` gets only + * the full list back. Wrapping that parser keeps the completion metadata and + * puts the near miss first, where a reader scanning eight names would not find + * it. + */ +function settingNameArgument` | `[${string}]`>( + spec: S, + description: string, +) { + const argument = createArgument(spec, description).choices(SETTING_NAMES); + const rejectUnlessAllowed = argument.parseArg; + + // Whether the value is allowed stays Commander's question — asking it here + // too would be a second copy of the rule, free to disagree with the first. + // This only adds to the answer when the answer is no. + argument.parseArg = (value: string, previous: T): T => { + try { + return rejectUnlessAllowed?.(value, previous) as T; + } catch (error) { + const suggestion = suggestSettingName(value); + if (!suggestion) throw error; + throw new InvalidArgumentError( + `Did you mean "${suggestion}"? Allowed choices are ${SETTING_NAMES.join(", ")}.`, + ); + } + }; + + return argument; +} + /** * Registers `settings list|set|clear` under the `migrate` group. * @@ -29,6 +65,10 @@ export function registerMigrateSettings( command: "clerk migrate settings set firebase-signer-key abc123", description: "Save a credential to the gitignored .env.clerk-migrate", }, + { + command: "clerk migrate settings clear firebase-signer-key", + description: "Forget one setting", + }, { command: "clerk migrate settings clear -y", description: "Forget this project's settings" }, ]); @@ -47,7 +87,7 @@ export function registerMigrateSettings( settingsCommand .command("set") .description("Set one setting for this project") - .addArgument(createArgument("", "Setting to change").choices(SETTING_NAMES)) + .addArgument(settingNameArgument("", "Setting to change")) .addArgument(createArgument("", "New value")) .setExamples([ { @@ -63,13 +103,18 @@ export function registerMigrateSettings( settingsCommand .command("clear") - .description("Forget the saved settings and remove the saved credentials") + .description("Forget one saved setting, or every setting and saved credential") + .addArgument(settingNameArgument("[name]", "Setting to clear; omit to clear them all")) .option("-y, --yes", "Skip the confirmation prompt") .setExamples([ - { command: "clerk migrate settings clear", description: "Clear after confirming" }, + { command: "clerk migrate settings clear", description: "Clear everything after confirming" }, + { + command: "clerk migrate settings clear file", + description: "Forget only the remembered export file", + }, { command: "clerk migrate settings clear -y", description: "Clear without prompting" }, ]) - .action((_opts, cmd) => - settings.clear(cmd.optsWithGlobals() as Parameters[0]), + .action((name, _opts, cmd) => + settings.clear(cmd.optsWithGlobals() as Parameters[0], name), ); } diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts index 7a471e855..f3eb1ffc7 100644 --- a/packages/cli-core/src/commands/migrate/settings/registry.ts +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -142,6 +142,51 @@ export function findSetting(name: string): SettingDef | undefined { return SETTINGS.find((setting) => setting.name === name); } +/** Levenshtein distance, iterative over a single row. */ +function distance(a: string, b: string): number { + const row = Array.from({ length: b.length + 1 }, (_, i) => i); + + for (let i = 1; i <= a.length; i++) { + let diagonal = row[0] as number; + row[0] = i; + for (let j = 1; j <= b.length; j++) { + const above = row[j] as number; + row[j] = Math.min( + above + 1, + (row[j - 1] as number) + 1, + diagonal + (a[i - 1] === b[j - 1] ? 0 : 1), + ); + diagonal = above; + } + } + + return row[b.length] as number; +} + +/** + * The setting a misspelling was probably reaching for. + * + * Every setting name is a compound of short words — `log-dir`, `firebase-mem-cost` + * — so the misses that matter are a pluralised segment or a transposed pair, + * not a different word entirely. One edit per three characters keeps + * `logs-dir` pointing at `log-dir` without letting an unrelated name match + * something and send the reader off after it. + * + * @returns The closest name within that budget, or `undefined` when nothing is + * close enough to be worth naming. + */ +export function suggestSettingName(name: string): string | undefined { + const budget = Math.max(1, Math.floor(name.length / 3)); + + let best: { name: string; distance: number } | undefined; + for (const candidate of SETTING_NAMES) { + const gap = distance(name, candidate); + if (gap <= budget && (!best || gap < best.distance)) best = { name: candidate, distance: gap }; + } + + return best?.name; +} + /** * Every variable an `env` setting answers to, highest priority first. * diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts index 946d475c6..220a6cf35 100644 --- a/packages/cli-core/src/commands/migrate/settings/settings.test.ts +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -9,7 +9,7 @@ import { MIGRATE_ENV_FILE } from "../lib/env-file.ts"; import { loadSettings, saveSettings } from "../lib/settings.ts"; import { clear } from "./clear.ts"; import { list } from "./list.ts"; -import { displayValue, findSetting } from "./registry.ts"; +import { displayValue, findSetting, suggestSettingName } from "./registry.ts"; import { set } from "./set.ts"; const captured = useCaptureLog(); @@ -250,3 +250,78 @@ describe("clear", () => { expect(envFileContent()).toBe("OTHER=keep\n"); }); }); + +describe("clear ", () => { + test("drops one config setting and keeps the rest", async () => { + await set("transformer", "firebase"); + await set("file", "users.json"); + + await clear({ yes: true }, "file"); + + expect(await loadSettings()).toEqual({ transformer: "firebase" }); + }); + + test("drops one credential and keeps the rest of the env file", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + await set("firebase-rounds", "8"); + + await clear({ yes: true }, "firebase-signer-key"); + + expect(envFileContent()).toContain("CLERK_FIREBASE_ROUNDS=8"); + expect(envFileContent()).not.toContain("CLERK_FIREBASE_SIGNER_KEY"); + }); + + // Clearing only the prefixed name would report success and leave the next run + // reading the alias. + test("drops every spelling the setting answers to", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "ROUNDS=8\nOTHER=keep\n"); + + await clear({ yes: true }, "firebase-rounds"); + + expect(envFileContent()).toBe("OTHER=keep\n"); + }); + + test("says so when the setting was not set here", async () => { + await clear({ yes: true }, "firebase-rounds"); + expect(captured.err).toContain("firebase-rounds"); + expect(captured.err).toContain("is not set here"); + + captured.clear(); + await clear({ yes: true }, "transformer"); + expect(captured.err).toContain("transformer"); + expect(captured.err).toContain("is not set here"); + }); + + // `log-dir` is remembered in the config but yields to an env var, so half a + // clear would report success and leave the run reading the same directory. + test("clears a setting that lives in both stores", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "CLERK_MIGRATE_LOG_DIR=./env-logs\n"); + await saveSettings({ logDir: "./saved-logs", transformer: "firebase" }); + + await clear({ yes: true }, "log-dir"); + + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + expect(await loadSettings()).toEqual({ transformer: "firebase" }); + }); + + test("rejects a name that is not a setting", async () => { + await expect(clear({ yes: true }, "nope")).rejects.toThrow(/Unknown setting "nope"/); + }); +}); + +describe("suggestSettingName", () => { + // `.choices()` rejects before the action runs, so this is the only thing + // standing between a one-character miss and a bare list of eight names. + test.each([ + ["logs-dir", "log-dir"], + ["log_dir", "log-dir"], + ["firebase-round", "firebase-rounds"], + ["transfomer", "transformer"], + ])("%s -> %s", (typo, expected) => { + expect(suggestSettingName(typo)).toBe(expected); + }); + + test.each(["banana", "secret", ""])("says nothing for %p", (unrelated) => { + expect(suggestSettingName(unrelated)).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index fbcf6fc64..999f67d30 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -78,6 +78,7 @@ export const NEXT_STEPS = { MIGRATE_DELETE: ["Run `clerk migrate logs list` to inspect the deletion log"], MIGRATE_SETTINGS: [ "Run `clerk migrate settings set ` to change one", + "Run `clerk migrate settings clear ` to forget one", "Run `clerk migrate settings clear` to forget them all, credentials included", ], // The only parameterized entry: a suggested import is worthless unless it From 0f3574492d5a2f0e24f2dad928d64588961446c0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 11 Sep 2026 00:03:31 -0400 Subject: [PATCH 29/34] fix(migrate): warn instead of refusing when an import may exceed the dev user limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `DEV_USER_LIMIT` was 500 and enforced: an import of more users into a development instance was refused outright. Both halves were wrong. The limit a development instance is created with is 100, Clerk raises it per instance on request, and the real value (`max_allowed_users`) is served by no public API — so the number can never be known to be this instance's, and refusing blocked imports the destination would happily accept. The import now reads the live user count from `GET /v1/users/count`, measures the file against the headroom that implies, and warns when it does not fit — naming what the instance already holds and roughly how many users will be rejected. A human is asked whether to continue; `-y` and agent mode proceed on the warning alone. The final prompt then restates the split ("Import 1 user and expect 1 to fail?") rather than a number the instance will not take. The summary's error breakdown gains notes for the two errors that read as account-level restrictions and are not: blocked SMS countries (a per-instance blocklist, with development instances pointed at Clerk's test numbers and production at the Dashboard setting) and the user quota. Both messages point at "contact support", which is the wrong first move for most readers. After a partial import the next steps now lead with the grep that names which users failed and why, since the breakdown only counts each error. --- .../cli-core/src/commands/migrate/README.md | 18 ++- .../src/commands/migrate/lib/clerk-config.ts | 25 ++++ .../src/commands/migrate/lib/instance.test.ts | 4 +- .../src/commands/migrate/lib/instance.ts | 12 +- .../commands/migrate/run-interactive.test.ts | 61 ++++++-- .../cli-core/src/commands/migrate/run.test.ts | 72 +++++++-- packages/cli-core/src/commands/migrate/run.ts | 141 ++++++++++++++++-- packages/cli-core/src/lib/next-steps.ts | 10 +- 8 files changed, 297 insertions(+), 46 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 1fbf3e1ea..4c70a6517 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -119,8 +119,21 @@ that, assuming ~100ms of API latency. Both are overridable: A non-numeric or non-positive value is ignored in favour of the default. -**Development instances refuse imports over 500 users**, matching Clerk's own -limit — the run fails before any request is sent. +**Development instances warn when an import may exceed their user limit.** New +development instances are created with a 100-user limit; production instances +have none. Before importing, the run reads the instance's current user count +(`GET /v1/users/count`) and warns when the file would take it past 100. + +The run then stops and asks before going ahead. It is a prompt rather than a +hard refusal because the number checked against may not be this instance's: +Clerk raises a development instance's limit on request, and the raised value +(`max_allowed_users`) is not served by BAPI, DAPI or FAPI — so the CLI can show +the live count but never the live limit. Declining aborts before anything is +written to Clerk; `-y` and agent mode proceed on the warning alone. + +Users that do exceed the limit come back in the error breakdown as +`You have reached your limit of N users`, annotated with what a development +instance can do about it. ### `clerk migrate export` @@ -1097,6 +1110,7 @@ NDJSON is. The original `.log` stays put. | `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | | `GET` | `/v1/users?external_id=…` | `migrate delete` — finds this migration's users, 100 IDs a call | | `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | | `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | | `GET` | `/v1/domains` | Readiness report and `--skip-unsupported-providers` — resolves the Frontend API host | diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts index 58bb6dceb..f69ae7fab 100644 --- a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -117,3 +117,28 @@ export async function fetchEnabledSocialProviders(secretKey: string): Promise { + try { + const response = await bapiRequest({ method: "GET", path: "/v1/users/count", secretKey }); + const total = (response.body as { total_count?: unknown })?.total_count; + return typeof total === "number" ? total : null; + } catch (error) { + log.debug( + `migrate: could not read the instance's user count: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return null; + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/instance.test.ts b/packages/cli-core/src/commands/migrate/lib/instance.test.ts index 6f6330f56..ef95b0b92 100644 --- a/packages/cli-core/src/commands/migrate/lib/instance.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/instance.test.ts @@ -35,8 +35,8 @@ describe("default limits", () => { expect(getDefaultConcurrencyLimit(rateLimit)).toBe(expected); }); - test("development instances are capped at 500 users", () => { - expect(DEV_USER_LIMIT).toBe(500); + test("development instances default to 100 users", () => { + expect(DEV_USER_LIMIT).toBe(100); }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/instance.ts b/packages/cli-core/src/commands/migrate/lib/instance.ts index d094c11cf..24f222a9d 100644 --- a/packages/cli-core/src/commands/migrate/lib/instance.ts +++ b/packages/cli-core/src/commands/migrate/lib/instance.ts @@ -6,8 +6,16 @@ * `resolveBapiSecretKey`, and only the two override knobs read the environment. */ -/** Development instances are capped at this many users by Clerk. */ -export const DEV_USER_LIMIT = 500; +/** + * The user limit a development instance is created with. + * + * Only a default: Clerk raises it per instance on request, and the real value + * (`max_allowed_users`) is not served by BAPI, DAPI or FAPI — only by Clerk's + * internal staff API. So this is a number to warn against, never one to refuse + * an import over; the instance in front of you may be allowed far more. + * Production instances have no limit at all. + */ +export const DEV_USER_LIMIT = 100; /** How many times a 429 is retried before the user is recorded as failed. */ export const MAX_RETRIES = 5; diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index a7e6909c8..1c7728c1d 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -59,7 +59,10 @@ mock.module("../../lib/keyless-target.ts", () => ({ // Every export of the real module must appear here — a missing one is a link // error at import time, which takes down the whole file rather than one prompt. mock.module("../../lib/prompts.ts", () => ({ - confirm: async () => confirmAnswer, + confirm: async ({ message }: { message: string }) => { + confirmMessages.push(message); + return confirmAnswer; + }, multiselect: (...args: unknown[]) => mockMultiselect(...(args as [MultiselectConfig])), text: (...args: unknown[]) => mockText(...(args as [])), password: async () => "", @@ -119,6 +122,7 @@ afterAll(() => { beforeEach(() => { requests = []; confirmAnswer = true; + confirmMessages = []; instanceTarget = ACCOUNT_TARGET; mockSelect.mockReset(); mockText.mockReset(); @@ -471,18 +475,51 @@ describe("fixing the instance's settings from the report", () => { }); describe("guards that still apply interactively", () => { - test("the dev-instance 500-user cap", async () => { - fs.writeFileSync( - path.join(workDir, "export.json"), - JSON.stringify( - Array.from({ length: 501 }, (_, i) => ({ - id: `u${i}`, - primary_email_address: `u${i}@x.dev`, - })), - ), - ); + /** Makes `GET /v1/users/count` report an instance with one seat left. */ + function stubNearlyFullInstance(): void { + const inner = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (input.toString().includes("/v1/users/count")) { + return Response.json({ object: "total_count", total_count: 99 }); + } + return inner(input, init); + }) as typeof fetch; + } + + test("the dev-instance user limit, which the operator can agree to import past", async () => { + stubNearlyFullInstance(); + confirmAnswer = true; + + await run(baseOptions); + + expect(captured.err).toContain("100-user limit"); + expect(created()).toHaveLength(2); + }); - await expect(run(baseOptions)).rejects.toThrow(/development instance/); + // Asking "Import 2 users?" after warning that one of them cannot fit is the + // report and the quota disagreeing in the same run. + test("the final prompt restates the quota split rather than the file size", async () => { + stubNearlyFullInstance(); + + await run(baseOptions); + + expect(confirmMessages).toContain("Import 1 user and expect 1 to fail?"); + }); + + test("the final prompt names the whole file when the quota is not in play", async () => { + await run(baseOptions); + + expect(confirmMessages).toContain("Import 2 users?"); + }); + + test("declining the user-limit prompt writes nothing to Clerk", async () => { + stubNearlyFullInstance(); + confirmAnswer = false; + + await expect(run(baseOptions)).rejects.toThrow(UserAbortError); + + // Aborted before the readiness report, so nothing was read from FAPI either. + expect(captured.err).not.toContain("Migration readiness"); expect(created()).toHaveLength(0); }); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 049a553ca..0dd898c03 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -8,7 +8,7 @@ import { useCaptureLog } from "../../test/lib/stubs.ts"; import { getLogDir } from "./lib/logger.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; import { loadSettings } from "./lib/settings.ts"; -import { applyResumeAfter, run, validateRunOptions } from "./run.ts"; +import { applyResumeAfter, explainErrors, run, validateRunOptions } from "./run.ts"; import type { User } from "./types.ts"; let workDir: string; @@ -183,19 +183,35 @@ describe("run", () => { expect(captured.err).toContain("1 user failed validation"); }); - test("refuses to exceed the development-instance user limit", async () => { - fs.writeFileSync( - path.join(workDir, "export.json"), - JSON.stringify( - Array.from({ length: 501 }, (_, i) => ({ - id: `u${i}`, - primary_email_address: `u${i}@x.dev`, - })), - ), - ); + /** Makes `GET /v1/users/count` report an instance that already holds users. */ + function stubUserCount(total: number): void { + const inner = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (input.toString().includes("/v1/users/count")) { + return Response.json({ object: "total_count", total_count: total }); + } + return inner(input, init); + }) as typeof fetch; + } - await expect(run(baseOptions)).rejects.toThrow(/development instance/); - expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + // `baseOptions` passes -y, which has nobody to answer the prompt this warning + // otherwise raises — see run-interactive.test.ts for the prompt itself. + test("warns under -y when an import may exceed the development-instance user limit", async () => { + stubUserCount(99); + + await run(baseOptions); + + expect(captured.err).toContain("100-user limit"); + expect(captured.err).toContain("already holds 99"); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + }); + + test("stays quiet when the instance has room for the whole file", async () => { + stubUserCount(10); + + await run(baseOptions); + + expect(captured.err).not.toContain("100-user limit"); }); test("aborts before any API call when the hasher is unrecognized", async () => { @@ -685,3 +701,33 @@ describe("run", () => { }); }); }); + +describe("explainErrors", () => { + const COUNTRY = + "Phone numbers from this country (France) are currently not supported. For more information, please contact support."; + const QUOTA = + "You have reached your limit of 100 users. If you need more users, please use a Production instance."; + + test("names the development instance as the reason countries are blocked", () => { + const [note] = explainErrors([COUNTRY], "dev"); + expect(note).toContain("Development instances block SMS to most countries"); + expect(note).toContain("test-emails-and-phones"); + }); + + test("sends a production operator to the Dashboard instead of support", () => { + const [note] = explainErrors([COUNTRY], "prod"); + expect(note).toContain("customization/sms/settings"); + expect(note).not.toContain("Development instances"); + }); + + test("explains the user quota only where one applies", () => { + expect(explainErrors([QUOTA], "dev").join(" ")).toContain("development-instance quota"); + // Production has no such quota, and the API's message already names the + // plan upgrade in the one case it does. + expect(explainErrors([QUOTA], "prod")).toEqual([]); + }); + + test("says nothing about errors it does not recognize", () => { + expect(explainErrors(["Something else went wrong."], "dev")).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index dd8b6b467..57e374f35 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -28,6 +28,7 @@ import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebas import { enabledSocialProviders, fetchInstanceSettings, + fetchUserCount, toClerkStrategy, } from "./lib/clerk-config.ts"; import { @@ -42,7 +43,7 @@ import { buildSettingChanges, type SettingChange, } from "./lib/modify-settings.ts"; -import { DEV_USER_LIMIT, resolveLimits } from "./lib/instance.ts"; +import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; import { startLogging, getLogFilePath } from "./lib/logger.ts"; import { saveSettings } from "./lib/settings.ts"; import { @@ -168,7 +169,59 @@ export function applyResumeAfter(users: User[], resumeAfter: string | undefined) return users.slice(index + 1); } -function formatSummary(summary: ImportSummary, logFile: string): string { +/** Where a production instance's operator changes the SMS country blocklist. */ +const SMS_SETTINGS_URL = "https://dashboard.clerk.com/~/customization/sms/settings"; + +/** Clerk's fictional email addresses and phone numbers, for development. */ +const TEST_NUMBERS_URL = "https://clerk.com/docs/guides/development/testing/test-emails-and-phones"; + +/** + * What the API's error messages leave out: whether the operator can do + * something about them, and where. + * + * Both of these read as account-level restrictions and are not. Blocked + * countries are a per-instance SMS blocklist that development instances are + * created with far more of, and the user limit is a development-instance quota + * that production does not have at all — so "contact support", which both + * messages point at, is the wrong first move for most readers. + * + * @returns One note per recognized error family, empty when none apply. + */ +export function explainErrors(errors: Iterable, instanceType: InstanceType): string[] { + const all = [...errors]; + const notes: string[] = []; + + if (all.some((error) => error.includes("Phone numbers from this country"))) { + notes.push( + instanceType === "dev" + ? `Development instances block SMS to most countries by default — this is not a limit on your account. ` + + `Use Clerk's test phone numbers while developing (${TEST_NUMBERS_URL}), and contact support only if ` + + `you need real numbers in a specific country before going to production.` + : `Unblock the countries you need under SMS settings in the Dashboard (${SMS_SETTINGS_URL}). ` + + `Plans without SMS support cannot remove them; contact support if the setting is refused.`, + ); + } + + // Production has no user limit unless a plan imposes one, and the API's own + // message already names the fix ("upgrade to a paid plan") in that case. + if ( + instanceType === "dev" && + all.some((error) => /You have reached your limit of \d+ users/.test(error)) + ) { + notes.push( + `The user limit is a development-instance quota (${DEV_USER_LIMIT} by default). Import into a production ` + + `instance to bring everyone across, or contact support to raise this instance's limit.`, + ); + } + + return notes; +} + +function formatSummary( + summary: ImportSummary, + logFile: string, + instanceType: InstanceType, +): string { const inFile = summary.totalProcessed + summary.validationFailed; const lines = [ `${bold("Total users in file:")} ${inFile}`, @@ -184,12 +237,66 @@ function formatSummary(summary: ImportSummary, logFile: string): string { for (const [error, count] of summary.errorBreakdown) { lines.push(` ${count} user${count === 1 ? "" : "s"}: ${error}`); } + for (const note of explainErrors(summary.errorBreakdown.keys(), instanceType)) { + lines.push("", note); + } } lines.push("", dim(`Log: ${logFile}`)); return lines.join("\n"); } +/** + * Stops an import that looks likely to exhaust a development instance's user + * quota, and asks before letting it through anyway. + * + * A prompt rather than a hard refusal, because the number it checks against + * cannot be trusted to be this instance's: {@link DEV_USER_LIMIT} is only what + * an instance is *created* with, Clerk raises it per instance on request, and + * no public endpoint serves the real value. The existing user count is live; + * the limit it is measured against is not. Refusing outright would block + * imports the destination would happily accept, so the operator — who can ask + * Clerk what their limit is — gets the last word. + * + * `-y` and agent mode proceed on the warning alone, matching the import + * confirmation below: neither has anyone to answer the question. + * + * @returns How many of `incoming` the quota is expected to reject, or `0` when + * the whole file fits. The final import prompt reports the same split, so + * that "yes" is never a bigger number than the instance will accept. + * @throws UserAbortError when the operator declines. + */ +async function confirmDevUserLimit( + incoming: number, + secretKey: string, + yes: boolean, +): Promise { + const existing = await withSpinner("Checking the instance's user count...", async () => + fetchUserCount(secretKey), + ); + const headroom = Math.max(0, DEV_USER_LIMIT - (existing ?? 0)); + if (incoming <= headroom) return 0; + + const rejected = incoming - headroom; + const held = existing === null ? "" : `, and this one already holds ${existing}`; + log.warn( + `Development instances default to a ${DEV_USER_LIMIT}-user limit${held}. About ${rejected} of the ` + + `${incoming} user${incoming === 1 ? "" : "s"} in this file will be rejected with a quota error unless ` + + `Clerk has raised this instance's limit — the limit itself is not readable from the API.\n` + + `Import into a production instance to bring everyone across, or contact support to raise the limit.`, + ); + + if (yes || !isHuman() || isAgent()) return rejected; + + const proceed = await confirm({ + message: `Continue anyway, expecting about ${rejected} user${rejected === 1 ? "" : "s"} to be rejected?`, + default: false, + }); + if (!proceed) throwUserAbort(); + + return rejected; +} + /** * Drops users whose only way into Clerk is a social provider the destination * instance has not enabled. @@ -522,13 +629,10 @@ export async function run(rawOptions: MigrateRunOptions): Promise { return; } - if (limits.instanceType === "dev" && users.length > DEV_USER_LIMIT) { - throw new CliError( - `Cannot import ${users.length} users into a development instance — the limit is ${DEV_USER_LIMIT}.\n` + - "Target a production instance, or reduce the import file.", - { code: ERROR_CODE.USAGE_ERROR }, - ); - } + const quotaRejections = + limits.instanceType === "dev" + ? await confirmDevUserLimit(users.length, secretKey, Boolean(options.yes)) + : 0; // `target` already carries the instance's environment ("My App // (development)"), so the detected type is only worth spelling out when @@ -549,8 +653,15 @@ export async function run(rawOptions: MigrateRunOptions): Promise { }); if (!options.yes && isHuman() && !isAgent()) { + // The readiness report counts the whole file, because settings decide + // what Clerk *accepts*. The quota decides how much of it gets in at all, + // so the last prompt — the one that starts writing — restates that split + // rather than asking about a number the instance will not take. + const importable = users.length - quotaRejections; const proceed = await confirm({ - message: `Import ${users.length} user${users.length === 1 ? "" : "s"}?`, + message: quotaRejections + ? `Import ${importable} user${importable === 1 ? "" : "s"} and expect ${quotaRejections} to fail?` + : `Import ${users.length} user${users.length === 1 ? "" : "s"}?`, default: false, }); if (!proceed) throwUserAbort(); @@ -576,11 +687,15 @@ export async function run(rawOptions: MigrateRunOptions): Promise { }), ); - log.info(formatSummary(summary, logFile)); + log.info(formatSummary(summary, logFile, limits.instanceType)); // Offered even when some users failed: a partial import is exactly when - // reading the log and knowing how to undo it matters most. - setNextSteps(NEXT_STEPS.MIGRATE_DONE); + // reading the log and knowing how to undo it matters most. When users did + // fail, the per-user record of *why* leads, since the breakdown above only + // counts each error and never names who hit it. + setNextSteps( + summary.failed > 0 ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(logFile) : NEXT_STEPS.MIGRATE_DONE, + ); if (summary.failed > 0) process.exitCode = 1; }); diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index 999f67d30..35c34dfdb 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -75,14 +75,20 @@ export const NEXT_STEPS = { "Run `clerk migrate logs list` to inspect the import log", "Run `clerk migrate delete` to undo this migration", ], + // `logs list` only names the file; after a partial import the operator needs + // the failures themselves, which live one line per user in that file. + MIGRATE_DONE_WITH_ERRORS: (logFile: string) => [ + `Run \`grep '"status":"error"' ${logFile}\` to see every user that failed and why`, + "Run `clerk migrate delete` to undo this migration", + ], MIGRATE_DELETE: ["Run `clerk migrate logs list` to inspect the deletion log"], MIGRATE_SETTINGS: [ "Run `clerk migrate settings set ` to change one", "Run `clerk migrate settings clear ` to forget one", "Run `clerk migrate settings clear` to forget them all, credentials included", ], - // The only parameterized entry: a suggested import is worthless unless it - // names the transformer that reads this export and the file just written. + // A suggested import is worthless unless it names the transformer that reads + // this export and the file just written. MIGRATE_EXPORT: (transformerKey: string, file: string) => [ `Run \`clerk migrate import --transformer ${transformerKey} --file ${file}\` to import them`, ], From 0faebbc3abeb5974509d5ec43de5e634df3735f7 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 11 Sep 2026 00:03:42 -0400 Subject: [PATCH 30/34] feat(migrate): sign in and link before an import starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this the first complaint came from deep inside the secret-key chain, which resolves the linked profile before it ever asks for a token — so a signed-out operator in an unlinked directory was told to run `clerk link`, a command that would only turn around and ask them to sign in. Both failures landed after the wizard had already walked them through picking a platform and a file. `migrate import` now checks for somewhere to import *into* first, mirroring `resolveBapiSecretKey`: `--secret-key`, `--app`, `CLERK_SECRET_KEY` and an unclaimed accountless application each name the destination on their own. A human gets the same sign-in-then-link flow `clerk link` already runs; an agent, which can answer neither a browser login nor an application picker, gets an error naming whichever half is missing. --- .../cli-core/src/commands/migrate/run.test.ts | 21 +++++- packages/cli-core/src/commands/migrate/run.ts | 71 ++++++++++++++++++- 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 0dd898c03..f7e5364c2 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -1,10 +1,14 @@ -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { _setConfigDir } from "../../lib/config.ts"; import { CliError } from "../../lib/errors.ts"; -import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +// Every test below names its own `--secret-key`, which short-circuits the +// signed-in check — except the one that asserts what happens without it. +mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); import { getLogDir } from "./lib/logger.ts"; import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; import { loadSettings } from "./lib/settings.ts"; @@ -126,6 +130,19 @@ describe("run", () => { secretKey: "sk_test_x", }; + test("refuses before the wizard when nobody is signed in", async () => { + const previous = process.env.CLERK_SECRET_KEY; + delete process.env.CLERK_SECRET_KEY; + try { + await expect(run({ transformer: "clerk", file: "export.json", yes: true })).rejects.toThrow( + /Not logged in/, + ); + expect(requests).toHaveLength(0); + } finally { + if (previous !== undefined) process.env.CLERK_SECRET_KEY = previous; + } + }); + test("imports every user in the file end to end", async () => { await run(baseOptions); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 57e374f35..8ef97230b 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -14,8 +14,21 @@ import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; import { bold, dim, green, red, yellow } from "../../lib/color.ts"; -import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; -import { resolveInstanceTarget, type InstanceTarget } from "../../lib/keyless-target.ts"; +import { resolveProfile } from "../../lib/config.ts"; +import { hasAccountCredentials } from "../../lib/credential-store.ts"; +import { + AUTH_ERROR_REASON, + AuthError, + CliError, + ERROR_CODE, + throwUsageError, + throwUserAbort, +} from "../../lib/errors.ts"; +import { + resolveInstanceTarget, + resolveKeylessTarget, + type InstanceTarget, +} from "../../lib/keyless-target.ts"; import { log } from "../../lib/log.ts"; import { NEXT_STEPS } from "../../lib/next-steps.ts"; import { confirm, multiselect } from "../../lib/prompts.ts"; @@ -57,6 +70,8 @@ import { loadCustomTransformer } from "./transformers/load-custom.ts"; import { registerCustomTransformer, transformerKeys } from "./transformers/registry.ts"; import type { ImportSummary, User } from "./types.ts"; import { runWizard, throwAgentFlagsRequired } from "./wizard.ts"; +import { login } from "../auth/login.ts"; +import { link } from "../link/index.ts"; export type MigrateRunOptions = { transformer?: string; @@ -579,7 +594,59 @@ async function applyCustomTransformer(options: MigrateRunOptions): Promise { + // Each of these names the destination instance on its own, with no account + // and no linked directory involved — mirroring resolveBapiSecretKey. + if (options.secretKey || options.app || process.env.CLERK_SECRET_KEY) return; + // An unclaimed accountless application keeps its only secret key on disk. + if (await resolveKeylessTarget({ instance: options.instance })) return; + + const interactive = isHuman() && !isAgent(); + + if (!(await hasAccountCredentials())) { + if (!interactive) { + throw new AuthError({ + reason: AUTH_ERROR_REASON.NOT_LOGGED_IN, + message: + "Not logged in, so there is no Clerk instance to import into. Run `clerk auth login`, then `clerk link`.", + examples: [ + { command: "clerk auth login", description: "Sign in, then re-run the import" }, + { + command: + "clerk migrate import -y --secret-key sk_test_... --transformer clerk --file users.json", + description: "Import without signing in", + }, + ], + }); + } + log.info("Not logged in. Signing in first..."); + await login({ showNextSteps: false }); + } + + // Left to the secret-key chain when non-interactive: its `not_linked` error + // is the one every other command raises, and there is nothing to add to it. + if (interactive && !(await resolveProfile(process.cwd()))) { + log.info("This directory isn't linked to a Clerk application. Linking one first..."); + await link({ skipIfLinked: true }); + } +} + export async function run(rawOptions: MigrateRunOptions): Promise { + await ensureImportTarget(rawOptions); rawOptions = await applyCustomTransformer(rawOptions); const options = await resolveMissingOptions(rawOptions); From 4bf8048c68223df2bb38614cd88086bed53f1723 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 11 Sep 2026 00:03:54 -0400 Subject: [PATCH 31/34] feat(migrate): accept libsql/Turso connection strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--db-url "libsql://app-org.turso.io"` fell through to the SQLite default and `bun:sqlite` tried to open a local file by that name. A libsql URL now routes to the server's HTTP pipeline endpoint instead: `bun:sqlite` only opens local files, and `@libsql/client` ships native optional dependencies that do not survive `bun build --compile`, so the wire protocol is fewer lines than the dependency would be. The client reports itself as `sqlite`, since that is the dialect — nothing downstream branches differently. The token comes from `?authToken=` on the URL, the form the Turso CLI prints, or from `TURSO_AUTH_TOKEN`/`LIBSQL_AUTH_TOKEN`; a self-hosted sqld with auth disabled needs neither. Redaction covers the query parameter as well as userinfo, so a token cannot reach an error message or `--verbose` output, and a 401 is explained rather than left as a bare status. --- .../cli-core/src/commands/migrate/README.md | 30 ++-- .../src/commands/migrate/export/authjs.ts | 2 +- .../src/commands/migrate/export/betterauth.ts | 2 +- .../src/commands/migrate/export/db-options.ts | 11 +- .../src/commands/migrate/export/index.ts | 2 +- .../src/commands/migrate/lib/db.test.ts | 71 +++++++++ .../cli-core/src/commands/migrate/lib/db.ts | 140 +++++++++++++++++- 7 files changed, 235 insertions(+), 23 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 4c70a6517..9a4d001e0 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -177,14 +177,14 @@ like every other path flag here. The question comes before any users are fetched, so a long export can be left unattended rather than stalling on a prompt with everything held in memory. -| Flag | Platforms | Description | -| -------------------------- | ---------------------------------- | -------------------------------------------- | -| `-o, --output ` | all | Where to write the export | -| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL or SQLite connection string | -| `--service-account ` | `firebase` | Path to a service account key JSON file | -| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | -| `--client-id ` | `auth0` | Machine-to-machine application client ID | -| `--client-secret ` | `auth0` | Machine-to-machine application client secret | +| Flag | Platforms | Description | +| -------------------------- | ---------------------------------- | --------------------------------------------------------- | +| `-o, --output ` | all | Where to write the export | +| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL, libsql/Turso or SQLite connection string | +| `--service-account ` | `firebase` | Path to a service account key JSON file | +| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | +| `--client-id ` | `auth0` | Machine-to-machine application client ID | +| `--client-secret ` | `auth0` | Machine-to-machine application client secret | `export clerk` also takes the targeting flags — it reads from a Clerk instance, so it resolves a key the same way `clerk migrate import` does, with one extra @@ -249,13 +249,17 @@ These three read the database directly, over **`--db-url`**: clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" clerk migrate export authjs --db-url "mysql://user:...@127.0.0.1:3306/authjs" clerk migrate export betterauth --db-url "./db.sqlite" +clerk migrate export betterauth --db-url "libsql://app-org.turso.io?authToken=..." # or set TURSO_AUTH_TOKEN ``` -Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`. Both are -built into the runtime, so nothing native ships in the binary — that is the -whole reason the `engines.bun` floor exists. Resolution is `--db-url`, then -`SUPABASE_DB_URL` / `AUTHJS_DB_URL` / `BETTERAUTH_DB_URL`, then a masked prompt, -since a connection string carries the password inline. +Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; +`libsql://` (Turso) over the server's HTTP pipeline endpoint, since `bun:sqlite` +only opens local files and `@libsql/client` ships native optional dependencies. +Nothing native ships in the binary — that is the whole reason the `engines.bun` +floor exists. Resolution is `--db-url`, then `SUPABASE_DB_URL` / `AUTHJS_DB_URL` +/ `BETTERAUTH_DB_URL`, then a masked prompt, since a connection string carries +the password inline. A libsql token comes from `?authToken=` on the URL, or from +`TURSO_AUTH_TOKEN` / `LIBSQL_AUTH_TOKEN`, and is redacted like a password. **Connection strings are redacted everywhere.** Errors show `postgres://***@host/db`, including when the password itself contains an diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index d378c93c1..d9b3f2c72 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -110,7 +110,7 @@ export async function exportAuthJs(options: DbExportOptions): Promise { platform: "authjs", envVar: "AUTHJS_DB_URL", prompt: "Auth.js database connection string", - hint: "Postgres, MySQL or a SQLite file — whichever your Auth.js adapter uses.", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Auth.js adapter uses.", }); const destination = await resolveOutputPath("authjs", options.output); diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index f2d26c6c6..348271a6d 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -157,7 +157,7 @@ export async function exportBetterAuth(options: DbExportOptions): Promise platform: "betterauth", envVar: "BETTERAUTH_DB_URL", prompt: "Better Auth database connection string", - hint: "Postgres, MySQL or a SQLite file — whichever your Better Auth install uses.", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Better Auth install uses.", }); const destination = await resolveOutputPath("betterauth", options.output); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index e2957697e..3ef77efbe 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -10,7 +10,7 @@ import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isAgent, isHuman } from "../../../mode.ts"; -import { detectDbType, redactConnectionString, type DbPlatform } from "../lib/db.ts"; +import { detectDbType, isLibsqlUrl, redactConnectionString, type DbPlatform } from "../lib/db.ts"; import { findMigrateEnvValue } from "../lib/env-file.ts"; export type DbExportOptions = { @@ -27,7 +27,7 @@ type ResolveConfig = { hint?: string; }; -const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2):\/\//i; +const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2|libsql):\/\//i; /** * True when the string parses as a URL with a host. @@ -103,7 +103,7 @@ export async function resolveDbUrl( if (fromFlag) { if (!looksLikeConnectionString(fromFlag)) { throwUsageError( - `--db-url does not look like a connection string. Expected postgres://…, mysql://… or a SQLite file path.\n` + + `--db-url does not look like a connection string. Expected postgres://…, mysql://…, libsql://… or a SQLite file path.\n` + "If the password contains @, # or /, URL-encode it.", ); } @@ -140,7 +140,7 @@ export async function resolveDbUrl( validate: (value) => looksLikeConnectionString(normalizeConnectionString(value ?? "")) ? undefined - : "Expected postgres://…, mysql://… or a SQLite file path", + : "Expected postgres://…, mysql://…, libsql://… or a SQLite file path", }); return normalizeConnectionString(answer); @@ -148,5 +148,6 @@ export async function resolveDbUrl( /** Describes the target for the run's opening line, credentials removed. */ export function describeTarget(connectionString: string): string { - return `${detectDbType(connectionString)} at ${redactConnectionString(connectionString)}`; + const label = isLibsqlUrl(connectionString) ? "libsql" : detectDbType(connectionString); + return `${label} at ${redactConnectionString(connectionString)}`; } diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index b11ea8814..e72066676 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -171,7 +171,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record.json)`, ) - .option("--db-url ", "Postgres, MySQL or SQLite connection string") + .option("--db-url ", "Postgres, MySQL, libsql/Turso or SQLite connection string") .option("-o, --output ", "Where to write the export, relative to the current directory") .setExamples([ { diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index 3fc969ee5..05ae45928 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -39,6 +39,7 @@ describe("detectDbType", () => { ["mysql://u:p@h/db", "mysql"], ["mysql2://u:p@h/db", "mysql"], ["./db.sqlite", "sqlite"], + ["libsql://app-org.turso.io", "sqlite"], ["file:./db.sqlite", "sqlite"], ["/abs/path.db", "sqlite"], [" postgres://u:p@h/db ", "postgres"], @@ -52,6 +53,7 @@ describe("redactConnectionString", () => { ["postgres://user:secret@host:5432/db", "postgres://***@host:5432/db"], ["mysql://root:hunter2@127.0.0.1:3306/app", "mysql://***@127.0.0.1:3306/app"], ["postgres://host/db", "postgres://host/db"], + ["libsql://app.turso.io?authToken=secret", "libsql://app.turso.io?authToken=***"], ])("%s -> %s", (input, expected) => { expect(redactConnectionString(input)).toBe(expected); }); @@ -89,6 +91,75 @@ describe("sqlitePath", () => { }); }); +describe("a libsql client", () => { + const originalFetch = globalThis.fetch; + let requests: { url: string; token?: string; body: any }[] = []; + + function stubFetch(result: unknown) { + requests = []; + globalThis.fetch = (async (url: string, init: RequestInit) => { + requests.push({ + url: String(url), + token: (init.headers as Record).authorization, + body: JSON.parse(String(init.body)), + }); + return new Response(JSON.stringify({ results: [result, { type: "ok" }] }), { + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + } + + const okRows = (cols: string[], rows: unknown[][]) => ({ + type: "ok", + response: { type: "execute", result: { cols: cols.map((name) => ({ name })), rows } }, + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + test("posts to the pipeline endpoint with the URL's token and decodes rows", async () => { + stubFetch( + okRows( + ["id", "count", "verified", "missing", "hash"], + [ + [ + { type: "text", value: "u1" }, + { type: "integer", value: "12" }, + { type: "float", value: 1.5 }, + { type: "null" }, + { type: "blob", base64: Buffer.from("hash").toString("base64") }, + ], + ], + ), + ); + + const client = await createDbClient("libsql://app-org.turso.io?authToken=t0ken"); + const rows = await client.query('SELECT * FROM "user" WHERE id = ?', ["u1"]); + await client.close(); + + expect(requests[0]?.url).toBe("https://app-org.turso.io/v2/pipeline"); + expect(requests[0]?.token).toBe("Bearer t0ken"); + expect(requests.at(-1)?.body.requests[0].stmt.args).toEqual([{ type: "text", value: "u1" }]); + expect(rows).toEqual([ + { + id: "u1", + count: 12, + verified: 1.5, + missing: null, + hash: Buffer.from("hash"), + }, + ] as never); + expect(client.dbType).toBe("sqlite"); + }); + + test("reports a server-side error", async () => { + stubFetch({ type: "error", error: { message: "no such table: user" } }); + + await expect(createDbClient("libsql://app-org.turso.io")).rejects.toThrow(CliError); + }); +}); + describe("a sqlite client", () => { test("connects and queries", async () => { const client = await createDbClient(dbPath); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index d048a3348..64191c125 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -36,7 +36,8 @@ export interface DbClient { * * Anything that is not a recognized URL scheme is treated as a SQLite path, * matching how the standalone tool behaved and how users actually pass - * `./db.sqlite`. + * `./db.sqlite`. `libsql://` (Turso) is SQLite too — it only differs in how + * the rows are fetched, so callers that branch on the dialect want "sqlite". */ export function detectDbType(connectionString: string): DbType { const lower = connectionString.trim().toLowerCase(); @@ -45,6 +46,11 @@ export function detectDbType(connectionString: string): DbType { return "sqlite"; } +/** True for a remote libsql/Turso URL, which is read over HTTP rather than opened. */ +export function isLibsqlUrl(connectionString: string): boolean { + return /^libsql:\/\//i.test(connectionString.trim()); +} + /** * Replaces any credentials in a connection string with `***`. * @@ -58,7 +64,10 @@ export function redactConnectionString(connectionString: string): string { // the rest of the password in the message. Everything before the final `@` // is userinfo, so redacting all of it is always safe. // Non-URL forms (SQLite paths) have no `://` and are left alone. - return connectionString.replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@"); + // Turso carries its credential as `?authToken=`, not as userinfo. + return connectionString + .replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@") + .replace(/([?&]authToken=)[^&]*/gi, "$1***"); } /** Strips a `file:` prefix and any URL query, leaving a filesystem path. */ @@ -93,6 +102,120 @@ function bunSqlClient(connectionString: string, dbType: "postgres" | "mysql"): D }; } +/** + * One value in Hrana's wire format, the protocol libsql servers speak. + * + * Integers arrive as strings so 64-bit values survive JSON. + */ +type HranaValue = { type: string; value?: string | number; base64?: string }; + +function decodeHrana(value: HranaValue): unknown { + switch (value.type) { + case "null": + return null; + case "integer": { + const raw = String(value.value ?? "0"); + const asNumber = Number(raw); + // Past 2^53 a number would silently lose digits; ids can get that big. + return Number.isSafeInteger(asNumber) ? asNumber : BigInt(raw); + } + case "float": + return Number(value.value); + case "blob": + // bun:sqlite hands back bytes for a BLOB, so this does too. + return Buffer.from(value.base64 ?? "", "base64"); + default: + return value.value ?? null; + } +} + +function encodeHrana(param: unknown): HranaValue { + if (param === null || param === undefined) return { type: "null" }; + if (typeof param === "bigint") return { type: "integer", value: param.toString() }; + if (typeof param === "boolean") return { type: "integer", value: param ? "1" : "0" }; + if (typeof param === "number") { + return Number.isInteger(param) + ? { type: "integer", value: String(param) } + : { type: "float", value: param }; + } + if (param instanceof Uint8Array) { + return { type: "blob", base64: Buffer.from(param).toString("base64") }; + } + return { type: "text", value: String(param) }; +} + +/** + * Talks to a libsql server (Turso) over its HTTP pipeline endpoint. + * + * `bun:sqlite` opens local files and cannot reach a remote database, and + * `@libsql/client` ships native optional dependencies that do not survive + * `bun build --compile`. The protocol is one POST per statement, so it is + * fewer lines to speak it directly than to carry the dependency. + * + * The token comes from `?authToken=` on the URL — the form the Turso CLI + * prints — or from `TURSO_AUTH_TOKEN`/`LIBSQL_AUTH_TOKEN`. A self-hosted sqld + * with auth disabled needs neither, so a missing token is not an error here. + */ +function libsqlClient( + connectionString: string, + env: Record = process.env, +): DbClient { + const url = new URL(connectionString.trim()); + const token = url.searchParams.get("authToken") || env.TURSO_AUTH_TOKEN || env.LIBSQL_AUTH_TOKEN; + const endpoint = `https://${url.host}/v2/pipeline`; + + return { + dbType: "sqlite", + async query>(query: string, params: unknown[] = []) { + const response = await fetch(endpoint, { + method: "POST", + headers: { + "content-type": "application/json", + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + // `close` keeps every request stateless: no baton to carry forward. + body: JSON.stringify({ + requests: [ + { type: "execute", stmt: { sql: query, args: params.map(encodeHrana) } }, + { type: "close" }, + ], + }), + }); + + if (!response.ok) { + throw new Error(`libsql request failed: ${response.status} ${response.statusText}`.trim()); + } + + const body = (await response.json()) as { + results?: { + type: string; + error?: { message?: string }; + response?: { result?: { cols?: { name?: string }[]; rows?: HranaValue[][] } }; + }[]; + }; + + const first = body.results?.[0]; + if (!first || first.type === "error") { + throw new Error(first?.error?.message ?? "libsql returned no result"); + } + + const cols = first.response?.result?.cols ?? []; + const rows = first.response?.result?.rows ?? []; + return rows.map( + (row) => + Object.fromEntries( + row.map((value, index) => [cols[index]?.name ?? String(index), decodeHrana(value)]), + ) as T, + ); + }, + placeholder: () => "?", + quote: QUOTING.sqlite, + close() { + return Promise.resolve(); + }, + }; +} + function sqliteClient(connectionString: string): DbClient { const database = new Database(sqlitePath(connectionString), { readonly: true }); @@ -124,6 +247,12 @@ export async function createDbClient( const dbType = detectDbType(connectionString); try { + if (isLibsqlUrl(connectionString)) { + const client = libsqlClient(connectionString); + await client.query("SELECT 1"); + return client; + } + if (dbType === "sqlite") { const client = sqliteClient(connectionString); // bun:sqlite opens lazily, so a missing file would not surface until the @@ -166,6 +295,13 @@ export function describeDbError(error: unknown, platform?: DbPlatform): string { return "Could not reach the database. Check the host and port, and that the server accepts connections from here."; } + if (/\b401\b|unauthorized|not authorized/i.test(message)) { + return ( + "The libsql server rejected that token.\n" + + "Append ?authToken=… to the URL, or set TURSO_AUTH_TOKEN (`turso db tokens create `)." + ); + } + if (/password authentication failed|access denied/i.test(message)) { return "The database rejected those credentials. Check the user and password in the connection string."; } From 94b15e82b46b89fed2c8820d9fd741c9efc048f4 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 14 Sep 2026 11:28:13 -0400 Subject: [PATCH 32/34] feat(migrate): ask again when a database connection fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A connection string is long, pasted by hand, masked as it is typed, and wrong in ways nothing can check until something connects: a typo'd host, an expired token, the pooler URL where the direct one was needed, the right server but the wrong database. Any of those ended the command, charging the operator a full re-run — platform, log directory, output path and all — for one mistyped line they could not see. `supabase`, `authjs` and `betterauth` now run the database work through `withDbRetry`, which explains the failure and puts the prompt back up. Only the database read is inside the loop, so an export that has already written its file cannot run twice. `-y`, agent mode and a non-TTY fail as before: there is nobody to ask, and a loop that cannot prompt is a loop that cannot end. A libsql 404 is explained rather than sent to the generic advice. Turso resolves every `*.turso.io` name, so a typo'd database answers 404 instead of failing to connect, and "check the host" points at the half that is right. --- .../cli-core/src/commands/migrate/README.md | 8 + .../src/commands/migrate/export/authjs.ts | 27 ++-- .../src/commands/migrate/export/betterauth.ts | 35 +++-- .../src/commands/migrate/export/db-options.ts | 53 ++++++- .../commands/migrate/export/db-retry.test.ts | 139 ++++++++++++++++++ .../src/commands/migrate/export/supabase.ts | 27 ++-- .../cli-core/src/commands/migrate/lib/db.ts | 10 ++ 7 files changed, 266 insertions(+), 33 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/export/db-retry.test.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 9a4d001e0..ca9e41364 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -252,6 +252,14 @@ clerk migrate export betterauth --db-url "./db.sqlite" clerk migrate export betterauth --db-url "libsql://app-org.turso.io?authToken=..." # or set TURSO_AUTH_TOKEN ``` +**A connection that fails is asked for again.** The string is long, pasted by +hand, masked as it is typed, and wrong in ways nothing can check until +something connects — a typo'd host, an expired token, the pooler URL where the +direct one was needed, the right server but the wrong database. The failure is +explained and the prompt comes back, so a mistyped line costs one line rather +than a re-run of the platform, log directory and output path already answered. +`-y`, agent mode and a non-TTY still fail outright: there is nobody to ask. + Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; `libsql://` (Turso) over the server's HTTP pipeline endpoint, since `bun:sqlite` only opens local files and `@libsql/client` ships native optional dependencies. diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index d9b3f2c72..08c36b965 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -16,7 +16,12 @@ import { log } from "../../../lib/log.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; -import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; +import { + resolveDbUrl, + withDbRetry, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; /** Table names to try, in order. Prisma capitalizes; Drizzle does not. */ const TABLE_CANDIDATES = ["User", "user", "users"] as const; @@ -105,21 +110,25 @@ export function buildAuthJsExport(rows: AuthJsRow[], dateTime: string) { }; } +const AUTHJS_DB = { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Auth.js adapter uses.", +} as const satisfies ResolveConfig; + export async function exportAuthJs(options: DbExportOptions): Promise { - const dbUrl = await resolveDbUrl(options, { - platform: "authjs", - envVar: "AUTHJS_DB_URL", - prompt: "Auth.js database connection string", - hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Auth.js adapter uses.", - }); + const dbUrl = await resolveDbUrl(options, AUTHJS_DB); const destination = await resolveOutputPath("authjs", options.output); await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const { rows, table } = await withSpinner("Reading the user table...", () => - withDbClient(dbUrl, "authjs", fetchAuthJsUsers), + const { rows, table } = await withDbRetry(dbUrl, AUTHJS_DB, async (connectionString) => + withSpinner("Reading the user table...", () => + withDbClient(connectionString, "authjs", fetchAuthJsUsers), + ), ); log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 348271a6d..6eea6262c 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -20,7 +20,12 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; -import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; +import { + resolveDbUrl, + withDbRetry, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; /** Columns a Better Auth plugin adds to the user table. */ export const PLUGIN_COLUMNS = [ @@ -152,25 +157,29 @@ export function buildBetterAuthExport(rows: BetterAuthRow[], dateTime: string) { }; } +const BETTERAUTH_DB = { + platform: "betterauth", + envVar: "BETTERAUTH_DB_URL", + prompt: "Better Auth database connection string", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Better Auth install uses.", +} as const satisfies ResolveConfig; + export async function exportBetterAuth(options: DbExportOptions): Promise { - const dbUrl = await resolveDbUrl(options, { - platform: "betterauth", - envVar: "BETTERAUTH_DB_URL", - prompt: "Better Auth database connection string", - hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Better Auth install uses.", - }); + const dbUrl = await resolveDbUrl(options, BETTERAUTH_DB); const destination = await resolveOutputPath("betterauth", options.output); await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const { rows, plugins } = await withSpinner("Reading the user table...", () => - withDbClient(dbUrl, "betterauth", async (client) => { - const plugins = await detectPluginColumns(client); - const rows = await client.query(buildBetterAuthQuery(client, plugins)); - return { rows, plugins }; - }), + const { rows, plugins } = await withDbRetry(dbUrl, BETTERAUTH_DB, async (connectionString) => + withSpinner("Reading the user table...", () => + withDbClient(connectionString, "betterauth", async (client) => { + const plugins = await detectPluginColumns(client); + const rows = await client.query(buildBetterAuthQuery(client, plugins)); + return { rows, plugins }; + }), + ), ); log.info( diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 3ef77efbe..253fc8e08 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -5,7 +5,7 @@ * string, from a flag, an environment variable, or a prompt. */ -import { throwUsageError } from "../../../lib/errors.ts"; +import { CliError, throwUsageError } from "../../../lib/errors.ts"; import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; @@ -18,7 +18,7 @@ export type DbExportOptions = { output?: string; }; -type ResolveConfig = { +export type ResolveConfig = { platform: DbPlatform; /** Environment variable checked when `--db-url` is absent. */ envVar: string; @@ -135,6 +135,17 @@ export async function resolveDbUrl( if (config.hint) log.info(dim(config.hint)); + return promptDbUrl(config); +} + +/** + * Asks for a connection string, masked. + * + * Masked because a connection string carries the database password inline. The + * validator runs on the normalized value, so a password that needed encoding is + * judged as the driver will see it, not as it was typed. + */ +async function promptDbUrl(config: ResolveConfig): Promise { const answer = await passwordPrompt({ message: config.prompt, validate: (value) => @@ -146,6 +157,44 @@ export async function resolveDbUrl( return normalizeConnectionString(answer); } +/** + * Runs `work` against the database, asking for another connection string each + * time it fails. + * + * A connection string is long, pasted by hand, and wrong in ways nothing can + * check until something connects: a typo'd host, an expired token, the pooler + * URL where the direct one was needed, the right server but the wrong database. + * Ending the command there charges the operator a full re-run — platform, log + * directory, output path and all — for a single mistyped line, and the string + * is masked as they type it, so they cannot even see what to correct. + * + * Only the database work belongs in `work`: everything retried here is retried + * whole, and an export that has already written its file must not run twice. + * + * `-y`, agent mode and a non-TTY get the failure as before — there is nobody to + * ask, and a loop that cannot prompt is a loop that cannot end. + */ +export async function withDbRetry( + dbUrl: string, + config: ResolveConfig, + work: (connectionString: string) => Promise, +): Promise { + let connectionString = dbUrl; + + for (;;) { + try { + return await work(connectionString); + } catch (error) { + // Everything the database layer raises is a CliError carrying its own + // explanation; anything else (an interrupt, a bug) is not ours to retry. + if (!(error instanceof CliError) || !isHuman() || isAgent()) throw error; + + log.error(error.message); + connectionString = await promptDbUrl(config); + } + } +} + /** Describes the target for the run's opening line, credentials removed. */ export function describeTarget(connectionString: string): string { const label = isLibsqlUrl(connectionString) ? "libsql" : detectDbType(connectionString); diff --git a/packages/cli-core/src/commands/migrate/export/db-retry.test.ts b/packages/cli-core/src/commands/migrate/export/db-retry.test.ts new file mode 100644 index 000000000..b0027dbc5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-retry.test.ts @@ -0,0 +1,139 @@ +/** + * `withDbRetry` — the loop that puts the connection-string prompt back up when + * the database work fails. + * + * Its own file because `mock.module` registrations last for the process, and + * `bun test --parallel` puts several files in each worker — a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +let answers: string[] = []; + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../../lib/prompts.ts", () => ({ + password: async () => answers.shift() ?? "", + text: async () => "", + confirm: async () => true, + multiselect: async () => [], + select: async () => "", + editor: async () => "{}", + note: () => {}, +})); + +const { withDbRetry } = await import("./db-options.ts"); + +const captured = useCaptureLog(); + +const CONFIG = { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", +} as const; + +const FIRST = "libsql://typo.turso.io?authToken=t"; +const SECOND = "libsql://right.turso.io?authToken=t"; + +let originalMode: Mode; + +beforeAll(() => { + originalMode = getMode(); +}); + +afterAll(() => { + setMode(originalMode); +}); + +beforeEach(() => { + setMode("human"); + answers = []; +}); + +describe("withDbRetry", () => { + test("returns the first result without prompting when the work succeeds", async () => { + const seen: string[] = []; + + const result = await withDbRetry(FIRST, CONFIG, (url) => { + seen.push(url); + return Promise.resolve("rows"); + }); + + expect(result).toBe("rows"); + expect(seen).toEqual([FIRST]); + }); + + test("asks again after a failure and runs with the new connection string", async () => { + answers = [SECOND]; + const seen: string[] = []; + + const result = await withDbRetry(FIRST, CONFIG, (url) => { + seen.push(url); + if (url === FIRST) { + throw new CliError("Could not reach libsql://***@typo.turso.io", { + code: ERROR_CODE.USAGE_ERROR, + }); + } + return Promise.resolve("rows"); + }); + + expect(result).toBe("rows"); + expect(seen).toEqual([FIRST, SECOND]); + // The operator has to be told what was wrong with the string they cannot see. + expect(captured.err).toContain("Could not reach"); + }); + + test("keeps asking until a connection string works", async () => { + answers = [FIRST, FIRST, SECOND]; + let attempts = 0; + + await withDbRetry(FIRST, CONFIG, (url) => { + attempts++; + if (url !== SECOND) throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); + return Promise.resolve("rows"); + }); + + expect(attempts).toBe(4); + }); + + // Cancelling the prompt is an answer: it ends the command rather than + // looping on a question the operator has already declined. + test("lets a cancelled prompt out of the loop", async () => { + mock.module("../../../lib/prompts.ts", () => ({ + password: async () => { + throw new UserAbortError(); + }, + text: async () => "", + confirm: async () => true, + multiselect: async () => [], + select: async () => "", + editor: async () => "{}", + note: () => {}, + })); + + await expect( + withDbRetry(FIRST, CONFIG, () => { + throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); + }), + ).rejects.toThrow(UserAbortError); + }); + + test("throws without prompting when there is nobody to ask", async () => { + setMode("agent"); + let attempts = 0; + + await expect( + withDbRetry(FIRST, CONFIG, () => { + attempts++; + throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); + }), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index cf257f35f..10d3ee18f 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -15,7 +15,12 @@ import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; -import { resolveDbUrl, type DbExportOptions } from "./db-options.ts"; +import { + resolveDbUrl, + withDbRetry, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; /** * `display_name` is coalesced into `first_name` here rather than in the @@ -103,21 +108,25 @@ export function buildSupabaseExport(rows: SupabaseRow[], dateTime: string) { }; } +const SUPABASE_DB = { + platform: "supabase", + envVar: "SUPABASE_DB_URL", + prompt: "Supabase Postgres connection string", + hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", +} as const satisfies ResolveConfig; + export async function exportSupabase(options: DbExportOptions): Promise { - const dbUrl = await resolveDbUrl(options, { - platform: "supabase", - envVar: "SUPABASE_DB_URL", - prompt: "Supabase Postgres connection string", - hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", - }); + const dbUrl = await resolveDbUrl(options, SUPABASE_DB); const destination = await resolveOutputPath("supabase", options.output); await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const rows = await withSpinner("Reading auth.users...", () => - withDbClient(dbUrl, "supabase", fetchSupabaseUsers), + const rows = await withDbRetry(dbUrl, SUPABASE_DB, async (connectionString) => + withSpinner("Reading auth.users...", () => + withDbClient(connectionString, "supabase", fetchSupabaseUsers), + ), ); const { users, coverage } = buildSupabaseExport(rows, dateTime); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index 64191c125..9b1681568 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -295,6 +295,16 @@ export function describeDbError(error: unknown, platform?: DbPlatform): string { return "Could not reach the database. Check the host and port, and that the server accepts connections from here."; } + // Turso resolves every `*.turso.io` name, so a typo'd database does not fail + // to connect — it answers 404. "Check the host" would send the reader after + // the half that is right. + if (/\b404\b/.test(message)) { + return ( + "No database at that libsql host. Check the database name in the URL —\n" + + "`turso db show ` prints the URL to use." + ); + } + if (/\b401\b|unauthorized|not authorized/i.test(message)) { return ( "The libsql server rejected that token.\n" + From 563ff35019ab4d22d7cdca24cbe389e66b474ed6 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 14 Sep 2026 12:20:30 -0400 Subject: [PATCH 33/34] refactor(migrate): retry any rejected credential, not just a connection string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `withDbRetry` only ever knew how to re-ask for a connection string, but the shape it handled is not specific to databases: every credential a migration takes is long, pasted by hand, masked as it is typed, and wrong in ways nothing local can check. A Firebase key revoked in the console and an Auth0 application missing `read:users` both read as valid input right up until the far end says otherwise — and both ended the command there, after the operator had already answered every other question it asked. It becomes `withInputRetry` in `migrate/lib/`, taking the input, a way to ask for another, and the step that proves it. `export firebase` and `export auth0` now run their token exchange through it, alongside the three database exports. The helper hands back the input that finally worked, so the rest of the export runs against that one — a Firebase export reads its project id off the key Google accepted, not the key first offered. Only the proving step goes inside the loop: a fetch already under way or a file already written must not run twice. `-y`, agent mode and a non-TTY fail as before, and a cancelled prompt leaves the loop, since declining the question is an answer. --- .../cli-core/src/commands/migrate/README.md | 19 +- .../src/commands/migrate/export/auth0.ts | 35 +++- .../src/commands/migrate/export/authjs.ts | 16 +- .../src/commands/migrate/export/betterauth.ts | 24 ++- .../src/commands/migrate/export/db-options.ts | 42 +--- .../commands/migrate/export/db-retry.test.ts | 139 ------------- .../src/commands/migrate/export/firebase.ts | 28 ++- .../src/commands/migrate/export/supabase.ts | 14 +- .../commands/migrate/lib/input-retry.test.ts | 189 ++++++++++++++++++ .../src/commands/migrate/lib/input-retry.ts | 62 ++++++ 10 files changed, 351 insertions(+), 217 deletions(-) delete mode 100644 packages/cli-core/src/commands/migrate/export/db-retry.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/input-retry.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/input-retry.ts diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index ca9e41364..3363a72c3 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -152,6 +152,17 @@ the registry; given, it runs directly. Each platform resolves its own flags — what Auth0 needs (a tenant domain and M2M credentials) has nothing in common with what a database export needs. +**A credential the far end rejects is asked for again.** Connection strings, +Firebase service account keys and Auth0 client secrets are all long, pasted by +hand, masked as they are typed, and wrong in ways nothing local can check: a +typo'd host, a revoked key, an expired token, the right server but the wrong +database. Only the connection or the token exchange can say, and by then the +operator has answered every other question the command asked. So that step — +and only that step, never a fetch already under way or a file already written — +runs inside a retry: the failure is explained, the prompt comes back, and the +rest of the export continues against whichever credential worked. `-y`, agent +mode and a non-TTY fail outright instead, having nobody to ask. + | Platform | Source | Feeds | | ------------ | -------------------------------- | -------------------------- | | `clerk` | Clerk Backend API | `--transformer clerk` | @@ -252,14 +263,6 @@ clerk migrate export betterauth --db-url "./db.sqlite" clerk migrate export betterauth --db-url "libsql://app-org.turso.io?authToken=..." # or set TURSO_AUTH_TOKEN ``` -**A connection that fails is asked for again.** The string is long, pasted by -hand, masked as it is typed, and wrong in ways nothing can check until -something connects — a typo'd host, an expired token, the pooler URL where the -direct one was needed, the right server but the wrong database. The failure is -explained and the prompt comes back, so a mistyped line costs one line rather -than a re-run of the platform, log directory and output path already answered. -`-y`, agent mode and a non-TTY still fail outright: there is nobody to ask. - Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; `libsql://` (Turso) over the server's HTTP pipeline endpoint, since `bun:sqlite` only opens local files and `@libsql/client` ships native optional dependencies. diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts index b8a1a6308..d45e74a73 100644 --- a/packages/cli-core/src/commands/migrate/export/auth0.ts +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -23,6 +23,7 @@ import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spin import { isAgent, isHuman } from "../../../mode.ts"; import { findMigrateEnvValue } from "../lib/env-file.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; const PAGE_SIZE = 100; @@ -113,20 +114,33 @@ export async function resolveAuth0Credentials( "Auth0 needs a machine-to-machine application with the `read:users` scope. Create one under Applications → APIs → Auth0 Management API → Machine to Machine Applications.", ); + return promptAuth0Credentials(resolved); +} + +/** + * Asks for whichever of the three are still missing. + * + * Called with nothing known after Auth0 has rejected a set: its error names no + * field, and the operator may have mistyped any of them — so all three are + * asked again rather than guessing which one to keep. + */ +export async function promptAuth0Credentials( + known: Partial = {}, +): Promise { const domain = - resolved.domain ?? + known.domain ?? (await text({ message: "Auth0 tenant domain (e.g. my-tenant.us.auth0.com)", validate: (value) => (value?.trim() ? undefined : "A domain is required"), })); const clientId = - resolved.clientId ?? + known.clientId ?? (await text({ message: "Machine-to-machine client ID", validate: (value) => (value?.trim() ? undefined : "A client ID is required"), })); const clientSecret = - resolved.clientSecret ?? + known.clientSecret ?? (await passwordPrompt({ message: "Machine-to-machine client secret", validate: (value) => (value?.trim() ? undefined : "A client secret is required"), @@ -314,16 +328,23 @@ export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0Exp } export async function exportAuth0(options: ExportAuth0Options): Promise { - const credentials = await resolveAuth0Credentials(options); + const resolved = await resolveAuth0Credentials(options); const destination = await resolveOutputPath("auth0", options.output); await withGutter("Exporting users from Auth0", async ({ setNextSteps }) => { const dateTime = await startLogging(); - log.info(`Exporting from ${credentials.domain}.`); - const token = await withSpinner("Authenticating with Auth0...", () => - fetchAuth0Token(credentials), + // Only Auth0 can say whether these three go together, and whether the + // application carries the `read:users` scope, so a rejected set is asked + // for again here. + const { value: token, input: credentials } = await withInputRetry( + resolved, + () => promptAuth0Credentials(), + async (candidate) => { + log.info(`Exporting from ${candidate.domain}.`); + return withSpinner("Authenticating with Auth0...", () => fetchAuth0Token(candidate)); + }, ); const users = await withSpinner("Fetching users from Auth0...", (spinner) => diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts index 08c36b965..2f80c1f9e 100644 --- a/packages/cli-core/src/commands/migrate/export/authjs.ts +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -17,11 +17,12 @@ import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { + promptDbUrl, resolveDbUrl, - withDbRetry, type DbExportOptions, type ResolveConfig, } from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; /** Table names to try, in order. Prisma capitalizes; Drizzle does not. */ const TABLE_CANDIDATES = ["User", "user", "users"] as const; @@ -125,10 +126,15 @@ export async function exportAuthJs(options: DbExportOptions): Promise { await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const { rows, table } = await withDbRetry(dbUrl, AUTHJS_DB, async (connectionString) => - withSpinner("Reading the user table...", () => - withDbClient(connectionString, "authjs", fetchAuthJsUsers), - ), + const { + value: { rows, table }, + } = await withInputRetry( + dbUrl, + () => promptDbUrl(AUTHJS_DB), + async (connectionString) => + withSpinner("Reading the user table...", () => + withDbClient(connectionString, "authjs", fetchAuthJsUsers), + ), ); log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts index 6eea6262c..e9b901063 100644 --- a/packages/cli-core/src/commands/migrate/export/betterauth.ts +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -21,11 +21,12 @@ import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { + promptDbUrl, resolveDbUrl, - withDbRetry, type DbExportOptions, type ResolveConfig, } from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; /** Columns a Better Auth plugin adds to the user table. */ export const PLUGIN_COLUMNS = [ @@ -172,14 +173,19 @@ export async function exportBetterAuth(options: DbExportOptions): Promise await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const { rows, plugins } = await withDbRetry(dbUrl, BETTERAUTH_DB, async (connectionString) => - withSpinner("Reading the user table...", () => - withDbClient(connectionString, "betterauth", async (client) => { - const plugins = await detectPluginColumns(client); - const rows = await client.query(buildBetterAuthQuery(client, plugins)); - return { rows, plugins }; - }), - ), + const { + value: { rows, plugins }, + } = await withInputRetry( + dbUrl, + () => promptDbUrl(BETTERAUTH_DB), + async (connectionString) => + withSpinner("Reading the user table...", () => + withDbClient(connectionString, "betterauth", async (client) => { + const plugins = await detectPluginColumns(client); + const rows = await client.query(buildBetterAuthQuery(client, plugins)); + return { rows, plugins }; + }), + ), ); log.info( diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 253fc8e08..c0f92a92f 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -5,7 +5,7 @@ * string, from a flag, an environment variable, or a prompt. */ -import { CliError, throwUsageError } from "../../../lib/errors.ts"; +import { throwUsageError } from "../../../lib/errors.ts"; import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; @@ -145,7 +145,7 @@ export async function resolveDbUrl( * validator runs on the normalized value, so a password that needed encoding is * judged as the driver will see it, not as it was typed. */ -async function promptDbUrl(config: ResolveConfig): Promise { +export async function promptDbUrl(config: ResolveConfig): Promise { const answer = await passwordPrompt({ message: config.prompt, validate: (value) => @@ -157,44 +157,6 @@ async function promptDbUrl(config: ResolveConfig): Promise { return normalizeConnectionString(answer); } -/** - * Runs `work` against the database, asking for another connection string each - * time it fails. - * - * A connection string is long, pasted by hand, and wrong in ways nothing can - * check until something connects: a typo'd host, an expired token, the pooler - * URL where the direct one was needed, the right server but the wrong database. - * Ending the command there charges the operator a full re-run — platform, log - * directory, output path and all — for a single mistyped line, and the string - * is masked as they type it, so they cannot even see what to correct. - * - * Only the database work belongs in `work`: everything retried here is retried - * whole, and an export that has already written its file must not run twice. - * - * `-y`, agent mode and a non-TTY get the failure as before — there is nobody to - * ask, and a loop that cannot prompt is a loop that cannot end. - */ -export async function withDbRetry( - dbUrl: string, - config: ResolveConfig, - work: (connectionString: string) => Promise, -): Promise { - let connectionString = dbUrl; - - for (;;) { - try { - return await work(connectionString); - } catch (error) { - // Everything the database layer raises is a CliError carrying its own - // explanation; anything else (an interrupt, a bug) is not ours to retry. - if (!(error instanceof CliError) || !isHuman() || isAgent()) throw error; - - log.error(error.message); - connectionString = await promptDbUrl(config); - } - } -} - /** Describes the target for the run's opening line, credentials removed. */ export function describeTarget(connectionString: string): string { const label = isLibsqlUrl(connectionString) ? "libsql" : detectDbType(connectionString); diff --git a/packages/cli-core/src/commands/migrate/export/db-retry.test.ts b/packages/cli-core/src/commands/migrate/export/db-retry.test.ts deleted file mode 100644 index b0027dbc5..000000000 --- a/packages/cli-core/src/commands/migrate/export/db-retry.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -/** - * `withDbRetry` — the loop that puts the connection-string prompt back up when - * the database work fails. - * - * Its own file because `mock.module` registrations last for the process, and - * `bun test --parallel` puts several files in each worker — a mocked - * `prompts.ts` would leak into any file that later lands in the same worker and - * imports the real one. - */ - -import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; -import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; -import { getMode, setMode, type Mode } from "../../../mode.ts"; -import { useCaptureLog } from "../../../test/lib/stubs.ts"; - -let answers: string[] = []; - -// Every export of the real module must appear here — a missing one is a link -// error at import time, which takes down the whole file rather than one prompt. -mock.module("../../../lib/prompts.ts", () => ({ - password: async () => answers.shift() ?? "", - text: async () => "", - confirm: async () => true, - multiselect: async () => [], - select: async () => "", - editor: async () => "{}", - note: () => {}, -})); - -const { withDbRetry } = await import("./db-options.ts"); - -const captured = useCaptureLog(); - -const CONFIG = { - platform: "authjs", - envVar: "AUTHJS_DB_URL", - prompt: "Auth.js database connection string", -} as const; - -const FIRST = "libsql://typo.turso.io?authToken=t"; -const SECOND = "libsql://right.turso.io?authToken=t"; - -let originalMode: Mode; - -beforeAll(() => { - originalMode = getMode(); -}); - -afterAll(() => { - setMode(originalMode); -}); - -beforeEach(() => { - setMode("human"); - answers = []; -}); - -describe("withDbRetry", () => { - test("returns the first result without prompting when the work succeeds", async () => { - const seen: string[] = []; - - const result = await withDbRetry(FIRST, CONFIG, (url) => { - seen.push(url); - return Promise.resolve("rows"); - }); - - expect(result).toBe("rows"); - expect(seen).toEqual([FIRST]); - }); - - test("asks again after a failure and runs with the new connection string", async () => { - answers = [SECOND]; - const seen: string[] = []; - - const result = await withDbRetry(FIRST, CONFIG, (url) => { - seen.push(url); - if (url === FIRST) { - throw new CliError("Could not reach libsql://***@typo.turso.io", { - code: ERROR_CODE.USAGE_ERROR, - }); - } - return Promise.resolve("rows"); - }); - - expect(result).toBe("rows"); - expect(seen).toEqual([FIRST, SECOND]); - // The operator has to be told what was wrong with the string they cannot see. - expect(captured.err).toContain("Could not reach"); - }); - - test("keeps asking until a connection string works", async () => { - answers = [FIRST, FIRST, SECOND]; - let attempts = 0; - - await withDbRetry(FIRST, CONFIG, (url) => { - attempts++; - if (url !== SECOND) throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); - return Promise.resolve("rows"); - }); - - expect(attempts).toBe(4); - }); - - // Cancelling the prompt is an answer: it ends the command rather than - // looping on a question the operator has already declined. - test("lets a cancelled prompt out of the loop", async () => { - mock.module("../../../lib/prompts.ts", () => ({ - password: async () => { - throw new UserAbortError(); - }, - text: async () => "", - confirm: async () => true, - multiselect: async () => [], - select: async () => "", - editor: async () => "{}", - note: () => {}, - })); - - await expect( - withDbRetry(FIRST, CONFIG, () => { - throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); - }), - ).rejects.toThrow(UserAbortError); - }); - - test("throws without prompting when there is nobody to ask", async () => { - setMode("agent"); - let attempts = 0; - - await expect( - withDbRetry(FIRST, CONFIG, () => { - attempts++; - throw new CliError("nope", { code: ERROR_CODE.USAGE_ERROR }); - }), - ).rejects.toThrow(CliError); - - expect(attempts).toBe(1); - }); -}); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index 93a6f53af..f016d7b08 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -32,6 +32,7 @@ import { password as passwordPrompt } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; /** Identity Toolkit's maximum for `accounts:batchGet`. */ @@ -166,6 +167,19 @@ async function resolveServiceAccount(options: ExportFirebaseOptions): Promise { const answer = await passwordPrompt({ message: "Path to the service account key file, or paste the key JSON", validate: (value) => { @@ -482,16 +496,22 @@ export function formatHashConfigGuidance( export async function exportFirebase(options: ExportFirebaseOptions): Promise { // Read and validate before anything reaches the network, so a wrong file // fails in a second rather than after an auth round-trip. - const account = await resolveServiceAccount(options); + const resolved = await resolveServiceAccount(options); const destination = await resolveOutputPath("firebase", options.output); await withGutter("Exporting users from Firebase", async ({ setNextSteps }) => { const dateTime = await startLogging(); - log.info(`Exporting from the ${account.project_id} project.`); - const token = await withSpinner("Authenticating with Google...", () => - fetchAccessToken(account), + // Only Google can say whether a well-formed key is still a valid one, so a + // revoked or deleted key fails here and is asked for again. + const { value: token, input: account } = await withInputRetry( + resolved, + promptServiceAccount, + async (candidate) => { + log.info(`Exporting from the ${candidate.project_id} project.`); + return withSpinner("Authenticating with Google...", () => fetchAccessToken(candidate)); + }, ); const users = await withSpinner("Fetching users from Firebase...", (spinner) => diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 10d3ee18f..275a2bbd6 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -16,11 +16,12 @@ import { exportLogger, startLogging } from "../lib/logger.ts"; import { withDbClient, type DbClient } from "../lib/db.ts"; import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; import { + promptDbUrl, resolveDbUrl, - withDbRetry, type DbExportOptions, type ResolveConfig, } from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; /** * `display_name` is coalesced into `first_name` here rather than in the @@ -123,10 +124,13 @@ export async function exportSupabase(options: DbExportOptions): Promise { await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { const dateTime = await startLogging(); - const rows = await withDbRetry(dbUrl, SUPABASE_DB, async (connectionString) => - withSpinner("Reading auth.users...", () => - withDbClient(connectionString, "supabase", fetchSupabaseUsers), - ), + const { value: rows } = await withInputRetry( + dbUrl, + () => promptDbUrl(SUPABASE_DB), + async (connectionString) => + withSpinner("Reading auth.users...", () => + withDbClient(connectionString, "supabase", fetchSupabaseUsers), + ), ); const { users, coverage } = buildSupabaseExport(rows, dateTime); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts new file mode 100644 index 000000000..8e7e6e26b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -0,0 +1,189 @@ +/** + * `withInputRetry` — the loop that puts a credential prompt back up when the + * far end rejects what it was given. + * + * Its own file because `mock.module` registrations last for the process, and + * `bun test --parallel` puts several files in each worker — a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +let answers: string[] = []; +let cancelPrompt = false; + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../../lib/prompts.ts", () => ({ + password: async () => { + if (cancelPrompt) throw new UserAbortError(); + return answers.shift() ?? ""; + }, + text: async () => answers.shift() ?? "", + confirm: async () => true, + multiselect: async () => [], + select: async () => "", + editor: async () => "{}", + note: () => {}, +})); + +const { withInputRetry } = await import("./input-retry.ts"); +const { promptDbUrl } = await import("../export/db-options.ts"); + +const captured = useCaptureLog(); + +const CONFIG = { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", +} as const; + +const FIRST = "libsql://typo.turso.io?authToken=t"; +const SECOND = "libsql://right.turso.io?authToken=t"; + +const rejected = () => new CliError("Could not reach it", { code: ERROR_CODE.USAGE_ERROR }); + +let originalMode: Mode; + +beforeAll(() => { + originalMode = getMode(); +}); + +afterAll(() => { + setMode(originalMode); +}); + +beforeEach(() => { + setMode("human"); + answers = []; + cancelPrompt = false; +}); + +describe("withInputRetry", () => { + test("returns the first result without prompting when the work succeeds", async () => { + const seen: string[] = []; + + const { value, input } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + seen.push(url); + return Promise.resolve("rows"); + }, + ); + + expect(value).toBe("rows"); + expect(input).toBe(FIRST); + expect(seen).toEqual([FIRST]); + }); + + test("asks again after a failure and runs with the new input", async () => { + answers = [SECOND]; + const seen: string[] = []; + + const { value } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + seen.push(url); + if (url === FIRST) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(value).toBe("rows"); + expect(seen).toEqual([FIRST, SECOND]); + // The operator has to be told what was wrong with a string they cannot see. + expect(captured.err).toContain("Could not reach it"); + }); + + // Later steps run against the credential that worked, not the one first tried + // — a Firebase export reads its project id off the key that Google accepted. + test("reports the input that finally worked", async () => { + answers = [SECOND]; + + const { input } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + if (url === FIRST) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(input).toBe(SECOND); + }); + + test("keeps asking until an input works", async () => { + answers = [FIRST, FIRST, SECOND]; + let attempts = 0; + + await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + attempts++; + if (url !== SECOND) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(attempts).toBe(4); + }); + + // Cancelling the prompt is an answer: it ends the command rather than looping + // on a question the operator has already declined. + test("lets a cancelled prompt out of the loop", async () => { + cancelPrompt = true; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + throw rejected(); + }, + ), + ).rejects.toThrow(UserAbortError); + }); + + test("throws without prompting when there is nobody to ask", async () => { + setMode("agent"); + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + + // A bug inside the work, or an interrupt, is not a wrong answer to a prompt. + test("does not retry an error the database layer did not raise", async () => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw new TypeError("undefined is not a function"); + }, + ), + ).rejects.toThrow(TypeError); + + expect(attempts).toBe(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts new file mode 100644 index 000000000..36805de7d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -0,0 +1,62 @@ +/** + * Retrying the *answer*, not the request. + * + * Distinct from `retry.ts`, which re-sends an identical request after a 429: + * there the request was right and the server was busy. Here the request was + * fine and the input was wrong, so nothing changes until the operator supplies + * something better. + * + * Every credential a migration takes — a connection string, a Firebase service + * account key, an Auth0 client secret — is long, pasted by hand, masked as it + * is typed, and wrong in ways nothing local can check: a typo'd host, an + * expired token, a key that was revoked, the right server but the wrong + * database. Only the remote end can say, and by then the operator has already + * answered every other question the command asked. Ending there charges them a + * full re-run for one line they could not see. + */ + +import { CliError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; + +/** + * Runs `work`, and on failure asks for the input again and runs it once more. + * + * Keep `work` to the step that *proves* the input — the connection, the token + * exchange. Everything inside runs again on each attempt, so work that has + * already written a file, or a long fetch the credential has already been + * accepted for, does not belong in here. + * + * `-y`, agent mode and a non-TTY get the failure unchanged: there is nobody to + * ask, and a loop that cannot prompt is a loop that cannot end. A cancelled + * prompt throws {@link UserAbortError}, which is not a `CliError` and so leaves + * the loop — declining the question is an answer. + * + * @param input - What to try first: a flag, an environment value, or the + * answer to the prompt the caller has already put up. + * @param reprompt - Asks for a replacement. Called once per failure. + * @param work - The step the input has to survive. + * @returns The result, and the input that produced it — which is not `input` + * when it took a retry, and later steps need the one that worked. + */ +export async function withInputRetry( + input: I, + reprompt: () => Promise, + work: (input: I) => Promise, +): Promise<{ value: T; input: I }> { + let candidate = input; + + for (;;) { + try { + return { value: await work(candidate), input: candidate }; + } catch (error) { + // Everything these steps raise for a bad credential is a CliError + // carrying its own explanation; anything else (an interrupt, a bug) is + // not ours to retry. + if (!(error instanceof CliError) || !isHuman() || isAgent()) throw error; + + log.error(error.message); + candidate = await reprompt(); + } + } +} From 534894f223c533a42dd40c0e3206597f3b7ecd69 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Mon, 14 Sep 2026 12:40:53 -0400 Subject: [PATCH 34/34] fix(migrate): keep the Firebase import command on one line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hash parameters are printed inside the gutter, which prefixes every line it is given with `│`. The command was split over four lines with backslash continuations, so copying it took three of those bars along with it and the shell read them as arguments: error: too many arguments for 'import'. Expected 0 arguments but got 3: │, │, │. The command a user is told to run has to survive being copied, so it is one line however long it gets. A line that wraps on screen carries no bar and pastes back as what was printed. Reported against a real Firebase export. --- packages/cli-core/src/commands/migrate/README.md | 11 ++++++++--- .../src/commands/migrate/export/firebase.test.ts | 10 ++++++++++ .../src/commands/migrate/export/firebase.ts | 13 +++++++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 3363a72c3..d2cf4b051 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -332,11 +332,16 @@ prints the exact import command: ``` Password hash parameters Read from the project. Import with: - clerk migrate import -y --transformer firebase --file exports/firebase-export.json \ - --firebase-signer-key "…" --firebase-salt-separator "…" \ - --firebase-rounds 8 --firebase-mem-cost 14 + clerk migrate import -y --transformer firebase --file exports/firebase-export.json --firebase-signer-key "…" --firebase-salt-separator "…" --firebase-rounds 8 --firebase-mem-cost 14 ``` +On one line however long it gets: this prints inside the gutter, which prefixes +every line given to it with `│`. Split over lines with backslash continuations, +that character lands in the middle of the command and is copied along with it — +the shell then reads each one as another argument and rejects the import. A line +that wraps on screen carries no such character and pastes back as what was +printed. + Reading the config needs a broader role than listing users, so if it is denied the export still succeeds and points at **Authentication → Users → (⋮) → Password hash parameters** instead. An export with no password hashes says so diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts index d4a440b7e..4d432e0a0 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.test.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -410,6 +410,16 @@ describe("formatHashConfigGuidance", () => { expect(text).toContain("--firebase-rounds 8 --firebase-mem-cost 14"); }); + // The command is printed inside the gutter, which prefixes every line it is + // given with `│`. Split over lines, that character lands mid-command and is + // copied with it — the shell then reads each one as another argument and + // rejects the import. + test("keeps the command on one line, so it can be copied out of the gutter", () => { + const [command] = formatHashConfigGuidance(config, "out.json", 3).slice(-1); + expect(command).not.toContain("\n"); + expect(command).not.toContain("\\"); + }); + test("says where to find them when the project would not say", () => { const text = formatHashConfigGuidance(null, "out.json", 3).join("\n"); expect(text).toContain("Password hash parameters"); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts index f016d7b08..3f983c5e9 100644 --- a/packages/cli-core/src/commands/migrate/export/firebase.ts +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -484,11 +484,16 @@ export function formatHashConfigGuidance( return [ bold("Password hash parameters"), "Read from the project. Import with:", + // One line, however long. Inside the gutter every line printed here is + // prefixed with `│`, and backslash continuations put that character in the + // middle of the command — copied along with it, and rejected by the shell + // as three extra arguments. A line that wraps on screen has no such + // character in it and pastes back as what was printed. dim( - ` clerk migrate import -y --transformer firebase --file ${outputPath} \\\n` + - ` --firebase-signer-key "${config.signerKey}" \\\n` + - ` --firebase-salt-separator "${config.saltSeparator}" \\\n` + - ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, + ` clerk migrate import -y --transformer firebase --file ${outputPath}` + + ` --firebase-signer-key "${config.signerKey}"` + + ` --firebase-salt-separator "${config.saltSeparator}"` + + ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, ), ]; }