diff --git a/.changeset/migrate-cli.md b/.changeset/migrate-cli.md new file mode 100644 index 000000000..6ba3096b0 --- /dev/null +++ b/.changeset/migrate-cli.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Add `clerk migrate` for importing users with `migrate import`, exporting from supported auth providers, reviewing migration logs, undoing a migration, and extending imports with custom transformers. diff --git a/.gitignore b/.gitignore index 91df74696..9ab506561 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ coverage # logs logs +!packages/cli-core/src/commands/migrate/logs/ +!packages/cli-core/src/commands/migrate/logs/** _.log report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json @@ -44,3 +46,7 @@ test/e2e/.har # Local planning/spec docs docs/superpowers/ + +# Local migration exports and the credentials that produced them +exports/ +*service-account*.json diff --git a/CLAUDE.md b/CLAUDE.md index 5a38ae8ca..a149d3b10 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,7 @@ Default to using Bun instead of Node.js. - `Bun.serve()` supports WebSockets, HTTPS, and routes. Don't use `express`. - `bun:sqlite` for SQLite. Don't use `better-sqlite3`. - `Bun.redis` for Redis. Don't use `ioredis`. -- `Bun.sql` for Postgres. Don't use `pg` or `postgres.js`. +- `Bun.sql` for Postgres and MySQL. Don't use `pg`, `postgres.js`, or `mysql2`. - `WebSocket` is built-in. Don't use `ws`. - Prefer `Bun.file` over `node:fs`'s readFile/writeFile - Bun.$`ls` instead of execa. @@ -56,6 +56,8 @@ When running multiple test files directly with `bun test`, always pass `--isolat These flags require Bun >= 1.3.13 — older versions silently ignore them and lose isolation. `bun run test` and `bun run test:e2e` run `scripts/check-bun-version.ts` first, which fails fast when the installed Bun is older than the `engines.bun` floor in package.json. +The same floor also covers `Bun.sql`'s MySQL adapter used by the DB-backed export commands: MySQL support landed in Bun 1.2.21, but binary columns (password hashes) only decoded correctly from 1.3.6. See the header of `scripts/check-bun-version.ts`. + ## Versioning The `CLI_VERSION` global is injected at compile time via `bun build --compile --define "CLI_VERSION=..."`. The CI release workflow injects the real version. diff --git a/README.md b/README.md index 817fcdaab..56d40fd14 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Commands: init [options] Initialize Clerk in your project link [options] Link this project to a Clerk application mcp Manage the Clerk remote MCP server connection for AI editors and CLIs + migrate Migrate users into Clerk from another auth provider or another Clerk instance open Open Clerk resources in your browser telemetry Control CLI usage telemetry (status, disable, enable) unlink [options] Unlink this project from its Clerk application diff --git a/bun.lock b/bun.lock index 5bd3c08aa..67c0f8e0b 100644 --- a/bun.lock +++ b/bun.lock @@ -1,9 +1,9 @@ { "lockfileVersion": 1, - "configVersion": 0, + "configVersion": 1, "workspaces": { "": { - "name": "marseille", + "name": "@clerk/cli-workspace", "devDependencies": { "@changesets/cli": "^2.31.1", "@clerk/testing": "^2.2.19", @@ -20,7 +20,7 @@ }, "packages/cli": { "name": "clerk", - "version": "3.2.0", + "version": "3.3.0", "bin": { "clerk": "./bin/clerk", }, @@ -37,11 +37,13 @@ "@commander-js/extra-typings": "^15.0.0", "@napi-rs/keyring": "^1.3.0", "commander": "^15.0.0", + "csv-parser": "^3.2.1", "env-paths": "^4.0.0", "external-editor": "^3.1.0", "magicast": "^0.5.3", "semver": "^7.8.5", "yaml": "^2.9.0", + "zod": "^4.4.3", }, "devDependencies": { "@clerk/shared": "^4.29.1", @@ -58,22 +60,19 @@ }, }, }, - "patchedDependencies": { - "playwright-core@1.60.0": "patches/playwright-core@1.60.0.patch", - }, "overrides": { "tmp": "^0.2.6", }, "packages": { - "@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="], + "@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], - "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="], + "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], - "@babel/parser": ["@babel/parser@7.29.3", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA=="], + "@babel/parser": ["@babel/parser@7.29.8", "", { "dependencies": { "@babel/types": "^7.29.8" }, "bin": "./bin/babel-parser.js" }, "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA=="], - "@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="], + "@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="], - "@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="], + "@babel/types": ["@babel/types@7.29.8", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg=="], "@changesets/apply-release-plan": ["@changesets/apply-release-plan@7.1.1", "", { "dependencies": { "@changesets/config": "^3.1.4", "@changesets/get-version-range-type": "^0.4.0", "@changesets/git": "^3.0.4", "@changesets/should-skip-package": "^0.1.2", "@changesets/types": "^6.1.0", "@manypkg/get-packages": "^1.1.3", "detect-indent": "^6.0.0", "fs-extra": "^7.0.1", "lodash.startcase": "^4.4.0", "outdent": "^0.5.0", "prettier": "^2.7.1", "resolve-from": "^5.0.0", "semver": "^7.5.3" } }, "sha512-9qPCm/rLx/xoOFXIHGB229+4GOL76S4MC+7tyOuTsR6+1jYlfFDQORdvwR5hDA6y4FL2BPt3qpbcQIS+dW85LA=="], @@ -113,15 +112,15 @@ "@clack/prompts": ["@clack/prompts@1.7.0", "", { "dependencies": { "@clack/core": "1.4.3", "fast-string-width": "^3.0.2", "fast-wrap-ansi": "^0.2.0", "sisteransi": "^1.0.5" } }, "sha512-y7/yvZ2TPAnR9+jnc00klvNNLkJiXFFrQA/hlLCcxA9a2A4zQIOimyFQ9XfwYKiGD1fb5GY8vbKIIgO8d5Tb2A=="], - "@clerk/backend": ["@clerk/backend@3.16.1", "", { "dependencies": { "@clerk/shared": "^4.27.1", "standardwebhooks": "^1.0.0", "tslib": "2.8.1" } }, "sha512-tbsrGqw45ar7PDnEV3gwkHE/HM1EwRcYPPLyL3Hw9H0Ms5fG7hZ8+EJegPJ4DpKdfvOw5M+hYMeCVKwv3NxahA=="], + "@clerk/backend": ["@clerk/backend@3.17.1", "", { "dependencies": { "@clerk/shared": "^4.31.0", "standardwebhooks": "^1.0.0", "tslib": "2.8.1" } }, "sha512-HDBknkYVTMknYsRNHeI0lUE+e4QfOty/LDzWKpbnaaKJUMzHZDUcVo9RQnLdg18XvhaBN7z1OoRiphJ82nimRA=="], "@clerk/cli-core": ["@clerk/cli-core@workspace:packages/cli-core"], "@clerk/cli-extras": ["@clerk/cli-extras@workspace:packages/extras"], - "@clerk/shared": ["@clerk/shared@4.29.1", "", { "dependencies": { "@tanstack/query-core": "^5.100.6", "dequal": "2.0.3", "glob-to-regexp": "0.4.1", "js-cookie": "3.0.7" }, "peerDependencies": { "react": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0", "react-dom": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-vl53gfMKHGhOskQgYfz7Qdkw0E/TYMCRuBkKRibi+8/Mz0JvR1yyVKA9k/w9CRvGfGLL9m/4/YDr1IlUJjx8ag=="], + "@clerk/shared": ["@clerk/shared@4.31.0", "", { "dependencies": { "@tanstack/query-core": "^5.100.6", "dequal": "2.0.3", "glob-to-regexp": "0.4.1", "js-cookie": "3.0.7" }, "peerDependencies": { "react": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0", "react-dom": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-jw7J8dmdcXzeGHFAaMA92r04Ej1eVhAeHlcF2XuCKU+teGt3SV4yc2I0eWSqoLmsYpzgBgA66SJ6lLk22R2+fQ=="], - "@clerk/testing": ["@clerk/testing@2.2.19", "", { "dependencies": { "@clerk/backend": "^3.16.1", "@clerk/shared": "^4.27.1", "dotenv": "17.2.2" }, "peerDependencies": { "@playwright/test": "^1", "cypress": "^13 || ^14 || ^15" }, "optionalPeers": ["@playwright/test", "cypress"] }, "sha512-A5dBdHfdNXw70PFeXcDPAonir/bLhblL3sz2NAJZmFOXWwp03rN8IYh8m0E4w4evFOR62vx7E46iJeXMH5MnMg=="], + "@clerk/testing": ["@clerk/testing@2.2.33", "", { "dependencies": { "@clerk/backend": "^3.17.1", "@clerk/shared": "^4.31.0", "dotenv": "17.2.2" }, "peerDependencies": { "@playwright/test": "^1", "cypress": "^13 || ^14 || ^15" }, "optionalPeers": ["@playwright/test", "cypress"] }, "sha512-+LzWrSwkU3j0jWylS4znQmbivDKmAPN7HuBcwFyBOU92Bp+/JHyylrzaR3RVOwex/Cea8Cztsl/Y+Ys4tXs4Ww=="], "@commander-js/extra-typings": ["@commander-js/extra-typings@15.0.0", "", { "peerDependencies": { "commander": "~15.0.0" } }, "sha512-yeJlba62xqmkgELUsn7356MEnzLLu/fw2x4lofFqGnXh6YysRdEs2BaLeLtg1+KU0AXvMeqQvTTp+3hBEBK+EA=="], @@ -217,51 +216,51 @@ "@oxlint-tsgolint/win32-x64": ["@oxlint-tsgolint/win32-x64@7.0.2001", "", { "os": "win32", "cpu": "x64" }, "sha512-FkDRm8hx9OwzGQqyWG1tO5QrTLRApff9DzSgpz9QZau37BR8d1VYKOxMLGf6shPZntJFoTwIIJYT68VndYDCog=="], - "@oxlint/binding-android-arm-eabi": ["@oxlint/binding-android-arm-eabi@1.79.0", "", { "os": "android", "cpu": "arm" }, "sha512-TebFaaMklO/RXzTv7PucaCq9l3X6D1gA+C8H6K4njtjFOV+zWE9MKLpulcJZN9bzytbUbQIY0mZuz12nQ5Kv4Q=="], + "@oxlint/binding-android-arm-eabi": ["@oxlint/binding-android-arm-eabi@1.81.0", "", { "os": "android", "cpu": "arm" }, "sha512-IcCRsXiedJoJopY6mpZUBEeVFsUrutmrG7dZ87zMuKJlhg70Ora9bBl1WcCxZQtyI10YpnVdEso5oCg7YcfSHw=="], - "@oxlint/binding-android-arm64": ["@oxlint/binding-android-arm64@1.79.0", "", { "os": "android", "cpu": "arm64" }, "sha512-KqqnOtAVgNsPPF0YSodkFZA1O80jcKoCZCTu3bgsszxA+MrMP9TLzfXitKjEj1FmrPprKDMdRDMmY3weESO9sg=="], + "@oxlint/binding-android-arm64": ["@oxlint/binding-android-arm64@1.81.0", "", { "os": "android", "cpu": "arm64" }, "sha512-GRrIPyTGVhx3L3h+0T5xT2A0jFAcdPv4+IfuXpGDLIdl6XeYhgg/zw72A5ILZoUgRqZuM8F1y+V/gfDriXSxzQ=="], - "@oxlint/binding-darwin-arm64": ["@oxlint/binding-darwin-arm64@1.79.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-BVC2nsMzqQzRDPc5RhixkZ+m1p7iH4bxRRvqkbwDXX0PlQKm1BPy8J8cRjnAFafOq2QzI+BfO3vE8w2GZ3CBag=="], + "@oxlint/binding-darwin-arm64": ["@oxlint/binding-darwin-arm64@1.81.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-qNQ9tXRgLuKbqSV1S2h9h4KPHjbovO7RRR2/enUOtHzTkFZ7B9X5zqqHJua8dRyc7dBy7Aoyq5pqTSLFVcAzGQ=="], - "@oxlint/binding-darwin-x64": ["@oxlint/binding-darwin-x64@1.79.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-p6Lm+snmhGuLKL1+CpCV8L6ijkE/qJzK2H2jG9+eKJT0n31RbY4FLsdhexekgP3bLpw4Kgde+9DZuDZQ4yIInA=="], + "@oxlint/binding-darwin-x64": ["@oxlint/binding-darwin-x64@1.81.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-q0QTm32jWga2Gv4j7IaVZN0jYMi9UV73sWVgFtDA4iIfqwMCLLZ3ve+9KwfYtsaKZSgQhmPaogeZWqDZpcY1Pw=="], - "@oxlint/binding-freebsd-x64": ["@oxlint/binding-freebsd-x64@1.79.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-qDMm0dXZnoHyRqSL4N4xUq82T4sqK5cbKSjvd/dF/YbMUXc2R1wEPf+vmA5S0qUmi0nwXfNbjXBtZaIqzQLIMg=="], + "@oxlint/binding-freebsd-x64": ["@oxlint/binding-freebsd-x64@1.81.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-/+8wVWDXEC7wHVAhOc59Fw/SkMc1arLkFD8iQCaSsmzenK1X4doFqquL9H1wrtGUzaiycVqkf/sSpcILK6W1UA=="], - "@oxlint/binding-linux-arm-gnueabihf": ["@oxlint/binding-linux-arm-gnueabihf@1.79.0", "", { "os": "linux", "cpu": "arm" }, "sha512-2od7s0nuKPzqyUZAWk9KkCyGg7eI9dwFPZg+20lB15fKFkVZ0c9ZFxqPfiBAyDTlTkh9stPI0t+JlPCqMbItVA=="], + "@oxlint/binding-linux-arm-gnueabihf": ["@oxlint/binding-linux-arm-gnueabihf@1.81.0", "", { "os": "linux", "cpu": "arm" }, "sha512-4xt422FEgioRq9hAL4Tq7fujGUWnc8z1BJ+Oi8RN8vB8axaP+sdK6a2xdlcQCCYnJg9QMuMFS0AucuIFx/EacA=="], - "@oxlint/binding-linux-arm-musleabihf": ["@oxlint/binding-linux-arm-musleabihf@1.79.0", "", { "os": "linux", "cpu": "arm" }, "sha512-ZOQUjkzDnvlhSE3+tWC3YXx94MMl+sYMlwH+u1+YGApGHOJP/YAc8ZBRFOXZ6eOBmxtXAWuS/fBcdZr8qqNO1A=="], + "@oxlint/binding-linux-arm-musleabihf": ["@oxlint/binding-linux-arm-musleabihf@1.81.0", "", { "os": "linux", "cpu": "arm" }, "sha512-u3vna8KdGplH4DRCW9K54D68fcMo7IxVrkCJWwXnIhwtBdnDnYrmzOUA/XjmBlPpcLsgw9Z5BNdY4za9+Dj+MQ=="], - "@oxlint/binding-linux-arm64-gnu": ["@oxlint/binding-linux-arm64-gnu@1.79.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-lu158FR4nGqGeRS3BQvtG85wRgU/Fy4MD5Cxp1hzJXizGiLo6u2742wJSCDKh8cFcZntvX7fcxlq4mMmfryH1g=="], + "@oxlint/binding-linux-arm64-gnu": ["@oxlint/binding-linux-arm64-gnu@1.81.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-3j9k+gsYsE7nv71GWotXsqsa2l9/aJenD7dVHNt/CBvsb0SgRjSMnHFeP59IXUAl1wvVFhqGl2wJNMwWU3UBlA=="], - "@oxlint/binding-linux-arm64-musl": ["@oxlint/binding-linux-arm64-musl@1.79.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-mbpKQeE2aflTjddaHK7MP8KP/OFbUM++lt5M635ENM8IyIdK0jm2t9pb+2v9mVVIvhF6TqA4l7F79Pll1mi+uw=="], + "@oxlint/binding-linux-arm64-musl": ["@oxlint/binding-linux-arm64-musl@1.81.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-k5iAp3dNxW0/uDCBY+WSm8jKB2szu7SkEQZdgRRpDXvuDd69vvDcqhB3A/pWCfCwXyenjNjFn9Td1fVoyAc+Yg=="], - "@oxlint/binding-linux-ppc64-gnu": ["@oxlint/binding-linux-ppc64-gnu@1.79.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-WpGNua7gaxaHnpSDeog2ji8IDHn/QLPl9LPzwkR/FvVv58vT5BcXjRXnU+wbu3N75cpeha8CdC7ho/U2OIsB4g=="], + "@oxlint/binding-linux-ppc64-gnu": ["@oxlint/binding-linux-ppc64-gnu@1.81.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-TFqLja3uYmVSte6nof9GWrex9Z8WgdZrNiLC6Te5rXGDqXB2y4j/26iFhwosXiAFqDhE9JJVuuCkDKLwptTn1g=="], - "@oxlint/binding-linux-riscv64-gnu": ["@oxlint/binding-linux-riscv64-gnu@1.79.0", "", { "os": "linux", "cpu": "none" }, "sha512-tK1E93A5LVzISg4ngpKJnfTs7EqtIUceGI7MQ4GyDjJiLi8wPCkEyKlj2xkyKWZ1yzkDJyLHTBJ5/iFWRdnJvg=="], + "@oxlint/binding-linux-riscv64-gnu": ["@oxlint/binding-linux-riscv64-gnu@1.81.0", "", { "os": "linux", "cpu": "none" }, "sha512-UEcySvGS0NOVo7h7n7CYyJL9+6gFAh7Zc/ToDXVScFvzHSTIxtzkMVU30rmQ6+nQ1LF+UdiRDdJajpDu+OylLg=="], - "@oxlint/binding-linux-riscv64-musl": ["@oxlint/binding-linux-riscv64-musl@1.79.0", "", { "os": "linux", "cpu": "none" }, "sha512-qhQvUIrngXivA2A9pQ+xPCychztn/5qUv7yS3gDwXv3w7Rag+eTeeXWmRyx+t7XsW5x6LuY/8AsTq36UgFIblg=="], + "@oxlint/binding-linux-riscv64-musl": ["@oxlint/binding-linux-riscv64-musl@1.81.0", "", { "os": "linux", "cpu": "none" }, "sha512-H+diDbhD00+wI1IRP8Kz88x/lat+DgtoBJzoTthS16xkTJGNaEkfb8gzmd1rzc/2uDQQMl7GNl+JFUacVeWxIA=="], - "@oxlint/binding-linux-s390x-gnu": ["@oxlint/binding-linux-s390x-gnu@1.79.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-sv6AaVgU/eE6u+6WFiQVDcPPwTxP6IJMSB9k701W2r/r6Tx465e8vPvVyRxquNH4Vy6KwRNu90mVbxXJN8+5gg=="], + "@oxlint/binding-linux-s390x-gnu": ["@oxlint/binding-linux-s390x-gnu@1.81.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-8znJ/5TekjOKg1j1Acho4PJMdiAHLtlcXuWEiipOhAMV6rQcXdmDdXCbheyDczN6TjBwiNfjcP81k4AthrKRzw=="], - "@oxlint/binding-linux-x64-gnu": ["@oxlint/binding-linux-x64-gnu@1.79.0", "", { "os": "linux", "cpu": "x64" }, "sha512-iFZL02deziHslb3jEX9KdqlAkYoo4fGyotchKDzdfK1f5mxlIBeiQeHhvK3iFpuEJSB4ma/qeFn9oxPiwnhUPQ=="], + "@oxlint/binding-linux-x64-gnu": ["@oxlint/binding-linux-x64-gnu@1.81.0", "", { "os": "linux", "cpu": "x64" }, "sha512-Q2Wj70yFsvn5QjlmifFzbj4H+kJy53bwqc41o1fzoM7MpLV1NIbhg/LpWXRfC6KOkSAdUx1Wd8VJsdPmhp/HRA=="], - "@oxlint/binding-linux-x64-musl": ["@oxlint/binding-linux-x64-musl@1.79.0", "", { "os": "linux", "cpu": "x64" }, "sha512-3DtZR2raqObnh7wXZoFYFd0Fw7skBvcb3f7A+/lkEiDuh8hrE6vv9b/62Qxao1a9/OeHLw/FcXlXzgsW9wTRFg=="], + "@oxlint/binding-linux-x64-musl": ["@oxlint/binding-linux-x64-musl@1.81.0", "", { "os": "linux", "cpu": "x64" }, "sha512-cPInHp/ddEe5qkyK2IiyQ8Q3Mp2oLLEhhsGgTK2oZx4L6+llGam1H1yBvJZ7qHfOXj8N3hxBS8sj4tO+gtFlIg=="], - "@oxlint/binding-openharmony-arm64": ["@oxlint/binding-openharmony-arm64@1.79.0", "", { "os": "none", "cpu": "arm64" }, "sha512-Oatt4GuA1WJkqzk2ozx4HrWROOi7opV3AKDw/U8qDIqeTqzsjn5K2x3REJMNjU3/KU/Bkq96Zi3CknaiDTaC/Q=="], + "@oxlint/binding-openharmony-arm64": ["@oxlint/binding-openharmony-arm64@1.81.0", "", { "os": "none", "cpu": "arm64" }, "sha512-0CQxSX4ajqm07AHBf5U33qQzXKdd7wtq/oTL/7vpY6RNNuxrRi8W4bqUV1Jyu/vj+9KmxQyDhxfeVX1nQL6kfg=="], - "@oxlint/binding-win32-arm64-msvc": ["@oxlint/binding-win32-arm64-msvc@1.79.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-NAgZr9Qp8nIA9rpo0JEvwiabTF/2UVqBNnupBG9X4kxXcQoScJUTi+qHhvabb9s/thgj5wQ4XcIaJvb+ZMgoKw=="], + "@oxlint/binding-win32-arm64-msvc": ["@oxlint/binding-win32-arm64-msvc@1.81.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-l0hbeISm9673hVrrQU8j/p2M7YH9Ouoj7p7E/QM55NTrKVLP+P3PF8hLu+OY+x0VtGRW+ggiQKZqmdYps9H+TA=="], - "@oxlint/binding-win32-ia32-msvc": ["@oxlint/binding-win32-ia32-msvc@1.79.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-+KyXjIvcpaXmWW/j9NNY5yWjrIVxaX18VyIheQy3jwc2GSYgpCr7MGI/HxIGQ/shAL5IWEKbhsqoMpAO5Stiog=="], + "@oxlint/binding-win32-ia32-msvc": ["@oxlint/binding-win32-ia32-msvc@1.81.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-ksqPP5jbFXcYreEQ7zdJh06rJQBymCTyGRCdaXjfcf2aG4f8KxUWY5wcgYHmaTK+FJ4bPG5sUAdOX+6trnH1JA=="], - "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.79.0", "", { "os": "win32", "cpu": "x64" }, "sha512-mEelcCMMBS57sIXh2veGMNy+pQwuGtcMxHxGIZWQ5Ba9pJ5jCCUFOZB9E2JhBaxGsURe+WGe0zJp4RVre52gpQ=="], + "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.81.0", "", { "os": "win32", "cpu": "x64" }, "sha512-IZuUCwGw9emG5JtCp+fYGB+Z4OWEoeEcM8R5BA1pYw63/ieYFVdcU2ylxTpHbVHSenZnsYE+ZZ20uHAJszQ4cA=="], "@stablelib/base64": ["@stablelib/base64@1.0.1", "", {}, "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ=="], - "@tanstack/query-core": ["@tanstack/query-core@5.100.10", "", {}, "sha512-8UR0yJR+GiQ40m3lPhUr0xbfAupe6GSQiksSBSa9SM2NjezFyxXCIA69/lz8cSoNKZLrw1/PktIyQBJcVeMi3w=="], + "@tanstack/query-core": ["@tanstack/query-core@5.102.8", "", {}, "sha512-ZNjkJ33CqvPNec/6lZBnHqLc3EVGPZ9ySLhYahU9TcuRFdmwXewuj0c4hwSWcGHqEUwcSrKeZ+oGcvPBqXcQcg=="], - "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + "@types/bun": ["@types/bun@1.4.1", "", { "dependencies": { "bun-types": "1.4.1" } }, "sha512-0AVGiTXGajf1rgKom3N+c5L7CBxuoyyv1i44M0nX4UDK0G/fnRAMiri93nHuVPIb429KKtAgj7HatVmmOjeQLA=="], - "@types/node": ["@types/node@25.2.3", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-m0jEgYlYz+mDJZ2+F4v8D1AyQb+QzsNqRuI7xg1VQX/KlKS0qT9r1Mo16yo5F/MtifXFgaofIFsdFMox2SxIbQ=="], + "@types/node": ["@types/node@26.4.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA=="], "@types/semver": ["@types/semver@7.8.0", "", {}, "sha512-1mAINjtQCXXeLkJ9ehXkwOcBpqtLxiVtKhpUf83DdRNdQKV0iXZpaHYqRr7nj+wvxuJzoAmAwXI+sCNMv1CzLQ=="], @@ -325,7 +324,7 @@ "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], - "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + "bun-types": ["bun-types@1.4.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-loKuVrAFZKfEv+JvWkHRS9GW5IqLuLRjVXN9p+vZvBN86O5hf/pBZQ5hSoyipsrMmWObZBDvWnlmKvjKTM0PdA=="], "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], @@ -351,6 +350,8 @@ "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "csv-parser": ["csv-parser@3.2.1", "", { "bin": { "csv-parser": "bin/csv-parser" } }, "sha512-v8RPMSglouR9od735SnwSxLBbCJqEPSbgm1R5qfr8yIiMUCEFjox56kRZid0SvgHJEkxeIEu3+a9QS3YRh7CuA=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], @@ -387,11 +388,11 @@ "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], - "eventsource-parser": ["eventsource-parser@3.1.0", "", {}, "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg=="], + "eventsource-parser": ["eventsource-parser@3.1.1", "", {}, "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ=="], "express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="], - "express-rate-limit": ["express-rate-limit@8.6.0", "", { "dependencies": { "debug": "^4.4.3", "ip-address": "^10.2.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-XKJXDsASUOo0LLtFwW5hCcQGH0N4WQc/Rn8/Pvoia+TJFOkkFPvrtW9lZOeeNcxQJspvOIERMwiRLsVFlhHEkA=="], + "express-rate-limit": ["express-rate-limit@8.7.0", "", { "dependencies": { "debug": "^4.4.3", "ip-address": "^10.2.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g=="], "extendable-error": ["extendable-error@0.1.7", "", {}, "sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg=="], @@ -407,11 +408,11 @@ "fast-string-width": ["fast-string-width@3.0.2", "", { "dependencies": { "fast-string-truncated-width": "^3.0.2" } }, "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg=="], - "fast-uri": ["fast-uri@3.1.6", "", {}, "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q=="], + "fast-uri": ["fast-uri@3.1.7", "", {}, "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg=="], - "fast-wrap-ansi": ["fast-wrap-ansi@0.2.0", "", { "dependencies": { "fast-string-width": "^3.0.2" } }, "sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w=="], + "fast-wrap-ansi": ["fast-wrap-ansi@0.2.2", "", { "dependencies": { "fast-string-width": "^3.0.2" } }, "sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q=="], - "fastq": ["fastq@1.20.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw=="], + "fastq": ["fastq@1.20.3", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw=="], "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], @@ -425,8 +426,6 @@ "fs-extra": ["fs-extra@7.0.1", "", { "dependencies": { "graceful-fs": "^4.1.2", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw=="], - "fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], - "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], @@ -447,11 +446,11 @@ "hasown": ["hasown@2.0.4", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A=="], - "hono": ["hono@4.13.5", "", {}, "sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw=="], + "hono": ["hono@4.13.7", "", {}, "sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ=="], "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], - "human-id": ["human-id@4.1.3", "", { "bin": { "human-id": "dist/cli.js" } }, "sha512-tsYlhAYpjCKa//8rXZ9DqKEawhPoSytweBC2eNvcaDK+57RZLHGqNs3PZTQO6yekLFSuvA6AlnAfrw1uBvtb+Q=="], + "human-id": ["human-id@4.2.1", "", { "bin": { "human-id": "dist/cli.js" } }, "sha512-zPGsiS+dWoTZtZ4AtpA9Y+BdSFSNWvnouNlWNoUFyAM6xHOHmdCvqO3k8AIbdamCOv4gUFUVNPf6rJFfc4UiJw=="], "iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="], @@ -459,7 +458,7 @@ "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], - "ip-address": ["ip-address@10.5.0", "", {}, "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g=="], + "ip-address": ["ip-address@10.7.0", "", {}, "sha512-BGFsyJd5mpXp3rK6jIdADLNgpJUK1jnjzvYF8lK+VyDab9JAmqN0YOKDdP17HlgKb2+ehPgDc8EtnRLbGCAMhA=="], "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], @@ -479,7 +478,7 @@ "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - "jose": ["jose@6.2.3", "", {}, "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw=="], + "jose": ["jose@6.2.12", "", {}, "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw=="], "js-cookie": ["js-cookie@3.0.7", "", {}, "sha512-z/wZZgDrkNV1eA0ULjM/F9/50Ya8fbzgKneSpoPsXSGd0KnpdtHfOZWK+GcwLk+EZbS4F9RBhU+K2RgzuDaItw=="], @@ -495,11 +494,11 @@ "lodash.startcase": ["lodash.startcase@4.4.0", "", {}, "sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg=="], - "magicast": ["magicast@0.5.3", "", { "dependencies": { "@babel/parser": "^7.29.3", "@babel/types": "^7.29.0", "source-map-js": "^1.2.1" } }, "sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw=="], + "magicast": ["magicast@0.5.4", "", { "dependencies": { "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7", "source-map-js": "^1.2.1" } }, "sha512-llBEhWm1SacoRwgHUoQJYtwp4PBLF4faQi5TCpIGyGs9n4y5+juI0tDgyKIfpqxckRHaHzouUEph3THklWh03w=="], "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], - "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="], + "media-typer": ["media-typer@1.1.1", "", {}, "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ=="], "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], @@ -517,7 +516,7 @@ "nano-staged": ["nano-staged@1.0.2", "", { "bin": { "nano-staged": "lib/bin.js" } }, "sha512-Fytar3zHLY99nlMfqPPbraxZodqQAHPpdPRyYaplL+lB9DCR6pUrafxbG+Btz4+7fO5Rm/+DO4ZeDO/nLSUMhw=="], - "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], + "negotiator": ["negotiator@1.1.0", "", { "dependencies": { "content-type": "^2.1.0" } }, "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg=="], "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], @@ -531,7 +530,7 @@ "oxfmt": ["oxfmt@0.64.0", "", { "dependencies": { "tinypool": "2.1.0" }, "optionalDependencies": { "@oxfmt/binding-android-arm-eabi": "0.64.0", "@oxfmt/binding-android-arm64": "0.64.0", "@oxfmt/binding-darwin-arm64": "0.64.0", "@oxfmt/binding-darwin-x64": "0.64.0", "@oxfmt/binding-freebsd-x64": "0.64.0", "@oxfmt/binding-linux-arm-gnueabihf": "0.64.0", "@oxfmt/binding-linux-arm-musleabihf": "0.64.0", "@oxfmt/binding-linux-arm64-gnu": "0.64.0", "@oxfmt/binding-linux-arm64-musl": "0.64.0", "@oxfmt/binding-linux-ppc64-gnu": "0.64.0", "@oxfmt/binding-linux-riscv64-gnu": "0.64.0", "@oxfmt/binding-linux-riscv64-musl": "0.64.0", "@oxfmt/binding-linux-s390x-gnu": "0.64.0", "@oxfmt/binding-linux-x64-gnu": "0.64.0", "@oxfmt/binding-linux-x64-musl": "0.64.0", "@oxfmt/binding-openharmony-arm64": "0.64.0", "@oxfmt/binding-win32-arm64-msvc": "0.64.0", "@oxfmt/binding-win32-ia32-msvc": "0.64.0", "@oxfmt/binding-win32-x64-msvc": "0.64.0" }, "peerDependencies": { "svelte": "^5.0.0", "vite-plus": "*" }, "optionalPeers": ["svelte", "vite-plus"], "bin": { "oxfmt": "bin/oxfmt" } }, "sha512-XZ4GFBN/PLbXKq+0zrgpQfPKYuJlUuj+nzZJY7UpIbFMNyefNLCdN9EwViycNqnYcv0wrn0jXcQLlqJp8RCKBg=="], - "oxlint": ["oxlint@1.79.0", "", { "optionalDependencies": { "@oxlint/binding-android-arm-eabi": "1.79.0", "@oxlint/binding-android-arm64": "1.79.0", "@oxlint/binding-darwin-arm64": "1.79.0", "@oxlint/binding-darwin-x64": "1.79.0", "@oxlint/binding-freebsd-x64": "1.79.0", "@oxlint/binding-linux-arm-gnueabihf": "1.79.0", "@oxlint/binding-linux-arm-musleabihf": "1.79.0", "@oxlint/binding-linux-arm64-gnu": "1.79.0", "@oxlint/binding-linux-arm64-musl": "1.79.0", "@oxlint/binding-linux-ppc64-gnu": "1.79.0", "@oxlint/binding-linux-riscv64-gnu": "1.79.0", "@oxlint/binding-linux-riscv64-musl": "1.79.0", "@oxlint/binding-linux-s390x-gnu": "1.79.0", "@oxlint/binding-linux-x64-gnu": "1.79.0", "@oxlint/binding-linux-x64-musl": "1.79.0", "@oxlint/binding-openharmony-arm64": "1.79.0", "@oxlint/binding-win32-arm64-msvc": "1.79.0", "@oxlint/binding-win32-ia32-msvc": "1.79.0", "@oxlint/binding-win32-x64-msvc": "1.79.0" }, "peerDependencies": { "oxlint-tsgolint": ">=7.0.2001", "vite-plus": "*" }, "optionalPeers": ["oxlint-tsgolint", "vite-plus"], "bin": { "oxlint": "bin/oxlint" } }, "sha512-hVJ9hq9m2unPS+Of4eJJgCPdIeCC+3DHEUX3tkmrPJr3OK2hz7PhXwgC+ZP71ZcYu8cCDEtQrqLxWNvxBppBVg=="], + "oxlint": ["oxlint@1.81.0", "", { "optionalDependencies": { "@oxlint/binding-android-arm-eabi": "1.81.0", "@oxlint/binding-android-arm64": "1.81.0", "@oxlint/binding-darwin-arm64": "1.81.0", "@oxlint/binding-darwin-x64": "1.81.0", "@oxlint/binding-freebsd-x64": "1.81.0", "@oxlint/binding-linux-arm-gnueabihf": "1.81.0", "@oxlint/binding-linux-arm-musleabihf": "1.81.0", "@oxlint/binding-linux-arm64-gnu": "1.81.0", "@oxlint/binding-linux-arm64-musl": "1.81.0", "@oxlint/binding-linux-ppc64-gnu": "1.81.0", "@oxlint/binding-linux-riscv64-gnu": "1.81.0", "@oxlint/binding-linux-riscv64-musl": "1.81.0", "@oxlint/binding-linux-s390x-gnu": "1.81.0", "@oxlint/binding-linux-x64-gnu": "1.81.0", "@oxlint/binding-linux-x64-musl": "1.81.0", "@oxlint/binding-openharmony-arm64": "1.81.0", "@oxlint/binding-win32-arm64-msvc": "1.81.0", "@oxlint/binding-win32-ia32-msvc": "1.81.0", "@oxlint/binding-win32-x64-msvc": "1.81.0" }, "peerDependencies": { "oxlint-tsgolint": ">=7.0.2001", "vite-plus": "*" }, "optionalPeers": ["oxlint-tsgolint", "vite-plus"], "bin": { "oxlint": "bin/oxlint" } }, "sha512-HyrJYqeoOCL0iqaLEzGewGT48ZX99P3hxYh8udAF9RGGIghSamkXE4ClUyBpEDNqasamThgmlPbuMOe7SAZmHg=="], "oxlint-tsgolint": ["oxlint-tsgolint@7.0.2001", "", { "optionalDependencies": { "@oxlint-tsgolint/darwin-arm64": "7.0.2001", "@oxlint-tsgolint/darwin-x64": "7.0.2001", "@oxlint-tsgolint/linux-arm64": "7.0.2001", "@oxlint-tsgolint/linux-x64": "7.0.2001", "@oxlint-tsgolint/win32-arm64": "7.0.2001", "@oxlint-tsgolint/win32-x64": "7.0.2001" }, "bin": { "tsgolint": "./bin/tsgolint.js" } }, "sha512-KjK/XLcXr1DSyonKhsuFqJRiuKqcyG9j3LJ8nkOsrLzGvodBPqzHOKauy10asLMDI0sUpvb+1sxlzff3udZvfg=="], @@ -565,15 +564,15 @@ "pkce-challenge": ["pkce-challenge@5.0.1", "", {}, "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ=="], - "playwright": ["playwright@1.60.0", "", { "dependencies": { "playwright-core": "1.60.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA=="], + "playwright": ["playwright@1.63.0", "", { "dependencies": { "playwright-core": "1.63.0" }, "bin": { "playwright": "cli.js" } }, "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg=="], - "playwright-core": ["playwright-core@1.60.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA=="], + "playwright-core": ["playwright-core@1.63.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg=="], "prettier": ["prettier@2.8.8", "", { "bin": { "prettier": "bin-prettier.js" } }, "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q=="], "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], - "qs": ["qs@6.15.3", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A=="], + "qs": ["qs@6.16.0", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA=="], "quansync": ["quansync@0.2.11", "", {}, "sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA=="], @@ -629,7 +628,7 @@ "sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="], - "standardwebhooks": ["standardwebhooks@1.0.0", "", { "dependencies": { "@stablelib/base64": "^1.0.0", "fast-sha256": "^1.3.0" } }, "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg=="], + "standardwebhooks": ["standardwebhooks@1.1.1", "", { "dependencies": { "@stablelib/base64": "^1.0.0", "fast-sha256": "^1.3.0" } }, "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ=="], "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], @@ -653,7 +652,7 @@ "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], - "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], "universalify": ["universalify@0.1.2", "", {}, "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="], @@ -667,23 +666,13 @@ "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], - "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + "zod": ["zod@4.5.4", "", {}, "sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA=="], "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="], - "@changesets/apply-release-plan/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], - - "@changesets/assemble-release-plan/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], - - "@changesets/get-dependents-graph/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], + "@inquirer/external-editor/chardet": ["chardet@2.2.0", "", {}, "sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA=="], - "@clerk/backend/@clerk/shared": ["@clerk/shared@4.28.1", "", { "dependencies": { "@tanstack/query-core": "^5.100.6", "dequal": "2.0.3", "glob-to-regexp": "0.4.1", "js-cookie": "3.0.7" }, "peerDependencies": { "react": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0", "react-dom": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-OhpUczN6t8CYJ+g8HdzN1O4SFC2EANOZRDwlE3rXxKu13eNtyFbLspt+d6Nhb/PmcThS/fIAKYnQQrwv0vIEwg=="], - - "@clerk/testing/@clerk/shared": ["@clerk/shared@4.28.1", "", { "dependencies": { "@tanstack/query-core": "^5.100.6", "dequal": "2.0.3", "glob-to-regexp": "0.4.1", "js-cookie": "3.0.7" }, "peerDependencies": { "react": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0", "react-dom": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-OhpUczN6t8CYJ+g8HdzN1O4SFC2EANOZRDwlE3rXxKu13eNtyFbLspt+d6Nhb/PmcThS/fIAKYnQQrwv0vIEwg=="], - - "@inquirer/external-editor/chardet": ["chardet@2.1.1", "", {}, "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ=="], - - "@inquirer/external-editor/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "@inquirer/external-editor/iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], "@manypkg/find-root/@types/node": ["@types/node@12.20.55", "", {}, "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ=="], @@ -693,15 +682,17 @@ "@manypkg/get-packages/fs-extra": ["fs-extra@8.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g=="], - "body-parser/content-type": ["content-type@2.0.0", "", {}, "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="], + "body-parser/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], + + "body-parser/iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], - "body-parser/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "negotiator/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], - "raw-body/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "raw-body/iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], "read-yaml-file/js-yaml": ["js-yaml@3.15.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w=="], - "type-is/content-type": ["content-type@2.0.0", "", {}, "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="], + "type-is/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], "read-yaml-file/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], } diff --git a/packages/cli-core/package.json b/packages/cli-core/package.json index b7db8c47e..8bf0bf0fe 100644 --- a/packages/cli-core/package.json +++ b/packages/cli-core/package.json @@ -22,11 +22,13 @@ "@commander-js/extra-typings": "^15.0.0", "@napi-rs/keyring": "^1.3.0", "commander": "^15.0.0", + "csv-parser": "^3.2.1", "env-paths": "^4.0.0", "external-editor": "^3.1.0", "magicast": "^0.5.3", "semver": "^7.8.5", - "yaml": "^2.9.0" + "yaml": "^2.9.0", + "zod": "^4.4.3" }, "devDependencies": { "@clerk/shared": "^4.29.1", diff --git a/packages/cli-core/src/cli-program.ts b/packages/cli-core/src/cli-program.ts index ad53816fc..1642c69f9 100644 --- a/packages/cli-core/src/cli-program.ts +++ b/packages/cli-core/src/cli-program.ts @@ -24,6 +24,7 @@ import { registerCompletion } from "./commands/completion/index.ts"; import { registerUpdate } from "./commands/update/index.ts"; import { registerDeploy } from "./commands/deploy/index.ts"; import { registerWebhooks } from "./commands/webhooks/index.ts"; +import { registerMigrate } from "./commands/migrate/index.ts"; import { getEnvironment } from "./lib/config.ts"; import { setCurrentEnv, @@ -83,6 +84,7 @@ const registrants: CommandRegistrant[] = [ registerUpdate, registerDeploy, registerWebhooks, + registerMigrate, registerExtras, ]; diff --git a/packages/cli-core/src/commands/init/scan.ts b/packages/cli-core/src/commands/init/scan.ts index 7dbc599a3..6f8458906 100644 --- a/packages/cli-core/src/commands/init/scan.ts +++ b/packages/cli-core/src/commands/init/scan.ts @@ -54,6 +54,11 @@ const AUTH_LIBRARY_SCANS: AuthLibraryScan[] = [ name: "Better Auth", docsUrl: "https://clerk.com/docs/migrations/overview", }, + { + packages: ["@workos-inc/authkit-nextjs", "@workos-inc/node"], + name: "WorkOS", + docsUrl: "https://clerk.com/docs/migrations/overview", + }, { packages: ["@kinde-oss/kinde-auth-nextjs"], name: "Kinde", diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md new file mode 100644 index 000000000..15d1e18ab --- /dev/null +++ b/packages/cli-core/src/commands/migrate/README.md @@ -0,0 +1,1271 @@ +# `clerk migrate` + +Migrate users into a Clerk instance from another auth provider, or from another +Clerk instance. + +## Targeting And Auth + +`clerk migrate import` resolves its Backend API key through the CLI's standard +chain: + +| Flag | Description | +| -------------------- | ------------------------------------------------------------- | +| `--secret-key ` | Use a specific Backend API secret key directly | +| `--app ` | Target an application directly, even outside a linked project | +| `--instance ` | Target `dev`, `prod`, or a full instance ID | + +Resolution order: `--secret-key` → `--app` + Platform API lookup → +`CLERK_SECRET_KEY` → the keyless project's own key → a linked project profile +from `clerk link`. + +The **instance type is read from the key**: `sk_live_…` is treated as +production, anything else as development. That choice drives the throughput +defaults and the hard development-instance cap below. + +## Commands + +`clerk migrate` on its own is a group name, not a command: it prints its help +and lists the subcommands below. The direction is always spelled out — +`migrate import` moves users **into** Clerk, `migrate export` gets them **out** +of a source platform — so neither is implied by the group. + +### `clerk migrate import` (interactive) + +Bare `clerk migrate import` walks a human through the import instead of +demanding flags. + +```sh +clerk migrate import +``` + +It picks the transformer from a list built off the registry, asks for the file, +collects Firebase's hash parameters when they are needed, and pre-fills the +platform and file from the last run so a repeat migration is mostly pressing +enter. Anything already passed as a flag is not asked for. Firebase's hash +parameters are never pre-filled — see [below](#--firebase--firebase). + +Then it prints the [Migration Readiness report](#migration-readiness-report), +offers to [change whatever it flagged](#changing-the-flagged-settings), and +waits for confirmation. Declining writes nothing to Clerk. + +**Agent mode never prompts.** `clerk migrate import` with no flags exits with a +usage error naming exactly what to pass: + +``` +`clerk migrate import` is interactive and cannot prompt in agent mode. +Pass --transformer and --file . +``` + +### `clerk migrate import` + +Reads an exported user file, maps it onto Clerk's user schema, validates every +record, and creates the users through the Backend API. + +```sh +clerk migrate import -y --transformer clerk --file users.json +``` + +| Flag | Description | +| --------------------------------------- | --------------------------------------------------------------- | +| `-t, --transformer ` | Source platform the file came from (see below) | +| `--transformer-file ` | A transformer you wrote, for a platform with no built-in | +| `-f, --file ` | Path to the export. `.json` or `.csv` | +| `-r, --resume-after ` | Skip every user up to and including this **source** ID | +| `--require-password` | Import only users that carry a password digest | +| `--skip-unsupported-providers` | Supabase: skip users whose only social provider is off in Clerk | +| `--firebase-signer-key ` | Firebase base64 signer key | +| `--firebase-salt-separator ` | Firebase base64 salt separator | +| `--firebase-rounds ` | Firebase scrypt rounds | +| `--firebase-mem-cost ` | Firebase scrypt memory cost | +| `-y, --yes` | Skip the confirmation prompt | + +Plus the targeting flags from the table above: `--secret-key`, `--app` and +`--instance`. + +`--transformer` and `--file` are required. Omitting either fails with a usage +error that names the valid values. + +Failures do not stop the run: each user's outcome is written to the log and the +import continues. A `429` backs off — honouring `Retry-After` when the response +carries it — and retries up to 5 times before the user is recorded as failed. +The command exits non-zero if any user failed. + +Two failures do abort the whole run, because continuing would produce a +corrupt instance: + +- An **unrecognized password hasher**, which would import credentials nobody can + sign in with. +- A **`--resume-after` ID that is not in the file**, which would otherwise + re-import every user the previous run already created. + +#### Additional identifiers + +Only the first verified email and phone go on `POST /v1/users`. Every +additional verified identifier, and every unverified one, is attached +afterwards with its own request. A failure there is logged and the user still +counts as imported — a duplicate secondary email should not undo an otherwise +successful user. + +#### Throughput + +Defaults follow Clerk's documented `POST /v1/users` limits: 100 req/s for +production instances, 10 req/s for development. Concurrency defaults to ~95% of +that, assuming ~100ms of API latency. Both are overridable: + +| Variable | Effect | +| --------------------------------- | ----------------------------- | +| `CLERK_MIGRATE_RATE_LIMIT` | Requests per second | +| `CLERK_MIGRATE_CONCURRENCY_LIMIT` | Concurrent in-flight requests | + +A non-numeric or non-positive value is ignored in favour of the default. + +**Development instances warn when an import may exceed their user limit.** New +development instances are created with a 100-user limit; production instances +have none. Before importing, the run reads the instance's current user count +(`GET /v1/users/count`) and warns when the file would take it past 100. + +The run then stops and asks before going ahead. It is a prompt rather than a +hard refusal because the number checked against may not be this instance's: +Clerk raises a development instance's limit on request, and the raised value +(`max_allowed_users`) is not served by BAPI, DAPI or FAPI — so the CLI can show +the live count but never the live limit. Declining aborts before anything is +written to Clerk; `-y` and agent mode proceed on the warning alone. + +Users that do exceed the limit come back in the error breakdown as +`You have reached your limit of N users`, annotated with what a development +instance can do about it. + +### `clerk migrate export` + +Gets users **out** of a source platform, so there is something to feed +`clerk migrate import`. + +```sh +clerk migrate export # pick a platform +clerk migrate export clerk --output users.json +clerk migrate export auth0 --domain my-tenant.us.auth0.com \ + --client-id … --client-secret … +clerk migrate export workos --api-key sk_… +``` + +The platform is an optional positional. Omitted, you get a picker built from +the registry; given, it runs directly. Each platform resolves its own flags — +what Auth0 needs (a tenant domain and M2M credentials) has nothing in common +with what a database export needs. + +**A credential the far end rejects is asked for again.** Connection strings, +Firebase service account keys and Auth0 client secrets are all long, pasted by +hand, masked as they are typed, and wrong in ways nothing local can check: a +typo'd host, a revoked key, an expired token, the right server but the wrong +database. Only the connection or the token exchange can say, and by then the +operator has answered every other question the command asked. So that step — +and only that step, never a fetch already under way or a file already written — +runs inside a retry: the failure is explained, the prompt comes back, and the +rest of the export continues against whichever credential worked. Agent mode +and a non-TTY fail outright instead, having nobody to ask, and `-y` fails too, +having been told not to. + +| Platform | Source | Feeds | +| ------------ | -------------------------------- | -------------------------- | +| `clerk` | Clerk Backend API | `--transformer clerk` | +| `auth0` | Auth0 Management API | `--transformer auth0` | +| `supabase` | Supabase Postgres (`auth.users`) | `--transformer supabase` | +| `authjs` | Auth.js database | `--transformer authjs` | +| `betterauth` | Better Auth database | `--transformer betterauth` | +| `firebase` | Firebase Identity Toolkit | `--transformer firebase` | +| `workos` | WorkOS User Management API | `--transformer workos` | + +Every export asks where to save the file before it starts, proposing +`./exports/-export-.json`. Press enter to take it, +or type over it to save somewhere else — the proposal is prefilled, so it is +one prompt rather than a confirm and a path question. + +The stamp is ISO 8601 basic format in local time, to the minute: it goes in a +name people read off the screen and tab-complete, and it means a second export +never silently overwrites the first. + +`--output` answers that prompt up front and skips it, as does agent mode, which +takes the proposed path. `--output` resolves against the **current directory**, +like every other path flag here. + +`-y` does neither: it **fails**, naming `--output` and handing back the whole +command with the proposed path already in it, to run again. This is the one +prompt whose default cannot be undone by re-running — a file written where +nobody chose it has to be found and moved, and the second run writes a second +copy. Every other question `-y` silences has a default that costs nothing to +land on. Agent mode keeps defaulting even when it also passes `-y`, since there +was no prompt on that path to suppress. + +The question comes before any users are fetched, so a long export can be left +unattended rather than stalling on a prompt with everything held in memory. + +| Flag | Platforms | Description | +| -------------------------- | ---------------------------------- | ----------------------------------------------------------- | +| `-o, --output ` | all | Where to write the export | +| `-y, --yes` | all | Do not prompt: require `--output`, fail on a bad credential | +| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL, libsql/Turso or SQLite connection string | +| `--service-account ` | `firebase` | Path to a service account key JSON file | +| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | +| `--client-id ` | `auth0` | Machine-to-machine application client ID | +| `--client-secret ` | `auth0` | Machine-to-machine application client secret | +| `--api-key ` | `workos` | WorkOS secret API key, the one starting `sk_` | +| `--with-identities` | `workos` | Also record each user's OAuth providers | + +`export clerk` also takes the targeting flags — it reads from a Clerk instance, +so it resolves a key the same way `clerk migrate import` does, with one extra +step. The linked project is usually the migration's _destination_, so taking it +as the source without asking is how a run exports an instance and imports it +back into itself. Instead: + +- **Naming the instance runs unquestioned.** `--secret-key `, `--app`, + `--instance`, or an exported `CLERK_SECRET_KEY` — any of them is a sentence + you typed for this run, so none of them opens a picker. That is what makes + the export scriptable outside agent mode, and it keeps an exported key + outranking the linked profile here the way it does everywhere else in the + CLI. +- Anything resolved on your behalf — the linked project, a keyless app — is + never taken silently. A picker of every **instance** on your account opens + instead — one flat row each, `my-app - Production instance (ins_…)`, not an + application picker followed by an instance picker — with the resolved + application's instances listed **first** so taking one is still a single + Enter. Only when there are no instances to offer does it stop and list + `--secret-key`, `--app`/`--instance` and `clerk link` instead. +- With nothing to resolve at all (no link, no key, no flags), you get the + application picker `clerk users` uses — `Select a Clerk application to use:`, + followed by an instance picker when the application has more than one — + rather than an error about an unlinked directory. That is `clerk link`'s + picker, so it does offer `+ Create a new application`; a brand-new + application has no users to export, so it is never the answer here. + +The instance picker (the second tier) has no "create a new application" choice. +Its rows are searchable by what they show, so typing an application name, +`production`, or an instance id all narrow it. + +In agent mode the resolved instance is used without a prompt; pass +`--secret-key` or `--app`/`--instance` to be explicit. + +After each export you get a field-coverage table — which Clerk-relevant fields +were present on how many users — so you know the data is thin _before_ you +import it, not after: + +``` +Field coverage + ✓ 3/3 have an email address + ✗ 0/3 have a phone number + ! 1/3 have a username + ! 2/3 have a password (not exportable — see below) + +Exported 3 users to /project/exports/clerk-export-20260817-1432.json +└ Next steps + → Run `clerk migrate import --transformer clerk --file exports/clerk-export-20260817-1432.json` to import them +``` + +Every export also writes `logs/export-.log`, so `migrate logs list` +sees it alongside imports and deletions. + +#### Three platforms export no passwords + +- **Clerk** never returns password digests, TOTP secrets or backup codes over + the API — only the `*_enabled` booleans. Migrated users must reset their + password in the destination instance. +- **Auth0**'s Management API does not return password hashes either; they come + only from a support request. Add a `passwordHash` field to each user before + importing, or migrate without passwords. +- **WorkOS** returns neither password hashes nor TOTP secrets, and has no + support-request escape hatch: hashes go in on import and never come back, and + `totp.secret` is returned on enrol only. There is nothing to add to the file. + +All three say so on every run. The coverage row counts users who _have_ a +password, so the size of the gap is visible up front — `workos` prints that row +at zero unconditionally, because zero is the only value it can take. + +#### Database-backed exports (`supabase`, `authjs`, `betterauth`) + +These three read the database directly, over **`--db-url`**: + +```sh +clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" +clerk migrate export authjs --db-url "mysql://user:...@127.0.0.1:3306/authjs" +clerk migrate export betterauth --db-url "./db.sqlite" +clerk migrate export betterauth --db-url "libsql://app-org.turso.io?authToken=..." # or set TURSO_AUTH_TOKEN +``` + +Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; +`libsql://` (Turso) over the server's HTTP pipeline endpoint, since `bun:sqlite` +only opens local files and `@libsql/client` ships native optional dependencies. +Nothing native ships in the binary — that is the whole reason the `engines.bun` +floor exists. Resolution is `--db-url`, then `SUPABASE_DB_URL` / `AUTHJS_DB_URL` +/ `BETTERAUTH_DB_URL`, then a masked prompt, since a connection string carries +the password inline. A libsql token comes from `?authToken=` on the URL, or from +`TURSO_AUTH_TOKEN` / `LIBSQL_AUTH_TOKEN`, and is redacted like a password. + +**Connection strings are redacted everywhere.** Errors show +`postgres://***@host/db`, including when the password itself contains an +unencoded `@` — the most common mistake, and exactly when the string ends up in +an error message. + +Connection failures get a hint rather than a driver error. Bun reports both an +unreachable host and a closed port as "Connection closed", so: + +| Situation | What you are told | +| ------------------------ | ------------------------------------------------------------------------------------------------- | +| Host or port unreachable | Check the host and port. On Supabase: use the pooler connection string, or enable the IPv4 add-on | +| Credentials rejected | Check the user and password | +| Table missing | Check the database name and SELECT permission. On Supabase: enable Auth, connect as `postgres` | +| SQLite file missing | Check the path and that the file is readable | + +**`supabase` reads the database rather than the Admin API** because +`encrypted_password` exists only there. An API-based export would force every +user to reset their password; this one carries the bcrypt digests across. It +also keeps `raw_app_meta_data`, which is what `--skip-unsupported-providers` +reads at import time. + +**`authjs` tries `User`, then `user`, then `users`.** Auth.js has no single +schema — Prisma capitalizes the table, Drizzle does not, and Postgres treats +the difference as significant once quoted. The run reports which one it found. +Auth.js core stores no passwords, so its users arrive without credentials. + +**`betterauth` detects its plugin columns from the schema.** The username +plugin adds `username`, admin adds `banned`, phone-number adds `phoneNumber`, +and so on; selecting a column that is not there fails the whole query, and the +database answers the question better than the user can. Passwords come from a +`LEFT JOIN` onto the credential `account` row — left, not inner, so a user who +only ever signed in with OAuth is still exported. + +#### `firebase` + +```sh +clerk migrate export firebase --service-account ./service-account.json +``` + +Needs a service account key from **Project settings → Service accounts → +Generate new private key**, with the Firebase Authentication Admin role. + +Without `--service-account` you are prompted for it, the way `export supabase` +prompts for its connection string. The answer can be a path to the downloaded +file _or_ the key's JSON pasted whole, so a key kept in a password manager or a +CI secret never has to be written to disk. The prompt is masked, since the key +carries a private key. Agent mode cannot prompt, so it names the flag instead. + +Either way the key is validated before anything reaches the network, so +downloading the web app config by mistake fails in a second with the right +console page named rather than after an auth round-trip. Key material never +appears in output. + +Firebase's scrypt is a modified variant, so a digest is worthless without the +project's four hash parameters. The export **reads them from the project** and +prints the exact import command: + +``` +Password hash parameters +Read from the project. Import with: + clerk migrate import -y --transformer firebase --file exports/firebase-export.json --firebase-signer-key "…" --firebase-salt-separator "…" --firebase-rounds 8 --firebase-mem-cost 14 +``` + +On one line however long it gets: this prints inside the gutter, which prefixes +every line given to it with `│`. Split over lines with backslash continuations, +that character lands in the middle of the command and is copied along with it — +the shell then reads each one as another argument and rejects the import. A line +that wraps on screen carries no such character and pastes back as what was +printed. + +Reading the config needs a broader role than listing users, so if it is denied +the export still succeeds and points at **Authentication → Users → (⋮) → +Password hash parameters** instead. An export with no password hashes says so +and asks for nothing. + +A user whose hash is present but whose salt is not (or the reverse) has both +dropped: half a credential produces a user nobody can sign in as. + +`FIREBASE_AUTH_EMULATOR_HOST` is honoured, so this works against the local +Firebase emulator as well as production. + +**No `firebase-admin`.** The spike the plan called for was run and _passed_ — a +compiled binary can import the SDK and complete `listUsers`, so the known +Firestore-under-compile bug does not reach the Auth Admin surface. It was still +not adopted: the SDK is 74 MB across 158 packages, including Firestore and +Cloud Storage, which would roughly double the ~62 MB binary every user +downloads, to serve one subcommand. What it does here is two REST calls and an +RS256 JWT, and Bun's Web Crypto signs RS256 with no dependency at all. + +#### Auth0 credentials + +Needs a machine-to-machine application with the `read:users` scope +(Applications → APIs → Auth0 Management API → Machine to Machine +Applications). Resolved from flags, then `AUTH0_DOMAIN` / `AUTH0_CLIENT_ID` / +`AUTH0_CLIENT_SECRET`, then a prompt. In agent mode a prompt is impossible, so +it exits naming **every** missing credential at once rather than one per run. + +Auth0 pages this endpoint only through the first **1000** users. Past that the +export stops and says so, pointing at Auth0's bulk export job — silently +returning the first thousand would read as "that is everyone". + +#### WorkOS credentials + +Needs a secret API key — the one starting `sk_`, from the WorkOS dashboard +under API Keys. Resolved from `--api-key`, then `WORKOS_API_KEY`, then a +prompt; agent mode exits naming both instead. + +There is no `--db-url` sibling because there is no database to point it at. +WorkOS is API-only: apps commonly mirror users into their own store through +webhooks, but that mirror is a derived copy holding no credentials, so the +User Management API is the only source. Pagination is cursor-based, so unlike +Auth0 there is no record ceiling — `after` runs to the end of the tenant. + +**`--with-identities` is off by default, and it is not free.** WorkOS has no +bulk endpoint for OAuth identities, so it is one request per user: ten requests +becomes 1,010 for a thousand users. Nothing it returns can be imported — +`POST /v1/users` has no external-accounts field — so it buys a provider +breakdown in the coverage report, and an `identities` array kept in the export +file for whoever runs the migration. The interactive path asks once, after the +user count is known, defaulting to no; agent mode takes the flag's answer and +asks nothing. + +The breakdown prints as its own **OAuth providers** block under the coverage +table, not as extra coverage rows: + +``` +Field coverage + ✓ 6/6 have an email address + ✗ 0/6 have a password (WorkOS returns none) + +OAuth providers + GoogleOAuth 2 users + MicrosoftOAuth 1 user + no OAuth provider 2 users + not readable 1 user + Those users have no `identities` field in the export, rather than an empty one. +``` + +Separate because the two kinds of row do not mean the same thing. A coverage +row is "N of the M users have this field"; a provider row has no such +denominator — one user holding two providers is counted under both, so the +counts can sum past the user count, and `not readable` is not a property of the +user at all. + +A lookup that fails is counted on its own row rather than folded into +`no OAuth provider`. "Lookup failed" and "has no providers" are different +facts, and flattening the first into the second would understate social +sign-in. + +Non-interactive runs get progress on stderr every 500 users during the fan-out, +and every 10 pages during the user fetch. `withSpinner` hands a no-op to +anything that is not a TTY, so without this an agent exporting a large tenant +would see nothing at all until the run finished. + +### `clerk migrate delete` + +The undo for a bad migration. Deletes the users a previous +`clerk migrate import` created in this directory, matched by the `external_id` +the import stamped on each one. + +```sh +clerk migrate delete # confirms first +clerk migrate delete -y # non-interactive +``` + +Takes the same targeting flags as `clerk migrate import` (`--secret-key`, `--app`, +`--instance`). + +Flat rather than under a noun group: it is the one command in this tree that +destroys data **in Clerk**, and is worth keeping short and prominent. (Contrast +`migrate logs clean`, which only removes local files.) + +#### What it will and will not touch + +The saved migration record is the only account of what a run created, so that +is what identifies the migration being undone. Without it the command fails and +explains — deleting nothing silently would look like a successful undo. + +Users are found with `GET /v1/users?external_id=…`, 100 IDs per request. Only a +user Clerk itself reports as carrying one of _this_ migration's external IDs is +ever deleted; anything else in the instance is out of scope. IDs with no +matching user are skipped and reported, which is the normal case for a partial +migration or one already partly undone. + +It confirms before acting — defaulting to **no** — and requires `-y` in +non-interactive or agent mode. + +#### Failures + +Rate limiting and 429 retries are literally the same code path as the import +(`lib/retry.ts`), not a second implementation that drifts. + +A failure on one user is logged and the rest continue: a half-undone migration +with no record of which half is far worse than a reported failure. Every +attempt lands in a timestamped `logs/delete-.log`, carrying +both the source ID and the Clerk ID. The command exits non-zero if any deletion +failed. + +### `clerk migrate logs` + +Everything that touches the local log directory — `./logs` unless the project +says otherwise; see [Where logs go](#where-logs-go). Noun-verb like every +other group in the CLI (`config pull`, `users list`), rather than the standalone +tool's `clean-logs`/`convert-logs`, which were npm script names. + +Grouping also disambiguates the two deletes in this tree: `migrate logs clean` +removes **local files**, `migrate delete` removes **users from a Clerk +instance**. + +```sh +clerk migrate logs # defaults to list +clerk migrate logs list --json +clerk migrate logs clean -y +clerk migrate logs convert --all +clerk migrate logs convert import-2026-01-01T12-00-00.log +``` + +| Subcommand | Takes | Description | +| -------------- | ------------------ | ----------------------------------------------- | +| `logs list` | `--json` | File, type, date, size and entry count per file | +| `logs clean` | `-y, --yes` | Delete the `.log` files in the log directory | +| `logs convert` | `[file…]`, `--all` | NDJSON → a JSON array, written as `.json` | + +All three read the directory through one shared enumerator, which is what makes +`logs list` nearly free. All three **resolve** the directory without ever asking +for one: they are read-only, and "where should logs go?" is not a question to +put in front of someone who asked to see the logs they already have. + +#### Where logs go + +`./logs`, relative to the current directory, until the project says otherwise. +Resolution order, highest first: + +| Source | Set by | +| --------------------------- | ----------------------------------------------------- | +| `CLERK_MIGRATE_LOG_DIR` | The shell, `.env`, `.env.local`, `.env.clerk-migrate` | +| `log-dir` in the CLI config | The first-run prompt, or `settings set` | +| `./logs` | The fallback | + +The first time `migrate import`, `migrate export` or `migrate delete` runs +interactively in a project with none of those set, it asks where logs should be +saved and offers `./logs`. The answer is saved under `log-dir`, so it is asked +once per project and never again. `-y`, agent mode and a non-TTY take `./logs` +without asking **and without saving it** — landing on a default is not a choice, +and recording one would retire the question for a human who never saw it. + +Logs are the only record of which users landed and which failed, and +`migrate delete` reads them to undo a run, so where they go is worth the one +question. Change it later with `clerk migrate settings set log-dir `, or +clear it with `clerk migrate settings clear log-dir` to be asked again. + +#### `logs list` + +The default, because listing is read-only and therefore safe to run by +accident. Reports each file's name, type, date, size and entry count, newest +first; `--json` gives an agent the same data without parsing NDJSON. + +``` +Each log represents a user export, user import, or a user delete run. +Each log consists of a single NDJSON entry per user. + +FILE TYPE DATE SIZE ENTRIES +import-2026-02-01T09-14-22.log import Feb 1, 2026 at 4:14 AM 4.1 KB 120 +delete-2026-01-30T17-02-51.log delete Jan 30, 2026 at 12:02 PM 612 B 18 + +2 log files in ./logs + +Log types: + export One entry per user pulled from the source platform. + import One entry per user created in Clerk, with any error. + delete One entry per user removed from Clerk, with any error. +``` + +A kind is the name of the command that wrote it — `migrate import` writes +`import-.log` — so a listing points straight at the run behind each +line. The legend is fixed rather than derived from what happens to be present, +because "what else could be here" is the other half of the question. + +The older `migration-` and `user-deletion-` names, written by the standalone +tool and by earlier CLI builds, still classify as `import` and `delete`, so a +directory of old logs lists and converts unchanged. + +The filename leads, because it is what `logs convert` and `logs clean` talk +about. The date column renders the filename's UTC stamp in the reader's own +zone — "which run was that" is a question about local time; `--json` keeps the +raw stamp. + +The directory is printed relative (`./logs`) when it sits under the current +directory and absolute when it does not, so the path can be pasted either way. + +Says so plainly when the log directory is empty or absent. + +#### `logs clean` + +Destructive, so the confirmation is not optional: interactive runs prompt +(defaulting to **no**), and non-interactive or agent runs must pass `-y` rather +than being allowed to assume. Deletes `.log` files only — converted `.json` +output is left alone. + +#### `logs convert` + +Turns NDJSON into a JSON array for spreadsheet or database analysis, written +alongside the original as `.json`. The original is left in place. + +Takes file positionals or `--all`; given neither, an interactive terminal +offers a multiselect and an agent gets a usage error naming both alternatives. + +A malformed line is reported with its line number and skipped, and the +remaining entries still convert: + +``` +import-2026-01-01T12-00-00.log:2 is not valid JSON and was skipped — … +1 malformed line skipped. +``` + +That beats failing the whole file: a run killed mid-write leaves one truncated +final line, and the hundreds of complete entries before it are still worth +having. It also beats dropping the line silently, which would leave a JSON +array that looks complete. + +## Transformers + +A transformer maps one platform's export onto Clerk's user schema. Adding a +platform is one file in `transformers/` plus one line in `transformers/registry.ts` — +`--transformer`'s accepted values and its tab-completion both read from that array. + +| Key | Source | Passwords | Notes | +| ------------ | ----------------------------- | ----------------- | ---------------------------------------------------------------- | +| `clerk` | Clerk Dashboard export | as exported | Instance to instance, e.g. development → production | +| `auth0` | Auth0 Export Users API | `bcrypt` | Hashes need a support request to Auth0; not in a standard export | +| `authjs` | Auth.js / NextAuth user table | none | Assumes `SELECT id, name, email, email_verified, created_at` | +| `betterauth` | Better Auth export | `bcrypt` | Reads the credential account's `password_hash` | +| `firebase` | `firebase auth:export` | `scrypt_firebase` | CSV or JSON; needs the four hash parameters below | +| `supabase` | Supabase `auth.users` export | `bcrypt` | Supports `--skip-unsupported-providers` | +| `workos` | WorkOS User Management API | none | No hasher default: WorkOS returns no digest to name one for | + +### `clerk migrate transformers list` + +Which mappings are available. New in the CLI: the standalone tool's interactive +picker was the only place these appeared, which was fine when the user had the +source tree to grep. A compiled binary's users have neither. + +```sh +clerk migrate transformers list +clerk migrate transformers list --json +clerk migrate transformers list --transformer-file ./my-transformer.ts +``` + +| Flag | Description | +| --------------------------- | --------------------------------- | +| `--json` | Output as JSON | +| `--transformer-file ` | Also list a transformer you wrote | + +Each entry prints its key, the platform label, and what the transformer assumes +about the export — wrapped to the terminal, capped at 80 columns so two runs of +the same command lay out the same way. A backticked span is never broken across +lines. There is no intro/outro gutter: this reads a static registry rather than +running anything. + +``` +A transformer maps one platform's export onto the fields Clerk imports. Pass the +one your export came from as `--transformer `. + +Transformers: + clerk Clerk + Migrate between Clerk instances (e.g. development to production, or to + another Clerk application). Export your users from the Clerk Dashboard + first. + + … + + workos WorkOS + Works with WorkOS's User Management API. WorkOS returns no password hashes, + so imported users sign in by reset or SSO. + +7 built-in transformers +Migrating from something else? Write a transformer and pass --transformer-file. +``` + +`--json` gives an agent the same data, including which source field each +transformer maps to `userId`. + +### `clerk migrate settings` + +What a run in this directory would pick up, and where each value comes from. +Listing is the default, because it is the read-only one: a bare `clerk migrate +settings` shows, never changes. + +```sh +clerk migrate settings # list +clerk migrate settings list --json +clerk migrate settings set transformer firebase +clerk migrate settings set firebase-signer-key abc123 +clerk migrate settings clear firebase-signer-key # forget one +clerk migrate settings clear -y # forget them all +``` + +| Subcommand | Takes | Description | +| ----------------------------- | --------------------- | -------------------------------------------------------- | +| `settings list` | `--json` | Every setting, its value and the source it resolved from | +| `settings set ` | ` ` | Change one setting | +| `settings clear [name]` | `[name]`, `-y, --yes` | Forget one setting, or every setting and its credentials | + +`settings clear ` leaves the rest of the project's settings alone. For a +credential it drops every variable the setting answers to, aliases included — +clearing `firebase-rounds` while a bare `ROUNDS` stayed behind in the same file +would report the setting cleared and leave the next run reading the old value. +It only ever edits `.env.clerk-migrate`; a value coming from the app's own env +file or the shell is named in the listing's source column and has to be removed +there. + +**A bare `settings clear` needs `-y` where it cannot ask.** It forgets every +setting and every credential in `.env.clerk-migrate`, so a non-interactive or +agent run refuses rather than assuming, the way `migrate logs clean` and +`migrate delete` already do. `settings clear ` does not: naming the one +setting to forget is itself the confirmation, the same way `settings set` needs +none. + +A misspelled name gets the closest match back, not just the list: + +``` +$ clerk migrate settings clear logs-dir +error: command-argument value 'logs-dir' is invalid for argument 'name'. + Did you mean "log-dir"? Allowed choices are transformer, file, … +``` + +Setting names are kebab-case and identical to the `clerk migrate import` flag +each one backs, so `firebase-signer-key` here is `--firebase-signer-key` there +rather than a second spelling to learn. The description column carries the +prose. + +The source column is the point. A migration reads from flags, the environment, +two of the app's env files and the CLI's config, so when a run picks up a stale +value the question is never "what is it" but "which of those won". A value that +arrived under one of the accepted aliases names the variable alongside the file. + +It names a **file** wherever there is one to name. Bun loads `.env`/`.env.local` +into the environment before the CLI runs, so a value a developer typed into +`.env.local` would otherwise be reported as "`ROUNDS` env var" — true, and no +help to someone asking which file to edit. Attribution is by value: a file +holding the same key with a _different_ value lost to something exported in the +shell, and that row keeps saying `ROUNDS env var`, because that is exactly the +case this column exists to catch. + +A setting with no value leaves the column empty rather than filling it with a +placeholder — the source column already reads `not set` on that row, and the +blank is what makes the settings that do have a value stand out. + +It closes on next steps naming the two commands that change what it just +showed — the same block `clerk mcp list` and `clerk whoami` end on, and human +only. The full command surface stays in `--help`. + +``` +A migration run in this directory picks these up unless a flag overrides them. +Each setting is named after the `clerk migrate import` flag it stands in for. + +SETTING VALUE SOURCE DESCRIPTION +transformer firebase clerk config Source platform the export came from +file users.json clerk config Export file to import users from +skip-unsupported-providers not set Skip users with no provider enabled in Clerk (Supabase) +log-dir ./logs clerk config Directory migration logs are written to +firebase-signer-key [REDACTED] .env.clerk-migrate Firebase base64 signer key +firebase-salt-separator not set Firebase base64 salt separator +firebase-rounds 8 .env.local (ROUNDS) Firebase scrypt rounds +firebase-mem-cost 14 MEM_COST env var Firebase scrypt memory cost + +6 of 8 settings set. Credentials are shown redacted. + + → Run `clerk migrate settings set ` to change one + → Run `clerk migrate settings clear ` to forget one + → Run `clerk migrate settings clear` to forget them all, credentials included +``` + +#### Where each setting is kept + +Two stores, split by what the value **is** rather than by which command wrote it: + +| Store | Holds | Why | +| -------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------- | +| CLI config | `transformer`, `file`, `skip-unsupported-providers`, `log-dir` | Project state, not secret, useless outside the CLI | +| `.env.clerk-migrate` | `firebase-*` | Credentials: gitignored on write, and hand-editable for rotation | + +`log-dir` is the one setting that answers to both: it is remembered in the CLI +config, and `CLERK_MIGRATE_LOG_DIR` outranks what is remembered, so a directory +can be pinned for one shell without disturbing the project. The listing's source +column says which is winning, and `settings clear log-dir` clears both — half a +clear would report the setting gone while the next run still read it. + +`.env.clerk-migrate` is the migration's own file rather than the app's +`.env.local`, because a Firebase signer key is of no use to the application +being migrated and does not belong in the file its developers read daily. The +CLI adds it to `.gitignore` the first time it writes it, and deletes it when +`settings clear` removes the last value. + +Credentials are withheld wherever they are displayed, including under `--json`, +so the output is safe to paste into an issue. They display as `[REDACTED]` — +the same thing `clerk users create --dry-run` prints for a password — rather +than a truncation like `aVer…3456`: the source column already says which value +is in play, and a partial secret is one the reader has to recognise as partial. + +### Custom transformers (`--transformer-file`) + +Migrating from a platform with no built-in, without recompiling the CLI: + +```sh +clerk migrate import --transformer-file ./my-platform.ts --file users.json +``` + +The file lives in **your** project, not in the CLI, and is imported at runtime. +It exports the same shape the built-ins use — plain data, no imports, since +there is nothing in a compiled binary for your file to import from: + +```ts +export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform's admin console.", + transformer: { + account_ref: "userId", // required: becomes the Clerk user's external_id + contact_email: "email", + given: "firstName", + family: "lastName", + pw_bcrypt: "password", + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { + if (!user.firstName) delete user.firstName; + }, +}; +``` + +TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, +`satisfies` and `as const` all work in a file the compiled binary imports. +Plain `.js` works too. + +`--transformer-file` and `--transformer` together is an error: both name a +transformer and there is no sensible precedence between the one you wrote and +the one we ship. + +#### Validation + +The file is code the CLI executes, so its shape is checked before use and +rejected with the specific problem rather than crashing mid-pipeline: + +| Problem | Message | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------- | +| Path does not exist | `No transformer file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in transformer` | +| A hook is not a function | `postTransform must be a function when present` | + +The `userId` check is the load-bearing one: without it the import would run to +completion and create every user with no `external_id`, which is what makes a +migration re-runnable and what `migrate delete` matches on. + +### Verified vs unverified identifiers + +Every platform records verification differently, and each transformer declares +which style it uses. An identifier the source never confirmed is routed to +`unverifiedEmailAddresses` / `unverifiedPhoneNumbers` rather than the primary +field, because Clerk creates primary identifiers **verified** — sending an +unconfirmed address there would silently promote it. + +- **Boolean** (`auth0`, `betterauth`, `firebase`): `true`/`false`. A CSV export + stringifies these, so `"false"` is read as false, not as a non-empty string. +- **Timestamp** (`authjs`, `supabase`): a nullable confirmation time. Any real + value means verified; `""`, `null` and `\N` do not. + +### Firebase hash parameters + +Firebase uses a modified scrypt, so Clerk needs the project's four parameters +alongside each digest. Find them in the Firebase console under +**Authentication → Users → (⋮) → Password hash parameters**. + +```sh +clerk migrate import -y -t firebase -f users.json \ + --firebase-signer-key --firebase-salt-separator \ + --firebase-rounds 8 --firebase-mem-cost 14 +``` + +All four are **required as a set** — supplying some but not all is a usage error +naming what is missing. A partial set produces a well-formed digest that +verifies against nothing, so users would import successfully and then be unable +to sign in. + +They never go into the CLI's config: the signer key is a Firebase secret, and +that file is not a secret store. To avoid re-passing all four on every run, set +them once with [`clerk migrate settings`](#clerk-migrate-settings), or export +them yourself: + +| Flag | Variable | Also accepted | +| --------------------------- | ------------------------------- | --------------------------------------------------------- | +| `--firebase-signer-key` | `CLERK_FIREBASE_SIGNER_KEY` | `FIREBASE_BASE64_SIGNER_KEY`, `BASE64_SIGNER_KEY` | +| `--firebase-salt-separator` | `CLERK_FIREBASE_SALT_SEPARATOR` | `FIREBASE_BASE64_SALT_SEPARATOR`, `BASE64_SALT_SEPARATOR` | +| `--firebase-rounds` | `CLERK_FIREBASE_ROUNDS` | `FIREBASE_ROUNDS`, `ROUNDS` | +| `--firebase-mem-cost` | `CLERK_FIREBASE_MEM_COST` | `FIREBASE_MEM_COST`, `MEM_COST` | + +The unprefixed names are what Firebase itself calls these (`base64_signer_key`, +`rounds`) and what every guide, Clerk's own standalone migration script +included, tells you to paste into `.env`. Someone who followed one has the +values the import needs, spelled the way the source platform spells them, so +they are read rather than reported as missing. + +They are a fallback, not a synonym: a `CLERK_FIREBASE_*` variable wins wherever +both exist, and `clerk migrate settings` names the variable it read alongside +the file — `ROUNDS` is generic enough to mean something else in an app that was +never a Firebase project, and that should be visible rather than silent. + +Resolution order is flag, then exported variable, then `.env.clerk-migrate`, +then the app's `.env.local`/`.env`. The sources can be mixed as long as all four +end up supplied. Run with `--verbose` to see which one each came from. + +An export with no password hashes needs no parameters at all. + +### `--skip-unsupported-providers` (Supabase) + +Reads each user's `raw_app_meta_data.providers` and cross-references it against +the social providers the destination instance has enabled (via BAPI +`/v1/domains` → the instance's Frontend API `/v1/environment`). + +A user is skipped **only when every one of their providers is disabled**. Anyone +who can still sign in another way — email, phone, or an enabled social provider +— is imported. The number skipped is reported, broken down by provider. + +If the instance configuration cannot be read, nobody is skipped and a warning is +printed: a failed lookup must not be mistaken for "no providers are enabled". + +## Schema fields + +What a transformer maps _onto_. Every user is validated against this schema +before any request is made, so a field a transformer produces that is not listed +here is silently dropped — Zod strips unknown keys — and never reaches Clerk. +Writing a custom transformer means targeting these names exactly. + +The schema lives in `validator.ts`; adding a source platform means adding a +transformer, not editing it. + +**Required:** `userId` (`string`). It becomes the Clerk user's `external_id`, +which is what makes a migration re-runnable and what `migrate delete` matches on. + +**Identifiers.** At least one of these must be present, or the user is logged as +a validation failure and skipped. Each accepts a single value or an array. + +| Field | Type | Description | +| -------------------------- | -------------------- | ---------------------------------- | +| `email` | `string \| string[]` | Primary verified email address(es) | +| `emailAddresses` | `string \| string[]` | Additional verified emails | +| `unverifiedEmailAddresses` | `string \| string[]` | Unverified emails | +| `phone` | `string \| string[]` | Primary verified phone number(s) | +| `phoneNumbers` | `string \| string[]` | Additional verified phones | +| `unverifiedPhoneNumbers` | `string \| string[]` | Unverified phones | +| `username` | `string` | Username | + +**Profile, password and 2FA.** + +| Field | Type | Description | +| -------------------- | ---------- | --------------------------------------------------- | +| `firstName` | `string` | First name | +| `lastName` | `string` | Last name | +| `password` | `string` | The hashed password from the source platform | +| `passwordHasher` | `enum` | **Required whenever `password` is set** (see below) | +| `totpSecret` | `string` | TOTP secret | +| `backupCodesEnabled` | `boolean` | Whether backup codes are enabled | +| `backupCodes` | `string[]` | Backup codes | + +Clerk verifies the digest as-is, so `passwordHasher` must name the algorithm the +source actually used: + +`argon2i`, `argon2id`, `awscognito`, `bcrypt`, `bcrypt_peppered`, +`bcrypt_sha256_django`, `hmac_sha256_utf16_b64`, `ldap_ssha`, `md5`, +`md5_phpass`, `md5_salted`, `pbkdf2_sha1`, `pbkdf2_sha256`, +`pbkdf2_sha256_django`, `pbkdf2_sha512`, `pbkdf2_sha512_hex`, `scrypt_firebase`, +`scrypt_werkzeug`, `sha256`, `sha256_salted`, `sha512_symfony` + +An unrecognized hasher aborts the run rather than importing credentials nobody +can sign in with. + +**Metadata.** + +| Field | Type | Description | +| ----------------- | -------- | -------------------------------------------------------- | +| `unsafeMetadata` | `object` | Readable **and writable** by the client — never trust it | +| `publicMetadata` | `object` | Readable by the client, writable only server-side | +| `privateMetadata` | `object` | Server-side only | + +**Account state.** These are passed straight through to `POST /v1/users`, and +are how a Clerk-to-Clerk migration keeps original signup dates instead of +stamping every user with today's. + +| Field | Type | Description | +| --------------------------- | --------- | ----------------------------------------- | +| `createdAt` | `string` | Original creation timestamp | +| `legalAcceptedAt` | `string` | When legal terms were accepted | +| `banned` | `boolean` | Whether the user is banned | +| `bypassClientTrust` | `boolean` | Skip client trust verification | +| `createOrganizationEnabled` | `boolean` | Whether the user can create orgs | +| `createOrganizationsLimit` | `number` | Maximum orgs the user can create | +| `deleteSelfEnabled` | `boolean` | Whether the user can delete their account | +| `skipLegalChecks` | `boolean` | Skip legal acceptance checks | +| `skipPasswordChecks` | `boolean` | Skip password requirements on import | + +## Migration Readiness report + +Printed immediately before the confirmation prompt, so declining aborts with +nothing written to Clerk. Skipped only for `-y`, which says "don't ask, don't +lecture" and should not pay for the two extra round-trips. Agent runs without +`-y` still get it — an agent can act on it exactly as a human would. + +It cross-references the file against the destination instance's live settings +(BAPI `/v1/domains` → that instance's Frontend API `/v1/environment`) and +answers the two questions worth answering before writing anything: **who won't +be imported**, and **who will arrive incomplete**. + +``` +Migration readiness + 120 users in this file + 3 failed validation and will be skipped + + ✗ 12 users will not be imported + 12 have no email, which this instance requires + If you import them, this applies to them too: + 12 have a phone, which this instance is not set up to store + ⚠ 20 users will be imported, but not everything they carry + 14 have no password, which this instance requires — they will have to reset it to sign in + 6 have a username, which this instance is not set up to store + ✓ 88 users will be imported in full + +Identifiers + ⚠ Email — required in Clerk, and not every user has one — 108/120 users + ⚠ Username — not enabled in Clerk — 6/120 users + +Social connections + ✓ Google — enabled in Clerk — 40/120 users + ⚠ Discord — not enabled in Clerk — 12/120 users + +⚠ 3 settings need attention +``` + +### The two blocks + +**The outcome block** classifies each user **once**, into the worst outcome that +applies to them, so its three totals add up to the file. This matters: per-field +coverage cannot answer "how many won't be imported", because the users missing +an email and the users missing a password overlap by an amount only a per-user +pass knows. A user rejected for their missing email is not also counted under +the missing password they happen to share. + +**"If you import them, this applies to them too"** is the part that stops the +settings interacting invisibly. A user who is not being created cannot lose a +field, so a setting that only affects rejected users costs nothing _today_ and +would otherwise never be mentioned — right up until the operator relaxes the +requirement rejecting them, at which point all of it lands at once. Naming it +up front is what turns + +> make email optional → re-check → discover the phones are being dropped → +> enable phone → re-check + +into a single decision with both offers visible. It is also why a setting can +be flagged in the section rows while contributing nothing to the ✗/⚠/✓ totals. + +**The section rows below** are the other question — per-field coverage against +each setting — and deliberately do not restate user counts, which would read as +contradicting the block above. + +### Which settings cost what + +| Setting | Consequence | +| ------------------------------------------ | --------------------------------------------------------------------------------------------- | +| Identifier (email/phone/username) required | **Not imported.** `POST /v1/users` enforces the sign-up identifier requirements. | +| Password required, user has none | **Imported without a password.** The import sends `skip_password_requirement`, so the user is | +| | created and has to reset their password before they can sign in with one. | +| Attribute disabled in Clerk | **Imported without that field.** The instance has nowhere to put it. | +| Social provider disabled | **Imported**, but that sign-in method is unavailable to them. | + +Social rows are not part of the per-user outcome counts: which providers a user +signed up with lives in the raw export rather than the transformed user, so it +cannot be attributed per user. Their coverage row still names them. + +If the instance settings cannot be read — the secret key is rejected, or FAPI +is unreachable — the report degrades to a coverage-only listing with a note. +Nothing is flagged in that case: "could not read" is not the same as "switched +off", and treating it as such would raise alarms about settings that are +perfectly fine. + +### Changing the flagged settings + +When the report flags anything, a human run offers one selectable change per +flagged row before the import confirmation, so acting on the report does not +mean leaving the CLI for the dashboard: + +``` +Update this instance's settings first? (enter to skip) + ◻ Make Email optional at sign-up + ◻ Enable Discord sign-in + ↑/↓ to navigate • Space: select • a: all • Enter: confirm +``` + +**Nothing is preselected** — relaxing an instance's sign-up requirements is a +real decision, not a default — and selecting nothing continues to the import +prompt with the instance untouched, which is what "enter to skip" is there to +say. + +`a: all` is added to clack's legend in `lib/prompts.ts`: `MultiSelectPrompt` +has always bound `a` to toggle everything (and `i` to invert), but clack's +footer never listed them and takes no override, so the key was undiscoverable. +It applies to every multiselect in the CLI, because it is a property of the +prompt rather than of any one question. + +These are offers, not corrections: **a flagged setting is not a wrong setting.** +An instance that genuinely requires an email address is configured exactly as +its owner intended, and the right answer may well be to fix the export instead. + +Whatever is selected becomes a single `PATCH` of the instance config document, +the same document `clerk config patch` writes. The report is then redrawn so +the confirmation that follows is against the settings the write established. + +**The offer repeats while anything is still flagged.** A redraw is another +decision point, not a receipt: applying one change routinely leaves others +worth making, and each round re-offers only what is left. It ends when the +report has nothing flagged, when the operator selects nothing, or when there is +nothing offerable for the rows that remain — so reaching the second change +never costs a second run of the command. + +The redraw is computed from the write, **not** from a second settings fetch. +Clerk's Frontend API is eventually consistent, so a `/v1/environment` read +issued this soon after the config write routinely still reports the pre-write +settings — which would redraw the report with every row the operator just +cleared still flagged. The Platform API accepting the write is the +authoritative statement of what took, exactly as `clerk config patch` treats +it (see that command's [round-trip verification](../config/README.md#round-trip-verification) +notes for the same reasoning). + +The config leaves each option writes are not shown in the prompt — internal +detail an operator cannot act on — but they are fixed and listed here: + +| Flagged row | Change offered | +| ------------------------------- | --------------------------------------------------------------------------------- | +| Email/Phone/Username — required | `auth_.required_for_sign_up → false` | +| Email — disabled | `auth_email.used_for_sign_up → true` + `verification_strategies → ["email_code"]` | +| Phone — disabled | `auth_phone.used_for_sign_up → true` + `verification_strategies → ["phone_code"]` | +| Username — disabled | `auth_username.used_for_sign_up → true` | +| Password — required / disabled | `auth_password.required → false` / `auth_password.enabled → true` | +| First/Last name | `user_model..required → false` / `user_model..enabled → true` | +| Social provider — disabled | `connection_oauth_.enabled → true` | + +`used_for_sign_up` is the enable field that matters: `POST /v1/users` validates +an import against the instance's sign-up requirements, not its sign-in +strategies. + +**Email and phone take two writes, not one.** They are _verifiable_ attributes, +and Clerk rejects one that is on with no way to verify it: + +``` +422 phone_number: verifiable attributes need to have at least one verification +``` + +Switching the attribute off empties `verification_strategies`, so whatever +turns it back on has to put a strategy back in the same request. Username, +password and the name fields are not verifiable and take one write each. + +The offer is skipped entirely for `-y` and in agent mode, both of which say +"don't prompt". It also stands down, with a warning rather than a failed run, +when the instance to configure cannot be resolved (a bare `--secret-key` in an +unlinked directory) or when it is a **keyless** application — the Backend API +those are reachable through has no route for any of these settings, so +`clerk auth login` is the way in. + +## Artifacts + +Both are written relative to the **current working directory**, not to the +CLI's config directory, because they describe "which file am I migrating" +rather than "which project is linked here". + +| Path | Contents | +| ------------------------------------------ | --------------------------------------------------------------------- | +| `./logs/export-.log` | NDJSON: one line per exported user | +| `./logs/import-.log` | NDJSON: one line per user, plus validation failures and retry notices | +| `./logs/delete-.log` | NDJSON: one line per `migrate delete` attempt | +| `./exports/-export-.json` | The export itself, unless `--output` says otherwise | +| `./.env.clerk-migrate` | Migration credentials, written by `settings set` and gitignored | + +The transformer and file of the last run are **not** written here. They go to +the `migrations` section of the CLI's own config file, keyed by project the +same way a linked profile is. That is what `migrate delete` reads to know which +migration to undo, so it is load-bearing rather than a convenience — and it has +no business being written into the repository being migrated. + +Log writes are synchronous appends, so a run interrupted with Ctrl-C still +leaves a complete record of everything already processed. Use the last +successful `userId` in that log with `--resume-after` to continue. + +### Why the logs are NDJSON + +One JSON object per line, rather than one JSON array per file. A migration is a +long append-only stream, and that format is the one that survives it: + +- **Appendable.** Each entry is written as it happens, without rewriting the + file. A JSON array would have to be re-serialized on every user. +- **Crash-safe.** Kill the process at any point and every line already written + is still valid. A truncated array is not parseable at all. +- **Streamable.** `tail -f` shows a long import progressing live, and analysis + reads line by line instead of loading a million-user log into memory. + +Which is also why it greps usefully without any tooling: + +```sh +grep '"status":"success"' logs/import-2026-01-01T12-00-00.log | wc -l +grep '"userId":"user_123"' logs/import-2026-01-01T12-00-00.log +``` + +The trade-off is that spreadsheets, databases and most JSON tooling want an +array. That is what `clerk migrate logs convert` is for — convert when you need +to open a log in Excel or hand it to someone who should not have to know what +NDJSON is. The original `.log` stays put. + +## API Endpoints + +| Method | Path | Used by | +| -------- | -------------------------- | ------------------------------------------------------------------------------------ | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | +| `GET` | `/v1/users?external_id=…` | `migrate delete` — finds this migration's users, 100 IDs a call | +| `GET` | `/v1/users?limit=&offset=` | `migrate export clerk` — pages the whole instance, 500 at a time | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | +| `DELETE` | `/v1/users/{user_id}` | `migrate delete` — removes one user | +| `GET` | `/v1/domains` | Readiness report and `--skip-unsupported-providers` — resolves the Frontend API host | + +The readiness report also reads the instance's Frontend API +`GET /v1/environment` (bootstrapping a dev browser first on development +instances), and its settings-change offer writes through the Platform API: + +| Method | Path | Used by | +| ------- | ----------------------------------------------------------------- | ------------------------------------------------------ | +| `PATCH` | `/v1/platform/applications/{appID}/instances/{instanceID}/config` | Applying the settings changes selected from the report | + +Three exports talk to their own platform rather than to Clerk: + +| Method | Path | Used by | +| ------ | ---------------------------------------------- | -------------------------------------------------------- | +| `POST` | `https:///oauth/token` | `export auth0` — Management API access token | +| `GET` | `https:///api/v2/users` | `export auth0` — 100 per page, 1000 users maximum | +| `POST` | `https://oauth2.googleapis.com/token` | `export firebase` — RS256 assertion → access token | +| `GET` | `…/v1/projects/{project_id}/accounts:batchGet` | `export firebase` — pages users, 1000 at a time | +| `GET` | `…/admin/v2/projects/{project_id}/config` | `export firebase` — reads the scrypt hash parameters | +| `GET` | `…/user_management/users` | `export workos` — 100 per page, cursor-paginated | +| `GET` | `…/user_management/users/{id}/identities` | `export workos` — `--with-identities` only, one per user | + +The two Identity Toolkit paths are on `identitytoolkit.googleapis.com`, or on +`FIREBASE_AUTH_EMULATOR_HOST` when that is set. The two WorkOS paths are on +`api.workos.com`. + +The three database exports (`supabase`, `authjs`, `betterauth`) make no HTTP +calls at all — they connect over `--db-url`. + +The readiness report and `--skip-unsupported-providers` additionally read the +instance's Frontend API `GET /v1/environment` for its attributes and enabled +social providers. + +## Notes + +- `userId` in the source file becomes the Clerk user's `external_id`. That is + what makes a migration re-runnable and reversible. +- CSV input is coerced before validation: `a@x.dev,b@x.dev` and `["a@x.dev"]` + both become arrays, `"true"`/`1` become booleans, and JSON metadata columns + are parsed. An empty column is dropped rather than sent as null. +- A user must end up with at least one identifier (email, phone or username). + Users that do not are logged as validation failures and skipped. diff --git a/packages/cli-core/src/commands/migrate/delete.test.ts b/packages/cli-core/src/commands/migrate/delete.test.ts new file mode 100644 index 000000000..3ae3131ac --- /dev/null +++ b/packages/cli-core/src/commands/migrate/delete.test.ts @@ -0,0 +1,428 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../lib/config.ts"; +import { CliError } from "../../lib/errors.ts"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { + batch, + deleteMigration, + deleteMigratedUsers, + findMigratedUsers, + readMigratedExternalIds, + resolveMigrationToUndo, +} from "./delete.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import { getLogDir } from "./lib/logger.ts"; +import { saveSettings } from "./lib/settings.ts"; + +const captured = useCaptureLog(); + +const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurrencyLimit: 8 }; +const DATE_TIME = "2026-01-01T00:00:00"; + +let workDir: string; +let configDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { method: string; url: string }[]; + +const EXPORT = [ + { id: "legacy_a", primary_email_address: "a@x.dev" }, + { id: "legacy_b", primary_email_address: "b@x.dev" }, +]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-delete-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-delete-config-")); + _setConfigDir(configDir); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + process.exitCode = 0; +}); + +/** + * Stubs BAPI: `GET /v1/users` answers with whichever of `present` the request + * asked for, mirroring how Clerk ignores external IDs it does not find. + */ +function stubBapi(present: Record, onDelete?: (id: string) => Response) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ method: init?.method ?? "GET", url }); + + if (url.includes("/v1/users?")) { + const asked = new URL(url).searchParams.getAll("external_id"); + return Response.json( + asked + .filter((externalId) => externalId in present) + .map((externalId) => ({ id: present[externalId], external_id: externalId })), + ); + } + + const match = /\/v1\/users\/([^/?]+)/.exec(url); + if (init?.method === "DELETE" && match) { + return onDelete ? onDelete(match[1] as string) : Response.json({ deleted: true }); + } + return Response.json({}); + }) as unknown as typeof fetch; +} + +const logEntries = () => + fs + .readdirSync(getLogDir()) + .flatMap((name) => fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n")) + .map((line) => JSON.parse(line) as Record); + +const deleteCalls = () => requests.filter((r) => r.method === "DELETE").map((r) => r.url); + +describe("resolveMigrationToUndo", () => { + test("reads the file and transformer from the saved migration", async () => { + await saveSettings({ transformer: "clerk", file: "export.json" }); + expect(await resolveMigrationToUndo()).toEqual({ file: "export.json", key: "clerk" }); + }); + + // Deleting nothing silently would look like a successful undo. + test("explains when there is no saved migration at all", async () => { + await expect(resolveMigrationToUndo()).rejects.toThrow(/no record of a previous/); + }); + + test.each([ + ["no file", { transformer: "clerk" }], + ["no transformer", { file: "export.json" }], + ])("explains when the saved migration has %s", async (_label, settings) => { + await saveSettings(settings); + await expect(resolveMigrationToUndo()).rejects.toThrow(CliError); + }); + + test("explains when the migration file has since been removed", async () => { + await saveSettings({ transformer: "clerk", file: "gone.json" }); + await expect(resolveMigrationToUndo()).rejects.toThrow(/no longer there/); + }); +}); + +describe("readMigratedExternalIds", () => { + test("returns the source IDs the import stamped as external_id", async () => { + expect(await readMigratedExternalIds("export.json", "clerk")).toEqual(["legacy_a", "legacy_b"]); + }); + + test("uses each transformer's own id field", async () => { + fs.writeFileSync( + path.join(workDir, "auth0.json"), + JSON.stringify([{ user_id: "auth0|1", email: "a@x.dev" }]), + ); + expect(await readMigratedExternalIds("auth0.json", "auth0")).toEqual(["auth0|1"]); + }); + + // Firebase's postTransform demands the project's hash parameters; deleting + // must not require them, so only the field mapping runs. + test("reads a firebase export without needing its password hash parameters", async () => { + fs.writeFileSync( + path.join(workDir, "firebase.json"), + JSON.stringify({ + users: [{ localId: "fb1", email: "a@x.dev", passwordHash: "H", salt: "S" }], + }), + ); + expect(await readMigratedExternalIds("firebase.json", "firebase")).toEqual(["fb1"]); + }); + + test("dedupes repeated IDs", async () => { + fs.writeFileSync( + path.join(workDir, "dupes.json"), + JSON.stringify([{ id: "legacy_a" }, { id: "legacy_a" }]), + ); + expect(await readMigratedExternalIds("dupes.json", "clerk")).toEqual(["legacy_a"]); + }); + + test("skips rows with no ID rather than matching on an empty string", async () => { + fs.writeFileSync( + path.join(workDir, "partial.json"), + JSON.stringify([{ id: "legacy_a" }, { primary_email_address: "b@x.dev" }, { id: "" }]), + ); + expect(await readMigratedExternalIds("partial.json", "clerk")).toEqual(["legacy_a"]); + }); +}); + +describe("batch", () => { + test.each([ + [0, 0], + [1, 1], + [100, 1], + [101, 2], + [250, 3], + ])("%i ids become %i request(s)", (count, expected) => { + const ids = Array.from({ length: count }, (_, i) => `u${i}`); + expect(batch(ids, 100)).toHaveLength(expected); + }); + + test("keeps every item, in order", () => { + expect(batch([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]); + }); +}); + +describe("findMigratedUsers", () => { + test("queries by external_id instead of listing the instance", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + const found = await findMigratedUsers({ + externalIds: ["legacy_a", "legacy_b"], + secretKey: "sk_test_x", + }); + + expect(found).toEqual([ + { id: "user_1", externalId: "legacy_a" }, + { id: "user_2", externalId: "legacy_b" }, + ]); + expect(requests).toHaveLength(1); + expect(requests[0]?.url).toContain("external_id=legacy_a"); + }); + + test("omits IDs the instance does not have", async () => { + stubBapi({ legacy_a: "user_1" }); + + const found = await findMigratedUsers({ + externalIds: ["legacy_a", "legacy_b"], + secretKey: "sk_test_x", + }); + + expect(found).toEqual([{ id: "user_1", externalId: "legacy_a" }]); + }); + + test("pages in batches of 100, the BAPI limit", async () => { + const ids = Array.from({ length: 150 }, (_, i) => `legacy_${i}`); + stubBapi(Object.fromEntries(ids.map((id, i) => [id, `user_${i}`]))); + + const found = await findMigratedUsers({ externalIds: ids, secretKey: "sk_test_x" }); + + expect(found).toHaveLength(150); + expect(requests).toHaveLength(2); + }); + + test("asks for a page large enough to hold the whole batch", async () => { + stubBapi({ legacy_a: "user_1" }); + await findMigratedUsers({ externalIds: ["legacy_a"], secretKey: "sk_test_x" }); + expect(requests[0]?.url).toContain("limit=100"); + }); + + // The guard that keeps this command from touching anything it did not create. + test("ignores a user whose external_id was not asked for", async () => { + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push({ method: "GET", url: input.toString() }); + return Response.json([ + { id: "user_1", external_id: "legacy_a" }, + { id: "user_999", external_id: "somebody_else" }, + { id: "user_888" }, + ]); + }) as unknown as typeof fetch; + + const found = await findMigratedUsers({ externalIds: ["legacy_a"], secretKey: "sk_test_x" }); + + expect(found).toEqual([{ id: "user_1", externalId: "legacy_a" }]); + }); +}); + +describe("deleteMigratedUsers", () => { + const users = [ + { id: "user_1", externalId: "legacy_a" }, + { id: "user_2", externalId: "legacy_b" }, + ]; + + test("deletes each user and logs the outcome", async () => { + stubBapi({}); + + const summary = await deleteMigratedUsers({ + users, + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ deleted: 2, failed: 0 }); + expect(deleteCalls()).toHaveLength(2); + expect(logEntries().filter((e) => e.status === "success")).toHaveLength(2); + }); + + test("records the source ID alongside the Clerk ID in the log", async () => { + stubBapi({}); + + await deleteMigratedUsers({ + users: [users[0] as (typeof users)[0]], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(logEntries()[0]).toMatchObject({ + userId: "legacy_a", + clerkUserId: "user_1", + status: "success", + }); + }); + + // A half-undone migration with no record of which half is worse than a + // reported failure. + test("keeps going after one user fails", async () => { + stubBapi({}, (id) => + id === "user_1" + ? new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { + status: 422, + }) + : Response.json({ deleted: true }), + ); + + const summary = await deleteMigratedUsers({ + users, + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ deleted: 1, failed: 1 }); + expect(deleteCalls()).toHaveLength(2); + expect(logEntries().some((e) => e.status === "error" && e.code === "422")).toBe(true); + }); + + test("retries a 429 and logs the attempt", async () => { + const attempts = new Map(); + stubBapi({}, (id) => { + const attempt = (attempts.get(id) ?? 0) + 1; + attempts.set(id, attempt); + return attempt === 1 + ? new Response(JSON.stringify({ errors: [{ code: "e", message: "slow down" }] }), { + status: 429, + headers: { "retry-after": "1" }, + }) + : Response.json({ deleted: true }); + }); + + const summary = await deleteMigratedUsers({ + users: [users[0] as (typeof users)[0]], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ deleted: 1, failed: 0 }); + expect(deleteCalls()).toHaveLength(2); + expect(logEntries().some((e) => e.status === "429_retry")).toBe(true); + }); + + test("groups identical failures in the breakdown", async () => { + stubBapi( + {}, + () => + new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { + status: 422, + }), + ); + + const summary = await deleteMigratedUsers({ + users, + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect([...summary.errorBreakdown.values()]).toEqual([2]); + }); +}); + +describe("deleteMigration", () => { + const baseOptions = { yes: true, secretKey: "sk_test_x" }; + + beforeEach(async () => { + await saveSettings({ transformer: "clerk", file: "export.json" }); + }); + + test("deletes the users the last run created", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + await deleteMigration(baseOptions); + + expect(deleteCalls()).toEqual([ + expect.stringContaining("/v1/users/user_1"), + expect.stringContaining("/v1/users/user_2"), + ]); + expect(captured.err).toContain("Deleted:"); + }); + + test("writes a timestamped NDJSON deletion log", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + await deleteMigration(baseOptions); + + const logs = fs.readdirSync(getLogDir()); + expect(logs).toHaveLength(1); + expect(logs[0]).toMatch(/^delete-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + }); + + test("leaves users the migration did not create alone", async () => { + stubBapi({ legacy_a: "user_1" }); + + await deleteMigration(baseOptions); + + expect(deleteCalls()).toEqual([expect.stringContaining("/v1/users/user_1")]); + expect(captured.err).toContain("1 of the file's users is not in this instance"); + }); + + test("does nothing when none of the migration's users are present", async () => { + stubBapi({}); + + await deleteMigration(baseOptions); + + expect(deleteCalls()).toHaveLength(0); + expect(captured.err).toContain("Nothing to delete"); + }); + + // Tests run non-TTY, which is the same signal an agent gives. + test("refuses without -y when it cannot prompt, and says how many are at stake", async () => { + stubBapi({ legacy_a: "user_1", legacy_b: "user_2" }); + + await expect(deleteMigration({ secretKey: "sk_test_x" })).rejects.toThrow( + /permanently deletes 2 users and cannot prompt here/, + ); + expect(deleteCalls()).toHaveLength(0); + }); + + test("fails before any API call when there is no saved migration", async () => { + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + stubBapi({ legacy_a: "user_1" }); + + await expect(deleteMigration(baseOptions)).rejects.toThrow(CliError); + expect(requests).toHaveLength(0); + }); + + test("exits non-zero when a deletion failed", async () => { + stubBapi( + { legacy_a: "user_1" }, + () => + new Response(JSON.stringify({ errors: [{ code: "e", message: "locked" }] }), { + status: 422, + }), + ); + + await deleteMigration(baseOptions); + + expect(process.exitCode).toBe(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/delete.ts b/packages/cli-core/src/commands/migrate/delete.ts new file mode 100644 index 000000000..5b5a78318 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/delete.ts @@ -0,0 +1,330 @@ +/** + * `clerk migrate delete` — undo a migration. + * + * Ported from the standalone migration-tool's `src/delete/index.ts`, with two + * substantive changes: + * + * - **Users are looked up by `external_id`, not by downloading the instance.** + * The original paged through every user in the instance 500 at a time and + * intersected client-side, which on a large instance means fetching hundreds + * of thousands of users to delete a few hundred. `GET /v1/users` filters on + * up to 100 `external_id`s per call and ignores IDs it does not find, so the + * work is proportional to the migration rather than to the instance. + * - **IDs come from the existing transform pipeline.** The original + * re-implemented per-format ID extraction with its own Firebase CSV header + * list and a chain of `userId`/`user_id`/`localId`/`id` fallbacks. The + * transformer already declares which source field becomes `userId`. + * + * This is the one command in the `migrate` tree that destroys data in Clerk, so + * it stays flat and prominent rather than buried under a noun group, and it + * confirms before acting. + */ + +import { bapiRequest } from "../../lib/bapi.ts"; +import { bold, dim, green, red } from "../../lib/color.ts"; +import { + BapiError, + CliError, + ERROR_CODE, + throwUsageError, + throwUserAbort, +} from "../../lib/errors.ts"; +import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; +import { log } from "../../lib/log.ts"; +import { NEXT_STEPS } from "../../lib/next-steps.ts"; +import { confirm } from "../../lib/prompts.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../mode.ts"; +import { normalizeErrorMessage } from "./import-users.ts"; +import { resolveLimits, type ResolvedLimits } from "./lib/instance.ts"; +import { deleteErrorLogger, deleteLogger, startLogging, getLogFilePath } from "./lib/logger.ts"; +import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; +import { createApiScheduler } from "./lib/scheduler.ts"; +import { loadSettings } from "./lib/settings.ts"; +import { fileExists, readRawUsers, transformKeys } from "./lib/transform.ts"; +import { getTransformer } from "./transformers/registry.ts"; + +/** BAPI accepts at most 100 `external_id` values per `GET /v1/users` call. */ +const EXTERNAL_ID_BATCH = 100; + +export type MigrateDeleteOptions = { + yes?: boolean; + secretKey?: string; + app?: string; + instance?: string; +}; + +export type MigratedUser = { + /** The Clerk user ID to delete. */ + id: string; + /** The source platform's ID, stamped on the user as `external_id`. */ + externalId: string; +}; + +/** + * Resolves which migration is being undone. + * + * The saved migration record is the only account of that — this command has no + * independent way to know what a previous run created, which is why it is + * coupled to `run`. + */ +export async function resolveMigrationToUndo(): Promise<{ file: string; key: string }> { + const settings = await loadSettings(); + + if (!settings.file || !settings.transformer) { + throw new CliError( + "No migration to undo: this project has no record of a previous `clerk migrate import`.\n" + + "Run `clerk migrate delete` from the project you migrated from.", + { code: ERROR_CODE.FILE_NOT_FOUND }, + ); + } + + if (!fileExists(settings.file)) { + throw new CliError( + `The migration file ${settings.file} is no longer there, so the users it created cannot be identified.`, + { code: ERROR_CODE.FILE_NOT_FOUND }, + ); + } + + return { file: settings.file, key: settings.transformer }; +} + +/** + * The source IDs a migration stamped onto Clerk users as `external_id`. + * + * Runs the transformer's field mapping but not its `postTransform`: only the + * ID matters here, and Firebase's post-transform would demand the project's + * password hash parameters to rebuild digests nobody is importing. + */ +export async function readMigratedExternalIds(file: string, key: string): Promise { + const transformer = getTransformer(key); + const rows = await readRawUsers(file, key); + + const ids = new Set(); + for (const row of rows) { + const userId = transformKeys(row, transformer).userId; + if (typeof userId === "string" && userId.length > 0) ids.add(userId); + } + return [...ids]; +} + +/** Splits `items` into chunks of at most `size`. */ +export function batch(items: T[], size: number): T[][] { + const batches: T[][] = []; + for (let i = 0; i < items.length; i += size) batches.push(items.slice(i, i + size)); + return batches; +} + +/** + * Finds the Clerk users a migration created, by `external_id`. + * + * IDs with no matching user are simply absent from the result — a partial + * migration, or one already partly undone, is the normal case. + */ +export async function findMigratedUsers(options: { + externalIds: string[]; + secretKey: string; + spinner?: SpinnerControls; +}): Promise { + const found: MigratedUser[] = []; + const batches = batch(options.externalIds, EXTERNAL_ID_BATCH); + + for (const [index, ids] of batches.entries()) { + options.spinner?.update(`Finding migrated users: batch ${index + 1}/${batches.length}...`); + + const params = new URLSearchParams(); + params.set("limit", String(EXTERNAL_ID_BATCH)); + for (const id of ids) params.append("external_id", id); + + const response = await retryOn429(async () => + bapiRequest({ + method: "GET", + path: `/v1/users?${params.toString()}`, + secretKey: options.secretKey, + }), + ); + + const users = (response.body ?? []) as { id?: string; external_id?: string }[]; + for (const user of Array.isArray(users) ? users : []) { + // Never delete on a partial match: only a user Clerk itself reports as + // carrying one of this migration's external IDs is in scope. + if (user.id && user.external_id && ids.includes(user.external_id)) { + found.push({ id: user.id, externalId: user.external_id }); + } + } + } + + return found; +} + +export type DeleteSummary = { + deleted: number; + failed: number; + errorBreakdown: Map; +}; + +/** Deletes each user, rate-limited and 429-retried exactly as the import is. */ +export async function deleteMigratedUsers(options: { + users: MigratedUser[]; + secretKey: string; + limits: ResolvedLimits; + dateTime: string; + spinner?: SpinnerControls; +}): Promise { + const { users, secretKey, limits, dateTime, spinner } = options; + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + const errorBreakdown = new Map(); + + let processed = 0; + let deleted = 0; + let failed = 0; + + const progress = () => + spinner?.update( + `Deleting users: [${processed}/${users.length}] (${deleted} deleted, ${failed} failed)...`, + ); + + // A failure on one user must not abort the rest: a half-undone migration + // with no record of which half is far worse than a reported failure. + const recordFailure = (user: MigratedUser, message: string, code: string) => { + failed++; + processed++; + const normalized = normalizeErrorMessage(message); + errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); + deleteLogger( + { userId: user.externalId, clerkUserId: user.id, status: "error", error: message, code }, + dateTime, + ); + progress(); + }; + + const deleteOne = async (user: MigratedUser): Promise => { + try { + await retryOn429( + async () => + schedule(async () => + bapiRequest({ method: "DELETE", path: `/v1/users/${user.id}`, secretKey }), + ), + { + onRetry: ({ message }) => + deleteErrorLogger( + { + userId: user.externalId, + status: "429_retry", + errors: [{ code: "rate_limit_retry", message, longMessage: message }], + }, + dateTime, + ), + }, + ); + + deleted++; + processed++; + deleteLogger({ userId: user.externalId, clerkUserId: user.id, status: "success" }, dateTime); + progress(); + } catch (error) { + if (error instanceof RateLimitExceededError) { + recordFailure(user, error.message, "429"); + return; + } + const apiError = error as BapiError; + const message = apiError.longMessage ?? apiError.message ?? "Unknown error"; + recordFailure(user, message, String(apiError.status ?? "unknown")); + } + }; + + progress(); + await Promise.all(users.map(deleteOne)); + + return { deleted, failed, errorBreakdown }; +} + +function formatSummary(summary: DeleteSummary, logFile: string): string { + const lines = [ + `${bold("Deleted:")} ${green(String(summary.deleted))}`, + `${bold("Failed:")} ${red(String(summary.failed))}`, + ]; + + if (summary.errorBreakdown.size > 0) { + lines.push("", bold("Error breakdown:")); + for (const [error, count] of summary.errorBreakdown) { + lines.push(` ${count} user${count === 1 ? "" : "s"}: ${error}`); + } + } + lines.push("", dim(`Log: ${logFile}`)); + + return lines.join("\n"); +} + +export async function deleteMigration(options: MigrateDeleteOptions): Promise { + const { file, key } = await resolveMigrationToUndo(); + + await withGutter("Undoing a migration", async ({ setNextSteps }) => { + const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); + const limits = resolveLimits(secretKey); + const dateTime = await startLogging(); + const logFile = getLogFilePath("delete", dateTime); + + const externalIds = await readMigratedExternalIds(file, key); + if (externalIds.length === 0) { + log.warn(`No user IDs found in ${file}; nothing to undo.`); + return; + } + + const users = await withSpinner("Finding migrated users...", async (spinner) => + findMigratedUsers({ externalIds, secretKey, spinner }), + ); + + if (users.length === 0) { + log.info( + `None of the ${externalIds.length} user${externalIds.length === 1 ? "" : "s"} in ${file} are in ${target ?? "this instance"}. Nothing to delete.`, + ); + return; + } + + log.warn( + `About to delete ${users.length} user${users.length === 1 ? "" : "s"} from ` + + `${target ?? "the resolved instance"}, matched to ${file} by external ID.`, + ); + if (users.length < externalIds.length) { + log.info( + dim( + `${externalIds.length - users.length} of the file's users ${externalIds.length - users.length === 1 ? "is" : "are"} not in this instance and will be left alone.`, + ), + ); + } + + if (!options.yes) { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate delete\` permanently deletes ${users.length} user${users.length === 1 ? "" : "s"} and cannot prompt here. Pass -y to confirm.`, + undefined, + undefined, + [ + { + command: "clerk migrate delete -y", + description: "Delete the migrated users without prompting", + }, + ], + ); + } + + const proceed = await confirm({ + message: `Permanently delete ${users.length} user${users.length === 1 ? "" : "s"}?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + const summary = await withSpinner(`Deleting users: [0/${users.length}]...`, async (spinner) => + deleteMigratedUsers({ users, secretKey, limits, dateTime, spinner }), + ); + + log.info(formatSummary(summary, logFile)); + + setNextSteps(NEXT_STEPS.MIGRATE_DELETE); + + if (summary.failed > 0) process.exitCode = 1; + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts new file mode 100644 index 000000000..14eaf9585 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -0,0 +1,337 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildAuth0Export, + exportAuth0, + fetchAllAuth0Users, + fetchAuth0Token, + mapAuth0UserToExport, + normalizeAuth0Domain, + resolveAuth0Credentials, +} from "./auth0.ts"; + +/** A cwd with no `.env` files, so these tests exercise only the injected env. */ +const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); + +const captured = useCaptureLog(); +useMigrateLogDir(); + +const CREDENTIALS = { domain: "t.auth0.com", clientId: "cid", clientSecret: "csec" }; + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { url: string; body: unknown }[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expauth0-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +const auth0User = (i: number, overrides: Record = {}) => ({ + user_id: `auth0|a${i}`, + email: `a${i}@x.dev`, + email_verified: true, + given_name: `Given${i}`, + family_name: `Family${i}`, + ...overrides, +}); + +/** Stubs the token exchange plus one page of users per entry in `pages`. */ +function stubAuth0(pages: Record[][], token: Response | null = null) { + let page = 0; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ url, body: init?.body ? JSON.parse(init.body as string) : null }); + + if (url.includes("/oauth/token")) { + return token ?? Response.json({ access_token: "tok" }); + } + return Response.json({ + users: pages[page++] ?? [], + total: pages.reduce((sum, p) => sum + p.length, 0), + }); + }) as unknown as typeof fetch; +} + +describe("normalizeAuth0Domain", () => { + test.each([ + ["t.auth0.com", "t.auth0.com"], + ["https://t.auth0.com", "t.auth0.com"], + ["http://t.auth0.com/", "t.auth0.com"], + [" t.auth0.com ", "t.auth0.com"], + ])("%s -> %s", (input, expected) => { + expect(normalizeAuth0Domain(input)).toBe(expected); + }); +}); + +describe("resolveAuth0Credentials", () => { + test("prefers flags", async () => { + const resolved = await resolveAuth0Credentials( + { domain: "flag.auth0.com", clientId: "f", clientSecret: "s" }, + NO_ENV_FILES, + { AUTH0_DOMAIN: "env.auth0.com" }, + ); + expect(resolved.domain).toBe("flag.auth0.com"); + }); + + test("falls back to the environment", async () => { + const resolved = await resolveAuth0Credentials({}, NO_ENV_FILES, { + AUTH0_DOMAIN: "env.auth0.com", + AUTH0_CLIENT_ID: "e", + AUTH0_CLIENT_SECRET: "s", + }); + expect(resolved).toEqual({ domain: "env.auth0.com", clientId: "e", clientSecret: "s" }); + }); + + test("normalizes a domain that came with a scheme", async () => { + const resolved = await resolveAuth0Credentials( + { domain: "https://t.auth0.com/", clientId: "c", clientSecret: "s" }, + NO_ENV_FILES, + {}, + ); + expect(resolved.domain).toBe("t.auth0.com"); + }); + + // Tests run non-TTY, the same signal an agent gives. + test("names every missing credential at once rather than one at a time", async () => { + await expect(resolveAuth0Credentials({}, NO_ENV_FILES, {})).rejects.toThrow( + /--domain \(or AUTH0_DOMAIN\), --client-id \(or AUTH0_CLIENT_ID\), --client-secret \(or AUTH0_CLIENT_SECRET\)/, + ); + }); + + test("names only what is actually missing", async () => { + await expect( + resolveAuth0Credentials({ domain: "t.auth0.com", clientId: "c" }, NO_ENV_FILES, {}), + ).rejects.toThrow(/Missing: --client-secret \(or AUTH0_CLIENT_SECRET\)\./); + }); +}); + +describe("fetchAuth0Token", () => { + test("exchanges client credentials for the Management API audience", async () => { + stubAuth0([[]]); + + expect(await fetchAuth0Token(CREDENTIALS)).toBe("tok"); + expect(requests[0]?.url).toBe("https://t.auth0.com/oauth/token"); + expect(requests[0]?.body).toEqual({ + grant_type: "client_credentials", + client_id: "cid", + client_secret: "csec", + audience: "https://t.auth0.com/api/v2/", + }); + }); + + test("explains a rejection instead of surfacing a raw status", async () => { + stubAuth0( + [[]], + new Response(JSON.stringify({ error_description: "Wrong client secret" }), { status: 401 }), + ); + + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow( + /Auth0 rejected the credentials \(401\): Wrong client secret/, + ); + }); + + test("mentions the read:users scope, the usual cause", async () => { + stubAuth0([[]], new Response("{}", { status: 403 })); + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(/read:users/); + }); + + test("fails when a 200 carries no token", async () => { + stubAuth0([[]], Response.json({})); + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(CliError); + }); +}); + +describe("fetchAllAuth0Users", () => { + test("pages until a short page arrives", async () => { + stubAuth0([ + Array.from({ length: 100 }, (_, i) => auth0User(i)), + Array.from({ length: 4 }, (_, i) => auth0User(100 + i)), + ]); + + const all = await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + + expect(all).toHaveLength(104); + expect(requests[0]?.url).toContain("page=0"); + expect(requests[1]?.url).toContain("page=1"); + expect(requests).toHaveLength(2); + }); + + test("asks for totals and the documented page size", async () => { + stubAuth0([[]]); + await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + expect(requests[0]?.url).toContain("per_page=100"); + expect(requests[0]?.url).toContain("include_totals=true"); + }); + + // Auth0 caps offset pagination at 1000. Returning the first thousand quietly + // would read as "that is everyone". + test("stops at Auth0's 1000-record ceiling and says so", async () => { + stubAuth0( + Array.from({ length: 12 }, () => Array.from({ length: 100 }, (_, i) => auth0User(i))), + ); + + const all = await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + + expect(all).toHaveLength(1000); + expect(captured.err).toContain("only pages through the first 1000 users"); + expect(captured.err).toContain("bulk user export job"); + }); + + test("raises a clear error on a failed page request", async () => { + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + + await expect(fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" })).rejects.toThrow( + /Auth0 returned 500 listing users/, + ); + }); +}); + +describe("mapAuth0UserToExport", () => { + test("keeps the fields the auth0 transformer maps from", () => { + expect( + mapAuth0UserToExport(auth0User(0, { phone_number: "+1555", created_at: "2025-01-01" })), + ).toEqual({ + user_id: "auth0|a0", + email: "a0@x.dev", + given_name: "Given0", + family_name: "Family0", + phone_number: "+1555", + created_at: "2025-01-01", + email_verified: true, + }); + }); + + // Dropping a false flag would import an unconfirmed address as verified. + test.each([ + ["email_verified", false], + ["phone_verified", false], + ])("keeps %s when it is %p", (field, value) => { + const mapped = mapAuth0UserToExport(auth0User(0, { [field]: value })); + expect(mapped[field]).toBe(value); + }); + + test("drops tenant internals the import has no use for", () => { + const mapped = mapAuth0UserToExport( + auth0User(0, { + identities: [{ provider: "auth0" }], + logins_count: 42, + last_login: "2026-01-01", + multifactor: ["guardian"], + }), + ); + for (const noise of ["identities", "logins_count", "last_login", "multifactor"]) { + expect(noise in mapped).toBe(false); + } + }); + + test("omits empty metadata", () => { + const mapped = mapAuth0UserToExport( + auth0User(0, { user_metadata: {}, app_metadata: { plan: "pro" } }), + ); + expect("user_metadata" in mapped).toBe(false); + expect(mapped.app_metadata).toEqual({ plan: "pro" }); + }); +}); + +describe("buildAuth0Export", () => { + test("counts coverage and logs each user", () => { + const { users, coverage } = buildAuth0Export( + [auth0User(0), auth0User(1, { given_name: undefined })], + "2026-01-01T00:00:00", + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + + const logged = fs.readdirSync(getLogDir()); + expect(logged[0]).toMatch(/^export-/); + }); +}); + +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + +describe("exportAuth0", () => { + test("writes the default path and reports coverage", async () => { + stubAuth0([[auth0User(0)], []]); + + await exportAuth0({ ...CREDENTIALS }); + + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^auth0-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; + expect(written[0]?.user_id).toBe("auth0|a0"); + expect(captured.err).toContain("Field coverage"); + }); + + test("names the command that consumes the file", async () => { + stubAuth0([[auth0User(0)], []]); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + // --output answers the destination prompt, which human mode would + // otherwise stop on. + await exportAuth0({ ...CREDENTIALS, output: "exports/mine.json" }); + } finally { + setMode(originalMode); + } + expect(captured.err).toContain("migrate import --transformer auth0 --file exports/mine.json"); + }); + + test("--output controls the destination", async () => { + stubAuth0([[auth0User(0)], []]); + + await exportAuth0({ ...CREDENTIALS, output: "tenant.json" }); + + expect(fs.existsSync(path.join(workDir, "tenant.json"))).toBe(true); + }); + + // Auth0 only releases hashes through a support request; finding that out + // after the import means nobody can sign in. + test("says plainly that password hashes are not in the file", async () => { + stubAuth0([[auth0User(0)], []]); + await exportAuth0({ ...CREDENTIALS }); + expect(captured.err).toContain("does not return password hashes"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts new file mode 100644 index 000000000..9ae0f4a32 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -0,0 +1,374 @@ +/** + * `clerk migrate export auth0` — pull users out of an Auth0 tenant. + * + * Ported from the standalone migration-tool's `src/export/auth0.ts`, but + * **without the `auth0` SDK**. The SDK is 28 MB across five transitive + * dependencies — including a bundled legacy copy of itself — to make two REST + * calls, and it does its own HTTP, so nothing it sends would appear under + * `--verbose`. `.claude/rules/debug-logging.md` requires library HTTP to go + * through `loggedFetch`; two direct calls satisfy that and ship nothing extra + * inside the compiled binary. + * + * **Passwords do not come out of the Management API.** Auth0 exports password + * hashes only via a support request. The coverage report says so rather than + * leaving it to be discovered when nobody can sign in. + */ + +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt, text } from "../../../lib/prompts.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; + +const PAGE_SIZE = 100; + +/** + * Auth0 caps offset pagination on `GET /api/v2/users` at 1000 records. + * Past that the tenant needs a bulk export job, so the run says so instead of + * quietly returning the first thousand as though that were everyone. + */ +const AUTH0_PAGINATION_CEILING = 1000; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/auth0"; + +export type ExportAuth0Options = { + domain?: string; + clientId?: string; + clientSecret?: string; + output?: string; +}; + +export type Auth0Credentials = { + domain: string; + clientId: string; + clientSecret: string; +}; + +/** Strips a scheme and trailing slash, so both forms of `--domain` work. */ +export function normalizeAuth0Domain(domain: string): string { + return domain + .trim() + .replace(/^https?:\/\//, "") + .replace(/\/+$/, ""); +} + +/** + * Resolves the tenant credentials: flags, then environment, then a prompt. + * + * @throws CliError in agent mode when anything is still missing, naming each + * absent flag rather than failing on the first one. + */ +export async function resolveAuth0Credentials( + options: ExportAuth0Options, + cwd: string = process.cwd(), + env: Record = process.env, +): Promise { + const fromEnv = async (name: string): Promise => + (await findMigrateEnvValue([name], cwd, env))?.value; + + const resolved = { + domain: options.domain ?? (await fromEnv("AUTH0_DOMAIN")), + clientId: options.clientId ?? (await fromEnv("AUTH0_CLIENT_ID")), + clientSecret: options.clientSecret ?? (await fromEnv("AUTH0_CLIENT_SECRET")), + }; + + const missing = ( + [ + ["domain", "--domain", "AUTH0_DOMAIN"], + ["clientId", "--client-id", "AUTH0_CLIENT_ID"], + ["clientSecret", "--client-secret", "AUTH0_CLIENT_SECRET"], + ] as const + ).filter(([key]) => !resolved[key]); + + if (missing.length === 0) { + return { + domain: normalizeAuth0Domain(resolved.domain as string), + clientId: resolved.clientId as string, + clientSecret: resolved.clientSecret as string, + }; + } + + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export auth0\` needs credentials for a machine-to-machine application and cannot prompt here.\n` + + `Missing: ${missing.map(([, flag, variable]) => `${flag} (or ${variable})`).join(", ")}.`, + DOCS_URL, + undefined, + [ + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export with explicit credentials", + }, + ], + ); + } + + log.info( + "Auth0 needs a machine-to-machine application with the `read:users` scope. Create one under Applications → APIs → Auth0 Management API → Machine to Machine Applications.", + ); + + return promptAuth0Credentials(resolved); +} + +/** + * Asks for whichever of the three are still missing. + * + * Called with nothing known after Auth0 has rejected a set: its error names no + * field, and the operator may have mistyped any of them — so all three are + * asked again rather than guessing which one to keep. + */ +export async function promptAuth0Credentials( + known: Partial = {}, +): Promise { + const domain = + known.domain ?? + (await text({ + message: "Auth0 tenant domain (e.g. my-tenant.us.auth0.com)", + validate: (value) => (value?.trim() ? undefined : "A domain is required"), + })); + const clientId = + known.clientId ?? + (await text({ + message: "Machine-to-machine client ID", + validate: (value) => (value?.trim() ? undefined : "A client ID is required"), + })); + const clientSecret = + known.clientSecret ?? + (await passwordPrompt({ + message: "Machine-to-machine client secret", + validate: (value) => (value?.trim() ? undefined : "A client secret is required"), + })); + + return { + domain: normalizeAuth0Domain(domain), + clientId: clientId.trim(), + clientSecret: clientSecret.trim(), + }; +} + +/** Exchanges the client credentials for a Management API access token. */ +export async function fetchAuth0Token(credentials: Auth0Credentials): Promise { + const url = new URL(`https://${credentials.domain}/oauth/token`); + + const response = await loggedFetch(url, { + tag: "auth0", + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + grant_type: "client_credentials", + client_id: credentials.clientId, + client_secret: credentials.clientSecret, + audience: `https://${credentials.domain}/api/v2/`, + }), + }); + + const body = (await response.json().catch(() => ({}))) as { + access_token?: string; + error_description?: string; + error?: string; + }; + + if (!response.ok || !body.access_token) { + throw new CliError( + `Auth0 rejected the credentials (${response.status}): ${body.error_description ?? body.error ?? "no access token returned"}\n` + + "Check the domain, client ID and secret, and that the application is authorized for the Management API with the `read:users` scope.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + return body.access_token; +} + +type Auth0User = Record & { user_id?: string }; + +/** Fetches one page of users from the Management API. */ +async function fetchAuth0Page( + credentials: Auth0Credentials, + token: string, + page: number, +): Promise<{ users: Auth0User[]; total: number }> { + const url = new URL(`https://${credentials.domain}/api/v2/users`); + url.searchParams.set("page", String(page)); + url.searchParams.set("per_page", String(PAGE_SIZE)); + url.searchParams.set("include_totals", "true"); + + const response = await loggedFetch(url, { + tag: "auth0", + method: "GET", + headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, + }); + + if (!response.ok) { + const body = await response.text(); + throw new CliError(`Auth0 returned ${response.status} listing users: ${body}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + } + + const body = (await response.json()) as { users?: Auth0User[]; total?: number }; + return { users: body.users ?? [], total: body.total ?? 0 }; +} + +/** + * Pages through the tenant's users. + * + * Stops at Auth0's 1000-record ceiling with a warning naming the bulk export + * job — silently truncating would read as "that is everyone". + */ +export async function fetchAllAuth0Users(options: { + credentials: Auth0Credentials; + token: string; + spinner?: SpinnerControls; +}): Promise { + const all: Auth0User[] = []; + + for (let page = 0; ; page++) { + const { users, total } = await fetchAuth0Page(options.credentials, options.token, page); + all.push(...users); + options.spinner?.update(`Fetching users from Auth0: ${all.length} so far...`); + + if (users.length < PAGE_SIZE) break; + + if (all.length >= AUTH0_PAGINATION_CEILING) { + log.warn( + `Auth0 only pages through the first ${AUTH0_PAGINATION_CEILING} users on this endpoint` + + (total > AUTH0_PAGINATION_CEILING ? `, and this tenant reports ${total}` : "") + + ". Exported what is reachable; use Auth0's bulk user export job for the rest.", + ); + break; + } + } + + return all; +} + +/** + * Keeps the fields the `auth0` transformer maps from. + * + * Deliberately a copy rather than the raw record: an Auth0 user carries + * identities, session counts and tenant internals that would bloat the export + * and mean nothing to the import. + */ +export function mapAuth0UserToExport(user: Auth0User): Record { + const exported: Record = {}; + + for (const field of [ + "user_id", + "email", + "username", + "given_name", + "family_name", + "phone_number", + "created_at", + ] as const) { + if (user[field]) exported[field] = user[field]; + } + + // Verification flags are meaningful when false, so they are copied on + // presence rather than on truthiness. + for (const field of ["email_verified", "phone_verified"] as const) { + if (user[field] !== undefined) exported[field] = user[field]; + } + + for (const field of ["user_metadata", "app_metadata"] as const) { + const value = user[field]; + if (value && typeof value === "object" && Object.keys(value).length > 0) { + exported[field] = value; + } + } + + return exported; +} + +export type Auth0ExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +export function buildAuth0Export(users: Auth0User[], dateTime: string): Auth0ExportResult { + const exported: Record[] = []; + const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0 }; + + for (const user of users) { + const userId = String(user.user_id ?? ""); + try { + const mapped = mapAuth0UserToExport(user); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.username) counts.username++; + if (mapped.given_name) counts.firstName++; + if (mapped.family_name) counts.lastName++; + if (mapped.phone_number) counts.phone++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a phone number", count: counts.phone }, + { label: "have a username", count: counts.username }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + ], + }; +} + +export async function exportAuth0(options: ExportAuth0Options): Promise { + const resolved = await resolveAuth0Credentials(options); + + const destination = await resolveOutputPath("auth0", options.output); + + await withGutter("Exporting users from Auth0", async ({ setNextSteps }) => { + const dateTime = await startLogging(); + + // Only Auth0 can say whether these three go together, and whether the + // application carries the `read:users` scope, so a rejected set is asked + // for again here. + const { value: token, input: credentials } = await withInputRetry( + resolved, + async () => promptAuth0Credentials(), + async (candidate) => { + log.info(`Exporting from ${candidate.domain}.`); + return withSpinner("Authenticating with Auth0...", async () => fetchAuth0Token(candidate)); + }, + ); + + const users = await withSpinner("Fetching users from Auth0...", async (spinner) => + fetchAllAuth0Users({ credentials, token, spinner }), + ); + + const { users: exported, coverage } = buildAuth0Export(users, dateTime); + const outputPath = writeExportOutput(exported, destination); + + setNextSteps( + reportExport({ + platform: "auth0", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "auth0", + }), + ); + + if (exported.length > 0) { + log.warn( + "Auth0's Management API does not return password hashes. Request a password hash export from Auth0 support and add a `passwordHash` field to each user before importing, or migrate without passwords.", + ); + log.info(dim(`See ${DOCS_URL}`)); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts new file mode 100644 index 000000000..c5b5b319d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -0,0 +1,160 @@ +/** + * `clerk migrate export authjs` — read users out of an Auth.js database. + * + * Ported from the standalone migration-tool's `src/export/authjs.ts`, on the + * `Bun.sql`/`bun:sqlite` client. + * + * Auth.js has no export tool and no single schema: the adapter decides the + * table name, and Prisma's `User` differs from Drizzle's `user` only in + * casing — which Postgres and SQLite treat as significant once quoted. The + * export tries the documented casing first and falls back rather than making + * the user find out from a driver error. + */ + +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import { log } from "../../../lib/log.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; +import { + promptDbUrl, + resolveDbUrl, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; + +/** Table names to try, in order. Prisma capitalizes; Drizzle does not. */ +const TABLE_CANDIDATES = ["User", "user", "users"] as const; + +type AuthJsRow = Record & { + id?: unknown; + name?: string | null; + email?: string | null; + email_verified?: unknown; +}; + +export function buildAuthJsQuery(client: DbClient, table: string): string { + const q = (identifier: string) => client.quote(identifier); + return ( + `SELECT ${q("id")}, ${q("name")}, ${q("email")}, ${q("emailVerified")} AS ${q("email_verified")} ` + + `FROM ${q(table)} ORDER BY ${q("id")} ASC` + ); +} + +/** True for an error that means "wrong table name", not "broken connection". */ +function isMissingTable(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /does not exist|no such table|doesn't exist|unknown table/i.test(message); +} + +/** + * Reads the user table, trying each casing until one answers. + * + * @returns The rows and the table they came from, so the run can say which. + */ +export async function fetchAuthJsUsers( + client: DbClient, +): Promise<{ rows: AuthJsRow[]; table: string }> { + let lastError: unknown; + + for (const table of TABLE_CANDIDATES) { + try { + return { rows: await client.query(buildAuthJsQuery(client, table)), table }; + } catch (error) { + if (!isMissingTable(error)) throw error; + lastError = error; + } + } + + throw lastError instanceof Error + ? new Error( + `No Auth.js user table found. Tried ${TABLE_CANDIDATES.join(", ")}. ${lastError.message}`, + ) + : new Error(`No Auth.js user table found. Tried ${TABLE_CANDIDATES.join(", ")}.`); +} + +export function buildAuthJsExport(rows: AuthJsRow[], dateTime: string) { + const users: Record[] = []; + const counts = { email: 0, emailVerified: 0, name: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + const user: Record = { id: userId }; + + if (row.name) { + user.name = row.name; + counts.name++; + } + if (row.email) { + user.email = row.email; + counts.email++; + } + // A nullable timestamp, not a boolean: the transformer reads presence. + if (row.email_verified) { + user.email_verified = + row.email_verified instanceof Date ? row.email_verified.toISOString() : row.email_verified; + counts.emailVerified++; + } + + users.push(user); + exportLogger({ userId, status: "success" }, dateTime); + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.emailVerified }, + { label: "have a name", count: counts.name }, + ], + }; +} + +const AUTHJS_DB = { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Auth.js adapter uses.", +} as const satisfies ResolveConfig; + +export async function exportAuthJs(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, AUTHJS_DB); + + const destination = await resolveOutputPath("authjs", options.output); + + await withGutter("Exporting users from Auth.js", async ({ setNextSteps }) => { + const dateTime = await startLogging(); + + const { + value: { rows, table }, + } = await withInputRetry( + dbUrl, + async () => promptDbUrl(AUTHJS_DB), + async (connectionString) => + withSpinner("Reading the user table...", async () => + withDbClient(connectionString, "authjs", fetchAuthJsUsers), + ), + ); + log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); + + const { users, coverage } = buildAuthJsExport(rows, dateTime); + const outputPath = writeExportOutput(users, destination); + + setNextSteps( + reportExport({ + platform: "authjs", + userCount: users.length, + outputPath, + coverage, + transformerKey: "authjs", + }), + ); + + if (users.length > 0) { + log.warn( + "Auth.js core stores no passwords — its users sign in with OAuth or email links, so they arrive without credentials and will use the same providers in Clerk.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts new file mode 100644 index 000000000..d637845a7 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -0,0 +1,210 @@ +/** + * `clerk migrate export betterauth` — read users out of a Better Auth database. + * + * Ported from the standalone migration-tool's `src/export/betterauth.ts`, on + * the `Bun.sql`/`bun:sqlite` client. + * + * Better Auth's schema depends on which plugins are enabled, so the columns + * are **detected from the schema** rather than asked for: the username plugin + * adds `username`, admin adds `banned`, phone-number adds `phoneNumber`, and + * so on. Selecting a column that is not there fails the whole query, and + * asking the user which plugins they run is a question their database can + * already answer. + * + * Passwords live on the `account` row for the credential provider, not on the + * user, which is why the export joins. + */ + +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; +import { + promptDbUrl, + resolveDbUrl, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; + +/** Columns a Better Auth plugin adds to the user table. */ +export const PLUGIN_COLUMNS = [ + "username", + "displayUsername", + "phoneNumber", + "phoneNumberVerified", + "role", + "banned", + "banReason", + "banExpires", + "twoFactorEnabled", +] as const; + +export type PluginColumn = (typeof PLUGIN_COLUMNS)[number]; + +/** Columns every Better Auth install has. */ +const CORE_COLUMNS = ["id", "email", "emailVerified", "name", "createdAt", "updatedAt"] as const; + +/** + * Asks the schema which plugin columns exist. + * + * SQLite has no `information_schema`, so it goes through `PRAGMA` — and the + * PRAGMA takes the table name inline rather than as a bind parameter. + */ +export async function detectPluginColumns(client: DbClient): Promise> { + const present = new Set(); + + if (client.dbType === "sqlite") { + const rows = await client.query<{ name: string }>(`PRAGMA table_info(${client.quote("user")})`); + const columns = new Set(rows.map((row) => row.name)); + for (const column of PLUGIN_COLUMNS) { + if (columns.has(column)) present.add(column); + } + return present; + } + + const scope = client.dbType === "mysql" ? "DATABASE()" : "current_schema()"; + const placeholders = PLUGIN_COLUMNS.map((_, index) => client.placeholder(index + 1)).join(", "); + + const rows = await client.query<{ column_name?: string; COLUMN_NAME?: string }>( + `SELECT column_name FROM information_schema.columns + WHERE table_name = 'user' AND table_schema = ${scope} + AND column_name IN (${placeholders})`, + [...PLUGIN_COLUMNS], + ); + + for (const row of rows) { + // MySQL 8 answers with an upper-case column label. + const name = (row.column_name ?? row.COLUMN_NAME) as PluginColumn | undefined; + if (name && (PLUGIN_COLUMNS as readonly string[]).includes(name)) present.add(name); + } + + return present; +} + +/** + * Builds the SELECT, including only the plugin columns that exist. + * + * @param pluginColumns - From {@link detectPluginColumns}. + */ +export function buildBetterAuthQuery(client: DbClient, pluginColumns: Set): string { + const q = (identifier: string) => client.quote(identifier); + const selected = [ + ...CORE_COLUMNS.map((column) => `u.${q(column)}`), + ...PLUGIN_COLUMNS.filter((column) => pluginColumns.has(column)).map( + (column) => `u.${q(column)}`, + ), + ]; + + // LEFT JOIN, not INNER: a user who only ever signed in with OAuth has no + // credential account, and dropping them would silently shrink the export. + return ( + `SELECT ${selected.join(", ")}, a.${q("password")} AS ${q("password_hash")} ` + + `FROM ${q("user")} u ` + + `LEFT JOIN ${q("account")} a ON a.${q("userId")} = u.${q("id")} ` + + `AND a.${q("providerId")} = 'credential' ` + + `ORDER BY u.${q("id")} ASC` + ); +} + +type BetterAuthRow = Record & { id?: unknown }; + +/** Renames the schema's camelCase onto what the betterauth transformer reads. */ +const FIELD_ALIASES: Record = { + id: "user_id", + emailVerified: "email_verified", + phoneNumber: "phone_number", + phoneNumberVerified: "phone_number_verified", + displayUsername: "display_username", + createdAt: "created_at", + updatedAt: "updated_at", +}; + +export function buildBetterAuthExport(rows: BetterAuthRow[], dateTime: string) { + const users: Record[] = []; + const counts = { email: 0, emailVerified: 0, password: 0, name: 0, username: 0, phone: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + const user: Record = {}; + + for (const [key, value] of Object.entries(row)) { + if (value === null || value === undefined) continue; + user[FIELD_ALIASES[key] ?? key] = value instanceof Date ? value.toISOString() : value; + } + + if (row.email) counts.email++; + if (row.emailVerified) counts.emailVerified++; + if (row.password_hash) counts.password++; + if (row.name) counts.name++; + if (row.username) counts.username++; + if (row.phoneNumber) counts.phone++; + + users.push(user); + exportLogger({ userId, status: "success" }, dateTime); + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.emailVerified }, + { label: "have a password hash", count: counts.password }, + { label: "have a name", count: counts.name }, + { label: "have a username", count: counts.username }, + { label: "have a phone number", count: counts.phone }, + ], + }; +} + +const BETTERAUTH_DB = { + platform: "betterauth", + envVar: "BETTERAUTH_DB_URL", + prompt: "Better Auth database connection string", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Better Auth install uses.", +} as const satisfies ResolveConfig; + +export async function exportBetterAuth(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, BETTERAUTH_DB); + + const destination = await resolveOutputPath("betterauth", options.output); + + await withGutter("Exporting users from Better Auth", async ({ setNextSteps }) => { + const dateTime = await startLogging(); + + const { + value: { rows, plugins }, + } = await withInputRetry( + dbUrl, + async () => promptDbUrl(BETTERAUTH_DB), + async (connectionString) => + withSpinner("Reading the user table...", async () => + withDbClient(connectionString, "betterauth", async (client) => { + const plugins = await detectPluginColumns(client); + const rows = await client.query(buildBetterAuthQuery(client, plugins)); + return { rows, plugins }; + }), + ), + ); + + log.info( + plugins.size > 0 + ? `Detected plugin columns: ${[...plugins].join(", ")}.` + : "No plugin columns detected; exporting the core user fields.", + ); + + const { users, coverage } = buildBetterAuthExport(rows, dateTime); + const outputPath = writeExportOutput(users, destination); + + setNextSteps( + reportExport({ + platform: "betterauth", + userCount: users.length, + outputPath, + coverage, + transformerKey: "betterauth", + }), + ); + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts new file mode 100644 index 000000000..1e5d95566 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -0,0 +1,241 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +const mockDescribeBapiTarget = mock(); +const mockResolveBapiSecretKey = mock(); +mock.module("../../../lib/bapi-command.ts", () => ({ + describeBapiTarget: (...args: unknown[]) => mockDescribeBapiTarget(...args), + resolveBapiSecretKey: (...args: unknown[]) => mockResolveBapiSecretKey(...args), +})); + +const mockResolveProfile = mock(); +mock.module("../../../lib/config.ts", () => ({ + resolveProfile: (...args: unknown[]) => mockResolveProfile(...args), +})); + +const mockFetchApps = mock(); +mock.module("../../../lib/app-picker.ts", () => ({ + fetchAppsTolerantly: (...args: unknown[]) => mockFetchApps(...args), +})); + +const mockSearch = mock(); +mock.module("../../../lib/listage.ts", () => ({ + search: (...args: unknown[]) => mockSearch(...args), +})); + +const mockResolveUsersInstanceContext = mock(); +mock.module("../../users/interactive/instance-context.ts", () => ({ + resolveUsersInstanceContext: (...args: unknown[]) => mockResolveUsersInstanceContext(...args), +})); + +let human = true; +mock.module("../../../mode.ts", () => ({ + isHuman: () => human, + isAgent: () => !human, + getMode: () => (human ? "human" : "agent"), + setMode: () => {}, +})); + +const { resolveClerkSource } = await import("./clerk-source.ts"); + +const captured = useCaptureLog(); + +beforeEach(() => { + human = true; + mockDescribeBapiTarget.mockReset(); + mockResolveBapiSecretKey.mockReset(); + mockResolveProfile.mockReset(); + mockResolveProfile.mockResolvedValue(undefined); + mockResolveUsersInstanceContext.mockReset(); + mockFetchApps.mockReset(); + mockSearch.mockReset(); + delete process.env.CLERK_SECRET_KEY; +}); + +/** The linked-project case: something resolved, nobody asked for it. */ +function stubResolved(target: string | undefined, secretKey = "sk_test_resolved") { + mockDescribeBapiTarget.mockResolvedValue(target); + mockResolveBapiSecretKey.mockResolvedValue(secretKey); +} + +describe("resolveClerkSource", () => { + test("--secret-key names the instance outright and is never questioned", async () => { + stubResolved(undefined, "sk_test_explicit"); + + const source = await resolveClerkSource({ secretKey: "sk_test_explicit" }); + + expect(source).toEqual({ secretKey: "sk_test_explicit", target: undefined }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // An export has to be scriptable outside agent mode. Before this, `chosen` + // keyed off `--secret-key` alone, so every other way of naming an instance + // still stopped at a picker no flag could answer. + test.each([ + ["--app", { app: "app_1" }], + ["--instance", { instance: "prod" }], + ["--app and --instance", { app: "app_1", instance: "prod" }], + ])("%s names the instance, so nothing is asked", async (_label, options) => { + stubResolved("my-app (production)"); + + const source = await resolveClerkSource(options); + + expect(source).toEqual({ secretKey: "sk_test_resolved", target: "my-app (production)" }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // `resolveBapiSecretKey` puts an exported key above the linked profile for + // every other command in this family. Opening a picker here — and then + // overriding the key with whatever it returned — made this the one command + // where exporting CLERK_SECRET_KEY did less than not exporting it. + test("an exported CLERK_SECRET_KEY is not second-guessed in a linked directory", async () => { + process.env.CLERK_SECRET_KEY = "sk_test_from_env"; + stubResolved("my-app (development)", "sk_test_from_env"); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_test_from_env", target: "my-app (development)" }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // Exporting the instance that is about to be imported *into* is the failure + // this whole module exists to prevent, so a resolved instance is offered as + // one choice among the account's applications rather than taken silently. + test("offers every instance, flat, with the linked application's first", async () => { + stubResolved("my-app (development)"); + mockResolveProfile.mockResolvedValue({ profile: { appId: "app_2" } }); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "my-app", + instances: [ + { instance_id: "ins_1d", environment_type: "development" }, + { instance_id: "ins_1p", environment_type: "production" }, + ], + }, + { + application_id: "app_2", + name: "other-app", + instances: [{ instance_id: "ins_2d", environment_type: "development" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + mockResolveUsersInstanceContext.mockResolvedValue({ + secretKey: "sk_live_other", + appLabel: "my-app", + instanceLabel: "production", + }); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_live_other", target: "my-app (production)" }); + // One row per instance, not per application: dev and prod are different + // user pools, and exporting the wrong one is silent. + const { message, source: listSource } = mockSearch.mock.calls[0]![0]; + expect(message).toBe("What Clerk instance do you want to export users from?"); + expect(listSource("")).toEqual([ + { + name: "other-app - Development instance (ins_2d)", + value: { app: "app_2", instance: "ins_2d" }, + }, + { + name: "my-app - Development instance (ins_1d)", + value: { app: "app_1", instance: "ins_1d" }, + }, + { + name: "my-app - Production instance (ins_1p)", + value: { app: "app_1", instance: "ins_1p" }, + }, + ]); + // Both halves are handed on, so the secret-key lookup runs against exactly + // the instance that was chosen and nothing prompts a second time. + expect(mockResolveUsersInstanceContext).toHaveBeenCalledWith({ + app: "app_1", + instance: "ins_1p", + }); + expect(captured.err).toBe(""); + }); + + // The list is searched by its rendered label, so an application id typed from + // a dashboard URL still finds its instances. + test("filters on the rendered label", async () => { + stubResolved("my-app (development)"); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "my-app", + instances: [{ instance_id: "ins_1p", environment_type: "production" }], + }, + { + application_id: "app_2", + name: "other-app", + instances: [{ instance_id: "ins_2d", environment_type: "development" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + mockResolveUsersInstanceContext.mockResolvedValue({ secretKey: "sk_live_other" }); + + await resolveClerkSource({}); + + const { source: listSource } = mockSearch.mock.calls[0]![0]; + expect(listSource("ins_2d")).toEqual([ + { + name: "other-app - Development instance (ins_2d)", + value: { app: "app_2", instance: "ins_2d" }, + }, + ]); + expect(listSource("production")).toHaveLength(1); + }); + + // An empty list is not a picker. PLAPI being degraded looks the same as an + // account with no applications, and neither one has an instance to offer. + test("no instances to offer falls back to the flags", async () => { + stubResolved("my-app (development)"); + // An application with no instances is not an offer either. + mockFetchApps.mockResolvedValue([{ application_id: "app_1", name: "my-app", instances: [] }]); + + await expect(resolveClerkSource({})).rejects.toBeInstanceOf(UserAbortError); + + expect(mockSearch).not.toHaveBeenCalled(); + expect(captured.err).toContain("--secret-key"); + expect(captured.err).toContain("--app"); + expect(captured.err).toContain("clerk link"); + }); + + test("agent mode takes the resolved instance without prompting", async () => { + human = false; + stubResolved("my-app (production)"); + + const source = await resolveClerkSource({}); + + expect(source.secretKey).toBe("sk_test_resolved"); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("an unlinked directory picks an application instead of failing", async () => { + mockDescribeBapiTarget.mockRejectedValue( + new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }), + ); + mockResolveUsersInstanceContext.mockResolvedValue({ + secretKey: "sk_test_picked", + appLabel: "other-app", + instanceLabel: "production", + }); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_test_picked", target: "other-app (production)" }); + // The picker just asked which application; asking again is noise. + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("an explicit --app that fails to resolve surfaces the error, not the picker", async () => { + const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); + mockDescribeBapiTarget.mockRejectedValue(failure); + + await expect(resolveClerkSource({ app: "app_123" })).rejects.toThrow(failure); + + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts new file mode 100644 index 000000000..78dfa201c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -0,0 +1,188 @@ +/** + * Which Clerk instance `migrate export clerk` reads *from*. + * + * Every other resolver in the CLI answers "where do I operate?" with the linked + * project, silently. For an export that default is actively dangerous: the + * linked instance is normally the migration's *destination*, so taking it + * without asking is how a run ends up exporting an instance and importing it + * straight back into itself. + * + * So the source is resolved in three tiers: + * + * 1. The user named the instance — `--secret-key`, `--app`, `--instance`, or + * an exported `CLERK_SECRET_KEY` — and it runs unquestioned. An exported + * key outranks the linked profile everywhere else in this CLI + * (`resolveBapiSecretKey`), and this command is not the one place that + * should differ. + * 2. Anything the CLI resolved on the user's behalf (the linked project, a + * keyless app) is not taken silently: every instance on the account is + * offered, with the resolved application's instances first so "yes, that + * one" is still a single Enter. + * 3. Nothing to resolve at all — no link, no key, no flags — falls back to the + * application picker `users list` uses, rather than failing on an unlinked + * directory. That one is `clerk link`'s, so it also offers "create a new + * application"; an empty application is never the right source, but the + * alternative here is an error, not a better list. + */ + +import { fetchAppsTolerantly } from "../../../lib/app-picker.ts"; +import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; +import { resolveProfile } from "../../../lib/config.ts"; +import { CliError, ERROR_CODE, throwUserAbort } from "../../../lib/errors.ts"; +import { search } from "../../../lib/listage.ts"; +import type { ApplicationInstance } from "../../../lib/plapi.ts"; +import { log } from "../../../lib/log.ts"; +import { isHuman } from "../../../mode.ts"; +import { resolveUsersInstanceContext } from "../../users/interactive/instance-context.ts"; + +/** e.g. `Development instance`. Unknown environment types print as-is. */ +function instanceLabel(instance: ApplicationInstance): string { + const type = instance.environment_type; + if (!type) return "instance"; + return `${type.charAt(0).toUpperCase()}${type.slice(1)} instance`; +} + +export type ResolveClerkSourceOptions = { + secretKey?: string; + app?: string; + instance?: string; + cwd?: string; +}; + +export type ClerkExportSource = { + secretKey: string; + /** + * Human-readable target, e.g. `my-app (production)`. Absent when + * `--secret-key` (or `CLERK_SECRET_KEY`) named the instance directly, since + * a bare key carries no application context to describe. + */ + target?: string; +}; + +/** {@link ClerkExportSource} plus whether the user already chose it out loud. */ +type ResolvedSource = ClerkExportSource & { chosen: boolean }; + +async function resolveSource(options: ResolveClerkSourceOptions): Promise { + // What separates the two tiers is not "did the CLI have to look anything up" + // but "did the user say which instance". A flag or an exported key is a + // sentence they typed for this run; the linked project is a choice they made + // for some other purpose, possibly months ago, and normally names the + // migration's destination rather than its source. Only the second is worth + // asking about. + // + // `--instance` counts on its own: paired with the linked application it + // names one instance, and pairing it with `--app` addresses any instance on + // the account. Without this an export could not be scripted at all outside + // agent mode — every run would stop at a picker no flag could answer. + const named = + Boolean(options.secretKey) || + Boolean(options.app) || + Boolean(options.instance) || + Boolean(process.env.CLERK_SECRET_KEY); + + try { + return { + target: await describeBapiTarget(options), + secretKey: await resolveBapiSecretKey(options), + chosen: named, + }; + } catch (error) { + if ( + !isHuman() || + named || + !(error instanceof CliError) || + error.code !== ERROR_CODE.NO_SECRET_KEY + ) { + throw error; + } + + const ctx = await resolveUsersInstanceContext({}); + return { + secretKey: ctx.secretKey, + target: ctx.appLabel ? `${ctx.appLabel} (${ctx.instanceLabel})` : undefined, + // The picker just asked. Confirming the answer to a question the user + // answered one prompt ago is noise. + chosen: true, + }; + } +} + +/** + * Offers every instance on the account, flat — one row per instance rather than + * an application picker followed by an instance picker. + * + * An application is not what an export reads from; an instance is. Picking + * "Migration Test" and then "development" is two questions with one answer, and + * it hides the thing that actually matters — dev and prod are different user + * pools, and exporting the wrong one is silent. + * + * Deliberately not `pickOrCreateApp`: its "+ Create a new application" choice + * makes sense when you are choosing somewhere to *write*, and no sense at all + * as an export source — a brand-new application has no users in it. + * + * @param currentAppId the application the CLI resolved on the user's behalf. + * Its instances lead the list, because they are the likeliest answer. + * @returns undefined when there is nothing to offer, so the caller can fall + * back to telling the user which flags to pass instead of showing an empty + * list. `fetchAppsTolerantly` returns empty on a degraded PLAPI, not just on + * an account with no applications. + */ +async function pickInstance(currentAppId?: string): Promise { + const apps = await fetchAppsTolerantly(); + + const ordered = currentAppId + ? [ + ...apps.filter((app) => app.application_id === currentAppId), + ...apps.filter((app) => app.application_id !== currentAppId), + ] + : apps; + + const choices = ordered.flatMap((app) => + (app.instances ?? []).map((instance) => ({ + name: `${app.name || app.application_id} - ${instanceLabel(instance)} (${instance.instance_id})`, + value: { app: app.application_id, instance: instance.instance_id }, + })), + ); + if (choices.length === 0) return undefined; + + const picked = await search<{ app: string; instance: string }>({ + message: "What Clerk instance do you want to export users from?", + source: (term) => + term + ? choices.filter((choice) => choice.name.toLowerCase().includes(term.toLowerCase())) + : choices, + }); + + // Both halves are passed on, so the secret-key lookup runs against exactly + // the instance that was chosen and nothing prompts a second time. + const ctx = await resolveUsersInstanceContext(picked); + return { + secretKey: ctx.secretKey, + target: ctx.appLabel ? `${ctx.appLabel} (${ctx.instanceLabel})` : undefined, + }; +} + +/** The application the CLI resolved on the user's behalf, if it knows one. */ +async function currentAppId(options: ResolveClerkSourceOptions): Promise { + if (options.app) return options.app; + const resolved = await resolveProfile(options.cwd ?? process.cwd()).catch(() => undefined); + return resolved?.profile.appId; +} + +export async function resolveClerkSource( + options: ResolveClerkSourceOptions, +): Promise { + const { chosen, ...source } = await resolveSource(options); + if (chosen || !source.target || !isHuman()) return source; + + const picked = await pickInstance(await currentAppId(options)); + if (picked) return picked; + + log.info( + "Export from a different instance with one of:\n" + + " `--secret-key ` — the source instance's secret key\n" + + " `--app --instance ` — another application on your account\n" + + " `clerk link` — link this directory to a different application first", + ); + throwUserAbort(); +} diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts new file mode 100644 index 000000000..67f825172 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -0,0 +1,327 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildClerkExport, + exportClerk, + fetchAllClerkUsers, + mapClerkUserToExport, +} from "./clerk.ts"; + +const captured = useCaptureLog(); +useMigrateLogDir(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: string[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expclerk-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +/** Answers `GET /v1/users` from `pages`, one page per call. */ +function stubPages(pages: unknown[][]) { + let call = 0; + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push(input.toString()); + return Response.json(pages[call++] ?? []); + }) as unknown as typeof fetch; +} + +const user = (overrides: Record = {}) => ({ + id: "user_1", + primary_email_address_id: "idn_1", + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + ], + phone_numbers: [], + ...overrides, +}); + +describe("mapClerkUserToExport", () => { + test("writes the field names the clerk transformer reads", () => { + expect( + mapClerkUserToExport(user({ first_name: "Ada", last_name: "L", username: "ada" })), + ).toMatchObject({ + id: "user_1", + primary_email_address: "a@x.dev", + first_name: "Ada", + last_name: "L", + username: "ada", + }); + }); + + // `migrate import` puts the first entry on POST /v1/users and attaches the rest + // afterwards, so a reordered list would change which address signs the user in. + test("keeps the primary identifier out of the additional list", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("a@x.dev"); + expect(mapped.verified_email_addresses).toEqual(["b@x.dev"]); + }); + + test("separates unverified identifiers", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "c@x.dev", verification: { status: "unverified" } }, + ], + }), + ); + expect(mapped.unverified_email_addresses).toEqual(["c@x.dev"]); + expect(mapped.verified_email_addresses).toBeUndefined(); + }); + + test("promotes the first verified address when none is flagged primary", () => { + const mapped = mapClerkUserToExport( + user({ + primary_email_address_id: null, + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("a@x.dev"); + expect(mapped.verified_email_addresses).toEqual(["b@x.dev"]); + }); + + test("maps phone numbers the same way", () => { + const mapped = mapClerkUserToExport( + user({ + primary_phone_number_id: "pn_1", + phone_numbers: [ + { id: "pn_1", phone_number: "+15555550100", verification: { status: "verified" } }, + { id: "pn_2", phone_number: "+15555550101", verification: { status: "unverified" } }, + ], + }), + ); + expect(mapped.primary_phone_number).toBe("+15555550100"); + expect(mapped.unverified_phone_numbers).toEqual(["+15555550101"]); + }); + + test("converts BAPI's Unix-millisecond timestamps to RFC3339", () => { + const mapped = mapClerkUserToExport(user({ created_at: 1704067200000 })); + expect(mapped.created_at).toBe("2024-01-01T00:00:00.000Z"); + }); + + test("omits empty metadata rather than writing empty objects", () => { + const mapped = mapClerkUserToExport( + user({ public_metadata: {}, private_metadata: { plan: "pro" } }), + ); + expect("public_metadata" in mapped).toBe(false); + expect(mapped.private_metadata).toEqual({ plan: "pro" }); + }); + + test("carries the account-state fields the import accepts", () => { + const mapped = mapClerkUserToExport( + user({ + banned: true, + create_organization_enabled: false, + create_organizations_limit: 3, + delete_self_enabled: true, + }), + ); + expect(mapped).toMatchObject({ + banned: true, + create_organization_enabled: false, + create_organizations_limit: 3, + delete_self_enabled: true, + }); + }); +}); + +describe("fetchAllClerkUsers", () => { + test("pages until a short page arrives", async () => { + stubPages([ + Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), + Array.from({ length: 12 }, (_, i) => user({ id: `v${i}` })), + ]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(512); + expect(requests).toHaveLength(2); + expect(requests[1]).toContain("offset=500"); + }); + + // A full final page must still trigger one more request, or an instance whose + // size is an exact multiple of the page size would look short by one page. + test("makes one more request when the last page is exactly full", async () => { + stubPages([Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), []]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(500); + expect(requests).toHaveLength(2); + }); + + test("asks for BAPI's maximum page size", async () => { + stubPages([[]]); + await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + expect(requests[0]).toContain("limit=500"); + }); + + test("copes with an instance that has no users", async () => { + stubPages([[]]); + expect(await fetchAllClerkUsers({ secretKey: "sk_test_x" })).toEqual([]); + }); +}); + +describe("buildClerkExport", () => { + test("counts coverage per field", () => { + const { coverage } = buildClerkExport( + [user({ id: "u1", first_name: "Ada", password_enabled: true }), user({ id: "u2" })], + "2026-01-01T00:00:00", + ); + + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + expect(byLabel["have a password (not exportable — see below)"]).toBe(1); + }); + + test("logs one NDJSON line per exported user", () => { + buildClerkExport([user({ id: "u1" }), user({ id: "u2" })], "2026-01-01T00:00:00"); + + const entries = fs + .readdirSync(getLogDir()) + .flatMap((name) => fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n")) + .map((line) => JSON.parse(line) as Record); + + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ userId: "u1", status: "success" }); + }); + + test("writes the export log where `logs list` will find it", () => { + buildClerkExport([user()], "2026-01-01T12:00:00"); + expect(fs.readdirSync(getLogDir())[0]).toBe("export-2026-01-01T12-00-00.log"); + }); +}); + +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + +describe("exportClerk", () => { + test("writes the default path and reports coverage", async () => { + stubPages([[user({ id: "u1", first_name: "Ada" })], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^clerk-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; + expect(written).toHaveLength(1); + expect(written[0]?.id).toBe("u1"); + expect(captured.err).toContain("Field coverage"); + expect(captured.err).toContain("Exported 1 user"); + }); + + test("names the command that consumes the file", async () => { + stubPages([[user()], []]); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + // --output answers the destination prompt, which human mode would + // otherwise stop on. + await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); + } finally { + setMode(originalMode); + } + expect(captured.err).toContain("migrate import --transformer clerk --file exports/mine.json"); + }); + + test("--output controls the destination, relative to the working directory", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x", output: "somewhere/mine.json" }); + + expect(fs.existsSync(path.join(workDir, "somewhere", "mine.json"))).toBe(true); + expect(fs.existsSync(path.join(workDir, "exports"))).toBe(false); + }); + + // Silence here would be the worst outcome: the operator finds out when + // nobody can sign in to the destination instance. + test("says plainly that passwords are not in the file", async () => { + stubPages([[user({ password_enabled: true })], []]); + await exportClerk({ secretKey: "sk_test_x" }); + expect(captured.err).toContain("never returns password digests"); + }); + + test("writes an empty file and says so when the instance has no users", async () => { + stubPages([[]]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("No users found to export"); + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toEqual([]); + }); + + test("an empty export warns but does not suggest importing it", async () => { + stubPages([[]]); + const originalMode = getMode(); + setMode("human"); + try { + await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); + } finally { + setMode(originalMode); + } + + expect(captured.err).toContain("No users found to export"); + expect(captured.err).not.toContain("Next steps"); + expect(captured.err).not.toContain("migrate --transformer"); + }); + + test("agent mode suppresses the Next steps block", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("Exported 1 user"); + expect(captured.err).not.toContain("Next steps"); + expect(captured.err).not.toContain("migrate --transformer"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts new file mode 100644 index 000000000..57af5cc88 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -0,0 +1,266 @@ +/** + * `clerk migrate export clerk` — pull users out of a Clerk instance. + * + * Ported from the standalone migration-tool's `src/export/clerk.ts`, rewritten + * onto `bapiRequest` instead of `@clerk/backend` so it shares the CLI's auth + * resolution, `--verbose` request tracing and error taxonomy. + * + * The output feeds `clerk migrate import --transformer clerk` unedited, which is + * what makes development → production a two-command operation. + * + * **Passwords do not come out of this endpoint.** Clerk never returns password + * digests, TOTP secrets or backup codes over the API; only the `*_enabled` + * booleans. The coverage report says how many users *have* a password so the + * gap is visible before the import, not after. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; +import { retryOn429 } from "../lib/retry.ts"; +import { resolveClerkSource } from "./clerk-source.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; + +/** BAPI's maximum page size for `GET /v1/users`. */ +const PAGE_SIZE = 500; + +export type ExportClerkOptions = { + output?: string; + secretKey?: string; + app?: string; + instance?: string; +}; + +type BapiIdentifier = { + email_address?: string; + phone_number?: string; + verification?: { status?: string } | null; +}; + +type BapiUser = { + id: string; + external_id?: string | null; + username?: string | null; + first_name?: string | null; + last_name?: string | null; + email_addresses?: BapiIdentifier[]; + phone_numbers?: BapiIdentifier[]; + primary_email_address_id?: string | null; + primary_phone_number_id?: string | null; + public_metadata?: Record; + private_metadata?: Record; + unsafe_metadata?: Record; + password_enabled?: boolean; + totp_enabled?: boolean; + banned?: boolean; + create_organization_enabled?: boolean; + create_organizations_limit?: number | null; + delete_self_enabled?: boolean; + created_at?: number; + legal_accepted_at?: number | null; +}; + +type IdentifierWithId = BapiIdentifier & { id?: string }; + +/** + * Splits identifiers into verified and unverified, primary first. + * + * The primary has to lead: `migrate import` puts the first entry on + * `POST /v1/users` and attaches the rest afterwards, so a reordered list would + * silently change which address the user signs in with. + */ +function splitIdentifiers( + entries: IdentifierWithId[] | undefined, + primaryId: string | null | undefined, + read: (entry: BapiIdentifier) => string | undefined, +): { primary?: string; verified: string[]; unverified: string[] } { + const verified: string[] = []; + const unverified: string[] = []; + let primary: string | undefined; + + for (const entry of entries ?? []) { + const value = read(entry); + if (!value) continue; + + if (entry.id && entry.id === primaryId) { + primary = value; + continue; + } + if (entry.verification?.status === "verified") verified.push(value); + else unverified.push(value); + } + + // No primary flagged: promote the first verified one so the export still has + // an identifier the import can lead with. + if (!primary && verified.length > 0) primary = verified.shift(); + + return { primary, verified, unverified }; +} + +/** Maps a BAPI user onto the shape the `clerk` transformer reads. */ +export function mapClerkUserToExport(user: BapiUser): Record { + const exported: Record = { id: user.id }; + + const emails = splitIdentifiers( + user.email_addresses, + user.primary_email_address_id, + (entry) => entry.email_address, + ); + if (emails.primary) exported.primary_email_address = emails.primary; + if (emails.verified.length > 0) exported.verified_email_addresses = emails.verified; + if (emails.unverified.length > 0) exported.unverified_email_addresses = emails.unverified; + + const phones = splitIdentifiers( + user.phone_numbers, + user.primary_phone_number_id, + (entry) => entry.phone_number, + ); + if (phones.primary) exported.primary_phone_number = phones.primary; + if (phones.verified.length > 0) exported.verified_phone_numbers = phones.verified; + if (phones.unverified.length > 0) exported.unverified_phone_numbers = phones.unverified; + + if (user.username) exported.username = user.username; + if (user.first_name) exported.first_name = user.first_name; + if (user.last_name) exported.last_name = user.last_name; + + for (const [source, target] of [ + ["public_metadata", "public_metadata"], + ["private_metadata", "private_metadata"], + ["unsafe_metadata", "unsafe_metadata"], + ] as const) { + const value = user[source]; + if (value && Object.keys(value).length > 0) exported[target] = value; + } + + if (user.banned) exported.banned = true; + if (user.create_organization_enabled !== undefined) { + exported.create_organization_enabled = user.create_organization_enabled; + } + if (user.create_organizations_limit !== null && user.create_organizations_limit !== undefined) { + exported.create_organizations_limit = user.create_organizations_limit; + } + if (user.delete_self_enabled !== undefined) { + exported.delete_self_enabled = user.delete_self_enabled; + } + + // BAPI reports timestamps as Unix milliseconds; the schema wants RFC3339. + if (user.created_at) exported.created_at = new Date(user.created_at).toISOString(); + if (user.legal_accepted_at) { + exported.legal_accepted_at = new Date(user.legal_accepted_at).toISOString(); + } + + return exported; +} + +/** Pages through every user in the instance. */ +export async function fetchAllClerkUsers(options: { + secretKey: string; + spinner?: SpinnerControls; +}): Promise { + const all: BapiUser[] = []; + + for (let offset = 0; ; offset += PAGE_SIZE) { + const response = await retryOn429(async () => + bapiRequest({ + method: "GET", + path: `/v1/users?limit=${PAGE_SIZE}&offset=${offset}`, + secretKey: options.secretKey, + }), + ); + + const page = Array.isArray(response.body) ? (response.body as BapiUser[]) : []; + all.push(...page); + options.spinner?.update(`Fetching users from Clerk: ${all.length} so far...`); + + // A short page means the end; anything else would loop forever on an + // instance whose size happens to be a multiple of the page size. + if (page.length < PAGE_SIZE) break; + } + + return all; +} + +export type ClerkExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +/** Maps every user and counts what the export actually contains. */ +export function buildClerkExport(users: BapiUser[], dateTime: string): ClerkExportResult { + const exported: Record[] = []; + const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0, password: 0 }; + + for (const user of users) { + try { + const mapped = mapClerkUserToExport(user); + exported.push(mapped); + + if (mapped.primary_email_address) counts.email++; + if (mapped.username) counts.username++; + if (mapped.first_name) counts.firstName++; + if (mapped.last_name) counts.lastName++; + if (mapped.primary_phone_number) counts.phone++; + if (user.password_enabled) counts.password++; + + exportLogger({ userId: user.id, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId: user.id, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a phone number", count: counts.phone }, + { label: "have a username", count: counts.username }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + { label: "have a password (not exportable — see below)", count: counts.password }, + ], + }; +} + +export async function exportClerk(options: ExportClerkOptions): Promise { + // Resolved before the gutter opens, the way `export auth0` resolves its + // credentials: confirming the source is a question about whether to run at + // all, not a step of the run. + const source = await resolveClerkSource({ + secretKey: options.secretKey, + app: options.app, + instance: options.instance, + }); + + const destination = await resolveOutputPath("clerk", options.output); + + await withGutter("Exporting users from Clerk", async ({ setNextSteps }) => { + const dateTime = await startLogging(); + + log.info(`Exporting from ${source.target ?? "the resolved instance"}.`); + + const users = await withSpinner("Fetching users from Clerk...", async (spinner) => + fetchAllClerkUsers({ secretKey: source.secretKey, spinner }), + ); + + const { users: exported, coverage } = buildClerkExport(users, dateTime); + const outputPath = writeExportOutput(exported, destination); + + setNextSteps( + reportExport({ + platform: "clerk", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "clerk", + }), + ); + + if (exported.length > 0) { + log.warn( + "Clerk's API never returns password digests, TOTP secrets or backup codes, so they are not in this file. " + + "Users will need to reset their password in the destination instance.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts new file mode 100644 index 000000000..62097f2b2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -0,0 +1,415 @@ +/** + * The three database-backed exports, driven against a real SQLite database. + * + * SQLite because it is the one engine that needs no container, and it + * exercises the same client, the same query building and the same plugin + * detection path (via `PRAGMA` rather than `information_schema`). Postgres and + * MySQL are covered by the manual matrix run recorded in the ticket. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { createDbClient, type DbClient } from "../lib/db.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { buildAuthJsExport, buildAuthJsQuery, exportAuthJs, fetchAuthJsUsers } from "./authjs.ts"; +import { + buildBetterAuthExport, + buildBetterAuthQuery, + detectPluginColumns, + exportBetterAuth, + PLUGIN_COLUMNS, +} from "./betterauth.ts"; +import { buildSupabaseExport } from "./supabase.ts"; +import { + looksLikeConnectionString, + normalizeConnectionString, + resolveDbUrl, +} from "./db-options.ts"; + +/** A cwd with no `.env` files, so these tests exercise only the injected env. */ +const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-dbexp-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +/** Builds a fresh SQLite file so each test starts from a known schema. */ +function makeDb(build: (db: Database) => void): string { + const file = path.join(workDir, `db-${counter++}.sqlite`); + const db = new Database(file, { create: true }); + build(db); + db.close(); + return file; +} + +function betterAuthDb(pluginColumns: string[], rows: Record[] = []): string { + return makeDb((db) => { + const extra = pluginColumns.map((column) => `, "${column}" TEXT`).join(""); + db.run( + `CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER, name TEXT, + "createdAt" TEXT, "updatedAt" TEXT${extra})`, + ); + db.run(`CREATE TABLE "account" (id TEXT, "userId" TEXT, "providerId" TEXT, password TEXT)`); + for (const row of rows) { + const keys = Object.keys(row); + db.run( + `INSERT INTO "user" (${keys.map((k) => `"${k}"`).join(",")}) VALUES (${keys.map(() => "?").join(",")})`, + keys.map((k) => row[k]) as never[], + ); + } + }); +} + +async function withClient(file: string, work: (client: DbClient) => Promise): Promise { + const client = await createDbClient(file); + try { + return await work(client); + } finally { + await client.close(); + } +} + +describe("looksLikeConnectionString", () => { + test.each([ + ["postgres://u:p@h:5432/db", true], + ["mysql://u:p@h:3306/db", true], + ["./db.sqlite", true], + ["file:./db.sqlite", true], + ["/abs/app.db", true], + ["", false], + [" ", false], + ["just some words", false], + ["postgres://", false], + ])("%p -> %p", (input, expected) => { + expect(looksLikeConnectionString(input)).toBe(expected); + }); +}); + +describe("normalizeConnectionString", () => { + test("encodes a password pasted in raw", () => { + const raw = "postgres://postgres:aB#c%92^d@db.example.supabase.co:5432/postgres"; + const normalized = normalizeConnectionString(raw); + + expect(looksLikeConnectionString(normalized)).toBe(true); + expect(decodeURIComponent(new URL(normalized).password)).toBe("aB#c%92^d"); + expect(new URL(normalized).hostname).toBe("db.example.supabase.co"); + }); + + test("encodes an unencoded @ in the password", () => { + const normalized = normalizeConnectionString("postgres://u:p@ss@host:5432/db"); + + expect(decodeURIComponent(new URL(normalized).password)).toBe("p@ss"); + expect(new URL(normalized).hostname).toBe("host"); + }); + + test("leaves an already-valid string alone", () => { + const encoded = "postgres://u:p%40ss@host:5432/db"; + expect(normalizeConnectionString(encoded)).toBe(encoded); + }); + + test("leaves non-URL forms alone", () => { + expect(normalizeConnectionString(" ./db.sqlite ")).toBe("./db.sqlite"); + }); +}); + +describe("resolveDbUrl", () => { + const config = { platform: "authjs" as const, envVar: "AUTHJS_DB_URL", prompt: "url" }; + + test("prefers the flag", async () => { + const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, NO_ENV_FILES, { + AUTHJS_DB_URL: "mysql://u:p@h/db", + }); + expect(url).toBe("postgres://u:p@h/db"); + }); + + test("falls back to the environment variable", async () => { + expect( + await resolveDbUrl({}, config, NO_ENV_FILES, { AUTHJS_DB_URL: "mysql://u:p@h/db" }), + ).toBe("mysql://u:p@h/db"); + }); + + test("encodes a raw password passed to the flag", async () => { + const url = await resolveDbUrl( + { dbUrl: "postgres://u:p#ss@host:5432/db" }, + config, + NO_ENV_FILES, + {}, + ); + expect(decodeURIComponent(new URL(url).password)).toBe("p#ss"); + }); + + test("rejects a flag that is not a connection string, naming the encoding trap", async () => { + await expect(resolveDbUrl({ dbUrl: "not a url" }, config, NO_ENV_FILES, {})).rejects.toThrow( + /URL-encode it/, + ); + }); + + test("warns and moves on when the environment variable is unusable", async () => { + // Tests run non-TTY, so it then hits the agent-mode branch. + await expect( + resolveDbUrl({}, config, NO_ENV_FILES, { AUTHJS_DB_URL: "garbage" }), + ).rejects.toThrow(/cannot prompt here/); + expect(captured.err).toContain("AUTHJS_DB_URL is not a valid connection string"); + }); + + // The env var reaching process.env is the runtime's job; this is the fallback + // for when it did not, and is the rung the secret key has always had. + test("falls back to a .env file when the variable is not in the environment", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-dburl-env-")); + fs.writeFileSync(path.join(dir, ".env.local"), "AUTHJS_DB_URL=postgres://u:p@h/db\n"); + + expect(await resolveDbUrl({}, config, dir, {})).toBe("postgres://u:p@h/db"); + }); + + test("names both the flag and the variable when it cannot prompt", async () => { + await expect(resolveDbUrl({}, config, NO_ENV_FILES, {})).rejects.toThrow( + /--db-url.*AUTHJS_DB_URL/s, + ); + }); +}); + +describe("authjs export", () => { + const authJsDb = (table: string) => + makeDb((db) => { + db.run( + `CREATE TABLE "${table}" (id TEXT PRIMARY KEY, name TEXT, email TEXT, "emailVerified" TEXT)`, + ); + db.run(`INSERT INTO "${table}" VALUES (?,?,?,?)`, [ + "aj1", + "Jane Doe", + "jane@x.dev", + "2024-01-15", + ]); + db.run(`INSERT INTO "${table}" VALUES (?,?,?,?)`, ["aj2", "John Smith", "john@x.dev", null]); + }); + + test("quotes identifiers for the dialect", async () => { + await withClient(authJsDb("User"), async (client) => { + expect(buildAuthJsQuery(client, "User")).toContain('"User"'); + expect(buildAuthJsQuery(client, "User")).toContain('"emailVerified" AS "email_verified"'); + }); + }); + + // Prisma capitalizes the table, Drizzle does not, and Auth.js has no single + // schema — so the export tries rather than making the user guess. + test.each([["User"], ["user"], ["users"]])("finds the %s table", async (table) => { + const { rows } = await withClient(authJsDb(table), fetchAuthJsUsers); + expect(rows).toHaveLength(2); + }); + + test("fails clearly when no candidate table exists", async () => { + const file = makeDb((db) => db.run(`CREATE TABLE unrelated (id TEXT)`)); + await expect(withClient(file, fetchAuthJsUsers)).rejects.toThrow( + /No Auth.js user table found. Tried User, user, users/, + ); + }); + + test("treats email_verified as a nullable timestamp, not a boolean", () => { + const { users } = buildAuthJsExport( + [ + { id: "a", email: "a@x.dev", email_verified: "2024-01-15" }, + { id: "b", email: "b@x.dev", email_verified: null }, + ], + "2026-01-01T00:00:00", + ); + expect(users[0]?.email_verified).toBe("2024-01-15"); + expect("email_verified" in (users[1] ?? {})).toBe(false); + }); + + test("counts coverage", () => { + const { coverage } = buildAuthJsExport( + [{ id: "a", email: "a@x.dev", name: "A", email_verified: "2024-01-01" }, { id: "b" }], + "2026-01-01T00:00:00", + ); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(1); + expect(byLabel["have a verified email"]).toBe(1); + }); + + test("exports end to end and says which table it read", async () => { + await exportAuthJs({ dbUrl: authJsDb("User"), output: "authjs.json" }); + + const written = JSON.parse(fs.readFileSync(path.join(workDir, "authjs.json"), "utf-8")); + expect(written).toHaveLength(2); + expect(captured.err).toContain("Read 2 rows from"); + expect(captured.err).toContain("stores no passwords"); + }); +}); + +describe("betterauth export", () => { + test("detects only the plugin columns that exist", async () => { + await withClient(betterAuthDb(["username", "banned"]), async (client) => { + expect([...(await detectPluginColumns(client))].sort()).toEqual(["banned", "username"]); + }); + }); + + test("detects nothing on a core-only schema", async () => { + await withClient(betterAuthDb([]), async (client) => { + expect((await detectPluginColumns(client)).size).toBe(0); + }); + }); + + test("detects every plugin column when all are present", async () => { + await withClient(betterAuthDb([...PLUGIN_COLUMNS]), async (client) => { + expect((await detectPluginColumns(client)).size).toBe(PLUGIN_COLUMNS.length); + }); + }); + + // Selecting a column that is not there fails the whole query, which is why + // the columns are detected rather than assumed. + test("selects only detected columns", async () => { + await withClient(betterAuthDb(["username"]), async (client) => { + const query = buildBetterAuthQuery(client, await detectPluginColumns(client)); + expect(query).toContain('"username"'); + expect(query).not.toContain('"twoFactorEnabled"'); + }); + }); + + test("the built query actually runs against the schema it was built for", async () => { + const file = betterAuthDb( + ["username", "role"], + [{ id: "u1", email: "a@x.dev", username: "a" }], + ); + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, await detectPluginColumns(client))), + ); + expect(rows).toHaveLength(1); + }); + + // A user who only ever signed in with OAuth has no credential account; + // an INNER JOIN would drop them and silently shrink the export. + test("keeps a user with no credential account", async () => { + const file = betterAuthDb( + [], + [ + { id: "u1", email: "a@x.dev" }, + { id: "u2", email: "b@x.dev" }, + ], + ); + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, new Set())), + ); + expect(rows).toHaveLength(2); + }); + + test("renames camelCase columns onto what the transformer reads", () => { + const { users } = buildBetterAuthExport( + [{ id: "u1", emailVerified: 1, phoneNumber: "+1555", createdAt: "2025-01-01" }], + "2026-01-01T00:00:00", + ); + expect(users[0]).toMatchObject({ + user_id: "u1", + email_verified: 1, + phone_number: "+1555", + created_at: "2025-01-01", + }); + }); + + test("exports end to end and reports the detected plugins", async () => { + const file = betterAuthDb(["username"], [{ id: "u1", email: "a@x.dev", username: "ada" }]); + + await exportBetterAuth({ dbUrl: file, output: "ba.json" }); + + expect(captured.err).toContain("Detected plugin columns: username"); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "ba.json"), "utf-8"))).toHaveLength(1); + }); + + test("says so plainly when no plugins are in use", async () => { + await exportBetterAuth({ dbUrl: betterAuthDb([]), output: "ba2.json" }); + expect(captured.err).toContain("No plugin columns detected"); + }); +}); + +describe("supabase export", () => { + test("serializes timestamps the transformer can parse", () => { + const { users } = buildSupabaseExport( + [{ id: "u1", email: "a@x.dev", created_at: new Date("2024-01-01T00:00:00Z") }], + "2026-01-01T00:00:00", + ); + expect(users[0]?.created_at).toBe("2024-01-01T00:00:00.000Z"); + }); + + test("omits null columns rather than exporting them", () => { + const { users } = buildSupabaseExport( + [{ id: "u1", email: "a@x.dev", phone: null, last_name: null }], + "2026-01-01T00:00:00", + ); + expect("phone" in (users[0] ?? {})).toBe(false); + expect("last_name" in (users[0] ?? {})).toBe(false); + }); + + test("counts the password hashes, the reason this reads the database", () => { + const { coverage } = buildSupabaseExport( + [ + { id: "u1", email: "a@x.dev", encrypted_password: "$2b$10$x" }, + { id: "u2", email: "b@x.dev" }, + ], + "2026-01-01T00:00:00", + ); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have a password hash"]).toBe(1); + }); + + test("keeps raw_app_meta_data, which --skip-unsupported-providers reads", () => { + const { users } = buildSupabaseExport( + [{ id: "u1", email: "a@x.dev", raw_app_meta_data: { providers: ["discord"] } }], + "2026-01-01T00:00:00", + ); + expect(users[0]?.raw_app_meta_data).toEqual({ providers: ["discord"] }); + }); + + test("logs one NDJSON line per exported user", () => { + buildSupabaseExport([{ id: "u1" }, { id: "u2" }], "2026-01-01T12:00:00"); + + const written = fs.readdirSync(getLogDir()); + expect(written[0]).toBe("export-2026-01-01T12-00-00.log"); + expect( + fs + .readFileSync(path.join(getLogDir(), written[0] as string), "utf-8") + .trim() + .split("\n"), + ).toHaveLength(2); + }); +}); + +describe("connection failures", () => { + afterEach(() => { + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); + }); + + test("a missing SQLite file fails before anything is written", async () => { + await expect(exportAuthJs({ dbUrl: "./definitely-not-here.sqlite" })).rejects.toThrow(CliError); + expect(fs.existsSync(path.join(workDir, "exports"))).toBe(false); + }); + + test("the failure never contains the password", async () => { + await expect( + exportBetterAuth({ dbUrl: "postgres://user:hunter2@127.0.0.1:1/db" }), + ).rejects.toThrow( + expect.objectContaining({ message: expect.not.stringContaining("hunter2") }) as Error, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts new file mode 100644 index 000000000..c0f92a92f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -0,0 +1,164 @@ +/** + * Resolving a `--db-url` for the database-backed exports. + * + * Shared by supabase, authjs and betterauth: all three take one connection + * string, from a flag, an environment variable, or a prompt. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { detectDbType, isLibsqlUrl, redactConnectionString, type DbPlatform } from "../lib/db.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; + +export type DbExportOptions = { + dbUrl?: string; + output?: string; +}; + +export type ResolveConfig = { + platform: DbPlatform; + /** Environment variable checked when `--db-url` is absent. */ + envVar: string; + prompt: string; + /** Extra guidance shown before prompting. */ + hint?: string; +}; + +const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2|libsql):\/\//i; + +/** + * True when the string parses as a URL with a host. + * + * A hostname is required: `postgres://` alone parses as a valid URL, and + * accepting it only defers the failure into the driver. + */ +function parsesAsUrl(value: string): boolean { + try { + return new URL(value).hostname.length > 0; + } catch { + return false; + } +} + +/** + * Percent-encodes the credentials when the raw string will not parse as a URL. + * + * Dashboards hand out `postgres://user:[YOUR-PASSWORD]@host/db` and people + * paste their real password in verbatim. A `#`, `@`, `/` or `^` in it makes the + * whole string unparseable — here and later inside `Bun.SQL` — so encode it for + * them rather than bouncing a paste they cannot even see (the prompt is + * masked). Strings that already parse are returned untouched, so a password + * that was correctly encoded is never double-encoded. + */ +export function normalizeConnectionString(value: string): string { + const trimmed = value.trim(); + if (!URL_SCHEME.test(trimmed) || parsesAsUrl(trimmed)) return trimmed; + + // Greedy up to the LAST `@`: everything before it is userinfo, so an + // unencoded `@` inside the password does not split the string early. + const match = /^([a-z0-9+]+:\/\/)(.*)@([^@]*)$/i.exec(trimmed); + if (!match) return trimmed; + + const [, scheme = "", userinfo = "", rest = ""] = match; + const separator = userinfo.indexOf(":"); + const user = separator === -1 ? userinfo : userinfo.slice(0, separator); + const secret = separator === -1 ? undefined : userinfo.slice(separator + 1); + const credentials = + secret === undefined + ? encodeURIComponent(user) + : `${encodeURIComponent(user)}:${encodeURIComponent(secret)}`; + + const encoded = `${scheme}${credentials}@${rest}`; + return parsesAsUrl(encoded) ? encoded : trimmed; +} + +/** True for something that could plausibly be a connection string. */ +export function looksLikeConnectionString(value: string): boolean { + const trimmed = value.trim(); + if (!trimmed) return false; + + if (URL_SCHEME.test(trimmed)) return parsesAsUrl(trimmed); + + return ( + trimmed.startsWith("file:") || /\.(sqlite3?|db)$/i.test(trimmed) || trimmed.startsWith("./") + ); +} + +/** + * Resolves the connection string: flag, then environment, then a prompt. + * + * Prompted as a password so it is not echoed — a connection string carries the + * database password inline. + */ +export async function resolveDbUrl( + options: DbExportOptions, + config: ResolveConfig, + cwd: string = process.cwd(), + env: Record = process.env, +): Promise { + const fromFlag = options.dbUrl ? normalizeConnectionString(options.dbUrl) : undefined; + if (fromFlag) { + if (!looksLikeConnectionString(fromFlag)) { + throwUsageError( + `--db-url does not look like a connection string. Expected postgres://…, mysql://…, libsql://… or a SQLite file path.\n` + + "If the password contains @, # or /, URL-encode it.", + ); + } + return fromFlag; + } + + const located = await findMigrateEnvValue([config.envVar], cwd, env); + const fromEnv = located ? normalizeConnectionString(located.value) : undefined; + if (fromEnv) { + if (looksLikeConnectionString(fromEnv)) return fromEnv; + // Falling through silently would make the prompt look unexplained. + log.warn(`${config.envVar} is not a valid connection string; ignoring it.`); + } + + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export ${config.platform}\` needs a database connection and cannot prompt here.\n` + + `Pass --db-url, or set ${config.envVar}.`, + undefined, + undefined, + [ + { + command: `clerk migrate export ${config.platform} --db-url "postgres://user:password@host:5432/db"`, + description: "Export from Postgres", + }, + ], + ); + } + + if (config.hint) log.info(dim(config.hint)); + + return promptDbUrl(config); +} + +/** + * Asks for a connection string, masked. + * + * Masked because a connection string carries the database password inline. The + * validator runs on the normalized value, so a password that needed encoding is + * judged as the driver will see it, not as it was typed. + */ +export async function promptDbUrl(config: ResolveConfig): Promise { + const answer = await passwordPrompt({ + message: config.prompt, + validate: (value) => + looksLikeConnectionString(normalizeConnectionString(value ?? "")) + ? undefined + : "Expected postgres://…, mysql://…, libsql://… or a SQLite file path", + }); + + return normalizeConnectionString(answer); +} + +/** Describes the target for the run's opening line, credentials removed. */ +export function describeTarget(connectionString: string): string { + const label = isLibsqlUrl(connectionString) ? "libsql" : detectDbType(connectionString); + return `${label} at ${redactConnectionString(connectionString)}`; +} diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts new file mode 100644 index 000000000..4d432e0a0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -0,0 +1,511 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildFirebaseExport, + exportFirebase, + fetchAccessToken, + fetchAllFirebaseUsers, + fetchHashConfig, + formatHashConfigGuidance, + loadServiceAccount, + mapFirebaseUserToExport, + readServiceAccount, + signServiceAccountJwt, + type ServiceAccount, +} from "./firebase.ts"; + +const captured = useCaptureLog(); +useMigrateLogDir(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { url: string; body: unknown }[]; +let account: ServiceAccount; + +beforeAll(async () => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-fb-"))); + process.chdir(workDir); + + // A real RSA key, so the signing path is genuinely exercised. + const pair = await crypto.subtle.generateKey( + { + name: "RSASSA-PKCS1-v1_5", + modulusLength: 2048, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, + true, + ["sign", "verify"], + ); + const pkcs8 = await crypto.subtle.exportKey("pkcs8", pair.privateKey); + const body = btoa(String.fromCharCode(...new Uint8Array(pkcs8))).replace(/(.{64})/g, "$1\n"); + + account = { + project_id: "demo-fb", + client_email: "exp@demo-fb.iam.gserviceaccount.com", + private_key: `-----BEGIN PRIVATE KEY-----\n${body}\n-----END PRIVATE KEY-----\n`, + }; + fs.writeFileSync( + path.join(workDir, "sa.json"), + JSON.stringify({ type: "service_account", ...account }), + ); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); + requests = []; + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; +}); + +/** Answers the token exchange, then one page per entry in `pages`. */ +function stubFirebase(pages: Record[][], hashConfig?: unknown) { + let page = 0; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ url, body: init?.body ?? null }); + + if (url.includes("oauth2.googleapis.com/token")) { + return Response.json({ access_token: "tok" }); + } + if (url.includes("/config")) { + return hashConfig === undefined + ? new Response("forbidden", { status: 403 }) + : Response.json(hashConfig); + } + const current = pages[page++] ?? []; + const hasMore = page < pages.length; + return Response.json({ users: current, ...(hasMore ? { nextPageToken: `p${page}` } : {}) }); + }) as unknown as typeof fetch; +} + +const fbUser = (i: number, overrides: Record = {}) => ({ + localId: `fb${i}`, + email: `u${i}@fb.dev`, + emailVerified: true, + displayName: `User ${i}`, + passwordHash: `SGFzaA${i}`, + salt: `U2FsdA${i}`, + createdAt: "1704067200000", + ...overrides, +}); + +describe("loadServiceAccount", () => { + // The prompt takes either, so a key pasted out of a password manager never + // has to be written to disk first. + test("accepts the key JSON pasted whole", () => { + expect(loadServiceAccount(` ${JSON.stringify(account)} `).project_id).toBe("demo-fb"); + }); + + test("accepts a path to the key file", () => { + expect(loadServiceAccount("./sa.json").project_id).toBe("demo-fb"); + }); + + test("rejects a paste that is not valid JSON", () => { + expect(() => loadServiceAccount('{"project_id":')).toThrow(/pasted key is not valid JSON/); + }); + + test("rejects a paste missing a required field", () => { + expect(() => loadServiceAccount('{"type":"service_account"}')).toThrow( + /The pasted key is not a usable service account key: "project_id" is missing/, + ); + }); +}); + +describe("readServiceAccount", () => { + test("reads a valid key file", () => { + expect(readServiceAccount("./sa.json").project_id).toBe("demo-fb"); + }); + + test("reports a path that is not there", () => { + expect(() => readServiceAccount("./nope.json")).toThrow(/No service account file at/); + }); + + test("reports a file that is not JSON", () => { + fs.writeFileSync(path.join(workDir, "bad.json"), "not json"); + expect(() => readServiceAccount("./bad.json")).toThrow(/is not valid JSON/); + }); + + // Downloading the web app config instead of a service account key is the + // usual mistake, and the two files look similar at a glance. + test("points at the right console page for a web app config", () => { + fs.writeFileSync(path.join(workDir, "web.json"), JSON.stringify({ apiKey: "x" })); + expect(() => readServiceAccount("./web.json")).toThrow(/"project_id" is missing/); + }); + + test("names a wrong type explicitly", () => { + fs.writeFileSync(path.join(workDir, "wrong.json"), JSON.stringify({ type: "authorized_user" })); + expect(() => readServiceAccount("./wrong.json")).toThrow( + /"type" is "authorized_user".*Generate new private key/s, + ); + }); + + test.each([["project_id"], ["client_email"], ["private_key"]])( + "reports a missing %s", + (field) => { + const partial: Record = { type: "service_account", ...account }; + delete partial[field]; + fs.writeFileSync(path.join(workDir, `no-${field}.json`), JSON.stringify(partial)); + expect(() => readServiceAccount(`./no-${field}.json`)).toThrow( + new RegExp(`"${field}" is missing`), + ); + }, + ); + + // Pasting a key through a form that eats newlines is common, and the failure + // would otherwise surface as an opaque crypto error. + test("catches a private key whose newlines were mangled", () => { + fs.writeFileSync( + path.join(workDir, "mangled.json"), + JSON.stringify({ type: "service_account", ...account, private_key: "mangled" }), + ); + expect(() => readServiceAccount("./mangled.json")).toThrow(/newlines survived copying/); + }); + + test("raises CliError so the global handler formats it", () => { + expect(() => readServiceAccount("./nope.json")).toThrow(CliError); + }); +}); + +describe("signServiceAccountJwt", () => { + test("produces a three-segment RS256 JWT", async () => { + const jwt = await signServiceAccountJwt(account); + expect(jwt.split(".")).toHaveLength(3); + }); + + test("claims the right issuer, audience and scopes", async () => { + const jwt = await signServiceAccountJwt(account, 1_700_000_000); + const claims = JSON.parse( + atob((jwt.split(".")[1] as string).replace(/-/g, "+").replace(/_/g, "/")), + ); + + expect(claims).toMatchObject({ + iss: "exp@demo-fb.iam.gserviceaccount.com", + aud: "https://oauth2.googleapis.com/token", + iat: 1_700_000_000, + exp: 1_700_003_600, + }); + expect(claims.scope).toContain("cloud-platform"); + }); + + test("declares RS256 in the header", async () => { + const jwt = await signServiceAccountJwt(account); + const header = JSON.parse( + atob((jwt.split(".")[0] as string).replace(/-/g, "+").replace(/_/g, "/")), + ); + expect(header).toEqual({ alg: "RS256", typ: "JWT" }); + }); + + test("rejects a private key that is not valid base64", async () => { + await expect( + signServiceAccountJwt({ + ...account, + private_key: "-----BEGIN PRIVATE KEY-----\n!!!\n-----END PRIVATE KEY-----", + }), + ).rejects.toThrow(CliError); + }); +}); + +describe("fetchAccessToken", () => { + test("exchanges the assertion for a token", async () => { + stubFirebase([[]]); + + expect(await fetchAccessToken(account)).toBe("tok"); + expect(String(requests[0]?.body)).toContain("grant-type%3Ajwt-bearer"); + }); + + test("explains a rejection rather than surfacing a raw status", async () => { + globalThis.fetch = (async () => + new Response(JSON.stringify({ error_description: "Invalid JWT Signature" }), { + status: 400, + })) as unknown as typeof fetch; + + await expect(fetchAccessToken(account)).rejects.toThrow( + /Google rejected the service account \(400\): Invalid JWT Signature/, + ); + }); + + test("names the role the service account usually lacks", async () => { + globalThis.fetch = (async () => new Response("{}", { status: 403 })) as unknown as typeof fetch; + await expect(fetchAccessToken(account)).rejects.toThrow(/Firebase Authentication Admin/); + }); + + // The emulator has no token endpoint; `firebase-admin` uses the same bearer. + test("skips the exchange entirely against the emulator", async () => { + process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; + globalThis.fetch = (async () => { + throw new Error("should not have been called"); + }) as unknown as typeof fetch; + + expect(await fetchAccessToken(account)).toBe("owner"); + }); +}); + +describe("fetchAllFirebaseUsers", () => { + test("follows nextPageToken until it stops coming", async () => { + stubFirebase([ + Array.from({ length: 1000 }, (_, i) => fbUser(i)), + Array.from({ length: 7 }, (_, i) => fbUser(1000 + i)), + ]); + + const all = await fetchAllFirebaseUsers({ account, token: "tok" }); + + expect(all).toHaveLength(1007); + expect(requests[1]?.url).toContain("nextPageToken=p1"); + }); + + test("asks for the endpoint's maximum page size", async () => { + stubFirebase([[]]); + await fetchAllFirebaseUsers({ account, token: "tok" }); + expect(requests[0]?.url).toContain("maxResults=1000"); + }); + + test("targets the project named in the key", async () => { + stubFirebase([[]]); + await fetchAllFirebaseUsers({ account, token: "tok" }); + expect(requests[0]?.url).toContain("/projects/demo-fb/accounts:batchGet"); + }); + + test("routes through the emulator when one is configured", async () => { + process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; + stubFirebase([[]]); + + await fetchAllFirebaseUsers({ account, token: "owner" }); + + expect(requests[0]?.url).toStartWith("http://127.0.0.1:9099/"); + }); + + test("raises a clear error on a failed page", async () => { + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + + await expect(fetchAllFirebaseUsers({ account, token: "tok" })).rejects.toThrow( + /Firebase returned 500 listing users/, + ); + }); +}); + +describe("mapFirebaseUserToExport", () => { + test("keeps the fields the firebase transformer maps from", () => { + expect(mapFirebaseUserToExport(fbUser(0))).toEqual({ + localId: "fb0", + email: "u0@fb.dev", + displayName: "User 0", + createdAt: "1704067200000", + emailVerified: true, + passwordHash: "SGFzaA0", + salt: "U2FsdA0", + }); + }); + + test("drops project internals the import has no use for", () => { + const mapped = mapFirebaseUserToExport( + fbUser(0, { + providerUserInfo: [{ providerId: "password" }], + lastLoginAt: "1704153600000", + customAttributes: '{"role":"x"}', + validSince: "1704067200", + }), + ); + for (const noise of ["providerUserInfo", "lastLoginAt", "customAttributes", "validSince"]) { + expect(noise in mapped).toBe(false); + } + }); + + // A digest without its salt cannot be verified, so exporting one alone would + // produce a user nobody can sign in as. + test.each([ + ["hash without salt", { passwordHash: "H", salt: undefined }], + ["salt without hash", { passwordHash: undefined, salt: "S" }], + ])("drops a %s", (_label, overrides) => { + const mapped = mapFirebaseUserToExport(fbUser(0, overrides)); + expect("passwordHash" in mapped).toBe(false); + expect("salt" in mapped).toBe(false); + }); + + test("keeps emailVerified when it is false", () => { + expect(mapFirebaseUserToExport(fbUser(0, { emailVerified: false })).emailVerified).toBe(false); + }); + + test("copes with a phone-only user", () => { + const mapped = mapFirebaseUserToExport({ localId: "fb9", phoneNumber: "+15555550100" }); + expect(mapped).toEqual({ localId: "fb9", phoneNumber: "+15555550100" }); + }); +}); + +describe("buildFirebaseExport", () => { + test("counts coverage and logs each user", () => { + const { users, coverage } = buildFirebaseExport( + [fbUser(0), { localId: "fb1", phoneNumber: "+1555" }], + "2026-01-01T12:00:00", + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have a password hash"]).toBe(1); + expect(byLabel["have a phone number"]).toBe(1); + expect(fs.readdirSync(getLogDir())[0]).toBe("export-2026-01-01T12-00-00.log"); + }); +}); + +describe("fetchHashConfig", () => { + test("reads the project's scrypt parameters", async () => { + stubFirebase([[]], { + signIn: { + hashConfig: { signerKey: "KEY==", saltSeparator: "Bw==", rounds: 8, memoryCost: 14 }, + }, + }); + + expect(await fetchHashConfig(account, "tok")).toEqual({ + signerKey: "KEY==", + saltSeparator: "Bw==", + rounds: 8, + memoryCost: 14, + }); + }); + + // Reading the config needs a broader role than listing users, so a project + // where it is denied must still export. + test("returns null rather than failing when the call is not permitted", async () => { + stubFirebase([[]]); + expect(await fetchHashConfig(account, "tok")).toBeNull(); + }); + + test("returns null when the response carries no hash config", async () => { + stubFirebase([[]], { signIn: {} }); + expect(await fetchHashConfig(account, "tok")).toBeNull(); + }); +}); + +describe("formatHashConfigGuidance", () => { + const config = { signerKey: "KEY==", saltSeparator: "Bw==", rounds: 8, memoryCost: 14 }; + + test("prints the exact import command when the parameters are known", () => { + const text = formatHashConfigGuidance(config, "exports/firebase-export.json", 3).join("\n"); + expect(text).toContain('--firebase-signer-key "KEY=="'); + expect(text).toContain('--firebase-salt-separator "Bw=="'); + expect(text).toContain("--firebase-rounds 8 --firebase-mem-cost 14"); + }); + + // The command is printed inside the gutter, which prefixes every line it is + // given with `│`. Split over lines, that character lands mid-command and is + // copied with it — the shell then reads each one as another argument and + // rejects the import. + test("keeps the command on one line, so it can be copied out of the gutter", () => { + const [command] = formatHashConfigGuidance(config, "out.json", 3).slice(-1); + expect(command).not.toContain("\n"); + expect(command).not.toContain("\\"); + }); + + test("says where to find them when the project would not say", () => { + const text = formatHashConfigGuidance(null, "out.json", 3).join("\n"); + expect(text).toContain("Password hash parameters"); + expect(text).toContain("Authentication → Users"); + }); + + // Nothing to configure, so nothing to tell them to configure. + test("says nothing is needed when the export has no hashes", () => { + expect(formatHashConfigGuidance(null, "out.json", 0).join("\n")).toContain( + "no hash parameters are needed", + ); + }); +}); + +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + +describe("exportFirebase", () => { + test("exports end to end and reports coverage", async () => { + stubFirebase([[fbUser(0), fbUser(1)]], { + signIn: { hashConfig: { signerKey: "K", saltSeparator: "S", rounds: 8, memoryCost: 14 } }, + }); + + await exportFirebase({ serviceAccount: "./sa.json" }); + + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^firebase-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; + expect(written).toHaveLength(2); + expect(captured.err).toContain("Field coverage"); + expect(captured.err).toContain("demo-fb project"); + }); + + test("names the command that consumes the file", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + // --output answers the destination prompt, which human mode would + // otherwise stop on. + await exportFirebase({ serviceAccount: "./sa.json", output: "exports/mine.json" }); + } finally { + setMode(originalMode); + } + expect(captured.err).toContain( + "migrate import --transformer firebase --file exports/mine.json", + ); + }); + + test("--output controls the destination", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json", output: "fb.json" }); + expect(fs.existsSync(path.join(workDir, "fb.json"))).toBe(true); + }); + + // Human runs get a prompt instead; an agent has nobody to ask, so it is told + // which flag to pass. + test("agent mode names the flag rather than prompting", async () => { + await expect(exportFirebase({})).rejects.toThrow(/needs a service account key file/); + }); + + test("validates the key file before making any request", async () => { + stubFirebase([[fbUser(0)]]); + await expect(exportFirebase({ serviceAccount: "./nope.json" })).rejects.toThrow(CliError); + expect(requests).toHaveLength(0); + }); + + test("never puts key material in the output", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(captured.err).not.toContain("BEGIN PRIVATE KEY"); + expect(captured.err).not.toContain(account.private_key.slice(40, 80)); + }); + + test("skips the hash-parameter section when nothing has a password", async () => { + stubFirebase([[{ localId: "fb9", phoneNumber: "+1555" }]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(captured.err).toContain("no hash parameters are needed"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts new file mode 100644 index 000000000..162c8e2be --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -0,0 +1,549 @@ +/** + * `clerk migrate export firebase` — pull users out of Firebase Authentication. + * + * Ported from the standalone migration-tool's `src/export/firebase.ts`, but + * **without `firebase-admin`**. + * + * The spike the ticket asked for was run first, and it passed: a + * `bun build --compile` binary imports `firebase-admin`, initializes it, and + * completes `listUsers` against Identity Toolkit. The known Firestore-under- + * compile bug does not reach the Auth Admin surface. + * + * The SDK was still not adopted, on the second measurement: it is **74 MB + * across 158 packages**, including `@google-cloud/firestore` and + * `@google-cloud/storage`, neither of which this command touches. The compiled + * `clerk` binary is ~65 MB today, so that roughly doubles the artifact every + * user downloads — to serve one subcommand. + * + * What the SDK actually does here is two REST calls and an RS256 JWT, and Bun's + * Web Crypto signs RS256 with no dependency at all (verified compiled). So this + * adds **zero** packages, and its HTTP goes through `loggedFetch`, so a + * `--verbose` run shows the requests — which an SDK doing its own fetch would + * not. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { bold, dim } from "../../../lib/color.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt } from "../../../lib/prompts.ts"; +import { isHuman } from "../../../mode.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; + +/** Identity Toolkit's maximum for `accounts:batchGet`. */ +const PAGE_SIZE = 1000; + +const TOKEN_URL = "https://oauth2.googleapis.com/token"; +const SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/firebase", +].join(" "); + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/firebase"; + +export type ExportFirebaseOptions = { + serviceAccount?: string; + output?: string; +}; + +export type ServiceAccount = { + project_id: string; + client_email: string; + private_key: string; +}; + +/** + * Validates already-parsed JSON as a service-account key. + * + * Every failure names the field, because the usual causes are downloading the + * wrong JSON from the console (a web app config rather than a service account) + * or pasting a key with its newlines mangled. + * + * @param label how to refer to the source in an error — a file name, or + * "the pasted key" when it came from the prompt. + */ +function validateServiceAccount(parsed: unknown, label: string): ServiceAccount { + const account = parsed as Partial & { type?: string }; + const invalid = (problem: string): never => { + throw new CliError(`${label} is not a usable service account key: ${problem}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + }; + + if (account.type && account.type !== "service_account") { + invalid( + `its "type" is "${account.type}", not "service_account". Download a private key from ` + + "Project settings → Service accounts → Generate new private key.", + ); + } + for (const field of ["project_id", "client_email", "private_key"] as const) { + if (typeof account[field] !== "string" || account[field].length === 0) { + invalid(`"${field}" is missing`); + } + } + if (!account.private_key?.includes("PRIVATE KEY")) { + invalid('"private_key" does not look like a PEM key — check its newlines survived copying'); + } + + return account as ServiceAccount; +} + +/** Reads and validates a service-account key file. */ +export function readServiceAccount(file: string): ServiceAccount { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No service account file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + + let parsed: unknown; + try { + parsed = JSON.parse(fs.readFileSync(resolved, "utf-8")); + } catch (error) { + throw new CliError(`${file} is not valid JSON: ${(error as Error).message}`, { + code: ERROR_CODE.INVALID_JSON, + docsUrl: DOCS_URL, + }); + } + + return validateServiceAccount(parsed, file); +} + +/** + * Accepts what the prompt accepts: a path to the downloaded key file, or the + * key's JSON pasted in whole. Console downloads land as a file, but a key + * copied out of a password manager or CI secret never touches disk. + */ +export function loadServiceAccount(source: string): ServiceAccount { + const trimmed = source.trim(); + if (!trimmed.startsWith("{")) return readServiceAccount(trimmed); + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch (error) { + throw new CliError(`The pasted key is not valid JSON: ${(error as Error).message}`, { + code: ERROR_CODE.INVALID_JSON, + docsUrl: DOCS_URL, + }); + } + + return validateServiceAccount(parsed, "The pasted key"); +} + +/** + * Resolves the key: the flag, then a prompt — the shape `export supabase` uses + * for its connection string. Prompted as a password: the JSON carries a private + * key, and a path typed blind is short enough to survive being masked. + */ +async function resolveServiceAccount(options: ExportFirebaseOptions): Promise { + if (options.serviceAccount) return loadServiceAccount(options.serviceAccount); + + if (!isHuman()) { + throwUsageError( + "`clerk migrate export firebase` needs a service account key file and cannot prompt here. " + + "Pass --service-account .", + DOCS_URL, + undefined, + [ + { + command: "clerk migrate export firebase --service-account ./service-account.json", + description: "Export using a downloaded service account key", + }, + ], + ); + } + + log.info( + dim("Firebase console → Project settings → Service accounts → Generate new private key."), + ); + + return promptServiceAccount(); +} + +/** + * Asks for the key, masked. + * + * Masked because the JSON carries a private key, and a path typed blind is + * short enough to survive it. What the file cannot tell us — whether Google + * still accepts the key — is left to the token exchange, which is why this is + * separate from {@link resolveServiceAccount}: a revoked key has to be asked + * for again after that call fails, not before it is made. + */ +async function promptServiceAccount(): Promise { + const answer = await passwordPrompt({ + message: "Path to the service account key file, or paste the key JSON", + validate: (value) => { + try { + loadServiceAccount(value ?? ""); + return undefined; + } catch (error) { + return error instanceof CliError ? error.message : String(error); + } + }, + }); + + return loadServiceAccount(answer); +} + +function base64Url(input: string | Uint8Array): string { + const binary = + typeof input === "string" ? input : String.fromCharCode(...(input as unknown as number[])); + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +/** Imports the PEM private key for RS256 signing. */ +async function importPrivateKey(pem: string): Promise { + const body = pem.replace(/-----[^-]+-----/g, "").replace(/\s+/g, ""); + let der: Uint8Array; + try { + der = Uint8Array.from(atob(body), (character) => character.charCodeAt(0)); + } catch { + throw new CliError("The service account's private_key is not valid base64.", { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + } + + try { + return await crypto.subtle.importKey( + "pkcs8", + der, + { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, + false, + ["sign"], + ); + } catch (error) { + throw new CliError( + `The service account's private_key could not be read: ${(error as Error).message}`, + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } +} + +/** + * Signs the assertion Google exchanges for an access token. + * + * @param now - Seconds since the epoch; injectable so tests are not clock-bound. + */ +export async function signServiceAccountJwt( + account: ServiceAccount, + now: number = Math.floor(Date.now() / 1000), +): Promise { + const key = await importPrivateKey(account.private_key); + const claims = { + iss: account.client_email, + scope: SCOPES, + aud: TOKEN_URL, + iat: now, + exp: now + 3600, + }; + const body = `${base64Url(JSON.stringify({ alg: "RS256", typ: "JWT" }))}.${base64Url(JSON.stringify(claims))}`; + + const signature = await crypto.subtle.sign( + "RSASSA-PKCS1-v1_5", + key, + new TextEncoder().encode(body), + ); + + return `${body}.${base64Url(new Uint8Array(signature))}`; +} + +/** + * Exchanges the signed assertion for an Identity Toolkit access token. + * + * Against the emulator there is nothing to exchange with — Google's token + * endpoint is not part of it — so the run uses the `owner` bearer the emulator + * accepts, matching what `firebase-admin` does. + */ +export async function fetchAccessToken(account: ServiceAccount): Promise { + if (process.env.FIREBASE_AUTH_EMULATOR_HOST) return "owner"; + + const assertion = await signServiceAccountJwt(account); + + const response = await loggedFetch(new URL(TOKEN_URL), { + tag: "firebase", + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", + assertion, + }).toString(), + }); + + const body = (await response.json().catch(() => ({}))) as { + access_token?: string; + error_description?: string; + error?: string; + }; + + if (!response.ok || !body.access_token) { + throw new CliError( + `Google rejected the service account (${response.status}): ${body.error_description ?? body.error ?? "no access token returned"}\n` + + "Check the key has not been revoked, and that the service account has the Firebase Authentication Admin role.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + return body.access_token; +} + +/** + * Base URL for Identity Toolkit. + * + * Honours `FIREBASE_AUTH_EMULATOR_HOST`, the variable Firebase's own tooling + * uses, so this works against the local emulator as well as production. + */ +function identityToolkitBase(): string { + const emulator = process.env.FIREBASE_AUTH_EMULATOR_HOST; + return emulator + ? `http://${emulator}/identitytoolkit.googleapis.com` + : "https://identitytoolkit.googleapis.com"; +} + +export type FirebaseUser = Record & { localId?: string }; + +/** Pages through every user in the project. */ +export async function fetchAllFirebaseUsers(options: { + account: ServiceAccount; + token: string; + spinner?: SpinnerControls; +}): Promise { + const all: FirebaseUser[] = []; + let pageToken: string | undefined; + + do { + const url = new URL( + `${identityToolkitBase()}/v1/projects/${options.account.project_id}/accounts:batchGet`, + ); + url.searchParams.set("maxResults", String(PAGE_SIZE)); + if (pageToken) url.searchParams.set("nextPageToken", pageToken); + + const response = await loggedFetch(url, { + tag: "firebase", + method: "GET", + headers: { Authorization: `Bearer ${options.token}`, Accept: "application/json" }, + }); + + if (!response.ok) { + throw new CliError( + `Firebase returned ${response.status} listing users: ${await response.text()}`, + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + const body = (await response.json()) as { users?: FirebaseUser[]; nextPageToken?: string }; + all.push(...(body.users ?? [])); + options.spinner?.update(`Fetching users from Firebase: ${all.length} so far...`); + pageToken = body.nextPageToken; + } while (pageToken); + + return all; +} + +export type HashConfig = { + signerKey: string; + saltSeparator: string; + rounds: number; + memoryCost: number; +}; + +/** + * Reads the project's scrypt parameters. + * + * These are the whole reason a Firebase migration keeps its passwords: without + * them Clerk cannot verify a single digest. Fetching them here saves the user + * hunting through the console — and if the call is not permitted, the run says + * exactly where to look instead. + * + * @returns `null` when the config could not be read. + */ +export async function fetchHashConfig( + account: ServiceAccount, + token: string, +): Promise { + try { + const url = new URL(`${identityToolkitBase()}/admin/v2/projects/${account.project_id}/config`); + const response = await loggedFetch(url, { + tag: "firebase", + method: "GET", + headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, + }); + if (!response.ok) { + log.debug(`firebase: ${response.status} reading the project config`); + return null; + } + + const body = (await response.json()) as { + signIn?: { hashConfig?: Partial & { algorithm?: string } }; + }; + const config = body.signIn?.hashConfig; + if (!config?.signerKey || !config.saltSeparator) return null; + + return { + signerKey: config.signerKey, + saltSeparator: config.saltSeparator, + rounds: Number(config.rounds ?? 8), + memoryCost: Number(config.memoryCost ?? 14), + }; + } catch (error) { + log.debug(`firebase: could not read the project config: ${String(error)}`); + return null; + } +} + +/** + * Keeps the fields the `firebase` transformer maps from. + * + * A Firebase user also carries provider records, custom claims and sign-in + * timestamps that would bloat the export and mean nothing to the import. + */ +export function mapFirebaseUserToExport(user: FirebaseUser): Record { + const exported: Record = {}; + + for (const field of ["localId", "email", "displayName", "phoneNumber", "createdAt"] as const) { + if (user[field]) exported[field] = user[field]; + } + // Meaningful when false, so copied on presence rather than truthiness. + if (user.emailVerified !== undefined) exported.emailVerified = user.emailVerified; + + // Both halves or neither: a digest without its salt cannot be verified. + if (user.passwordHash && user.salt) { + exported.passwordHash = user.passwordHash; + exported.salt = user.salt; + } + + return exported; +} + +export function buildFirebaseExport(users: FirebaseUser[], dateTime: string) { + const exported: Record[] = []; + const counts = { email: 0, verified: 0, password: 0, name: 0, phone: 0 }; + + for (const user of users) { + const userId = String(user.localId ?? ""); + try { + const mapped = mapFirebaseUserToExport(user); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.emailVerified) counts.verified++; + if (mapped.passwordHash) counts.password++; + if (mapped.displayName) counts.name++; + if (mapped.phoneNumber) counts.phone++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.verified }, + { label: "have a password hash", count: counts.password }, + { label: "have a display name", count: counts.name }, + { label: "have a phone number", count: counts.phone }, + ], + }; +} + +/** The exact `migrate import` invocation, with the project's own parameters. */ +export function formatHashConfigGuidance( + config: HashConfig | null, + outputPath: string, + passwordCount: number, +): string[] { + if (passwordCount === 0) { + return [dim("No password hashes in this export, so no hash parameters are needed.")]; + } + + if (!config) { + return [ + bold("Password hash parameters"), + "This export carries password hashes, which Clerk can only verify with the project's", + "scrypt parameters. Find them in the Firebase console under", + "Authentication → Users → (⋮) → Password hash parameters, then pass:", + dim( + " --firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost", + ), + ]; + } + + return [ + bold("Password hash parameters"), + "Read from the project. Import with:", + // One line, however long. Inside the gutter every line printed here is + // prefixed with `│`, and backslash continuations put that character in the + // middle of the command — copied along with it, and rejected by the shell + // as three extra arguments. A line that wraps on screen has no such + // character in it and pastes back as what was printed. + dim( + ` clerk migrate import -y --transformer firebase --file ${outputPath}` + + ` --firebase-signer-key "${config.signerKey}"` + + ` --firebase-salt-separator "${config.saltSeparator}"` + + ` --firebase-rounds ${config.rounds} --firebase-mem-cost ${config.memoryCost}`, + ), + ]; +} + +export async function exportFirebase(options: ExportFirebaseOptions): Promise { + // Read and validate before anything reaches the network, so a wrong file + // fails in a second rather than after an auth round-trip. + const resolved = await resolveServiceAccount(options); + + const destination = await resolveOutputPath("firebase", options.output); + + await withGutter("Exporting users from Firebase", async ({ setNextSteps }) => { + const dateTime = await startLogging(); + + // Only Google can say whether a well-formed key is still a valid one, so a + // revoked or deleted key fails here and is asked for again. + const { value: token, input: account } = await withInputRetry( + resolved, + promptServiceAccount, + async (candidate) => { + log.info(`Exporting from the ${candidate.project_id} project.`); + return withSpinner("Authenticating with Google...", async () => + fetchAccessToken(candidate), + ); + }, + ); + + const users = await withSpinner("Fetching users from Firebase...", async (spinner) => + fetchAllFirebaseUsers({ account, token, spinner }), + ); + + const { users: exported, coverage } = buildFirebaseExport(users, dateTime); + const outputPath = writeExportOutput(exported, destination); + + setNextSteps( + reportExport({ + platform: "firebase", + userCount: exported.length, + outputPath, + coverage, + transformerKey: "firebase", + }), + ); + + const passwordCount = coverage.find((entry) => entry.label.includes("password"))?.count ?? 0; + const hashConfig = passwordCount > 0 ? await fetchHashConfig(account, token) : null; + + log.blank(); + for (const line of formatHashConfigGuidance(hashConfig, outputPath, passwordCount)) { + log.info(line); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts new file mode 100644 index 000000000..07eae878d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -0,0 +1,224 @@ +import type { Command } from "@commander-js/extra-typings"; +import { throwUsageError } from "../../../lib/errors.ts"; +import { select } from "../../../lib/listage.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { exportAuth0 } from "./auth0.ts"; +import { exportAuthJs } from "./authjs.ts"; +import { exportBetterAuth } from "./betterauth.ts"; +import { exportClerk } from "./clerk.ts"; +import { exportFirebase } from "./firebase.ts"; +import { exportSupabase } from "./supabase.ts"; +import { exportWorkOs } from "./workos.ts"; +import type { DbExportOptions } from "./db-options.ts"; +import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; + +/** + * Bare `clerk migrate export` — pick a platform, then run its export. + * + * The picker is built from the registry, so a new platform appears without a + * second place to update. Whatever the chosen platform needs beyond the + * platform name, it prompts for itself. + */ +export async function exportPicker(options: Record = {}): Promise { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export\` needs a platform and cannot prompt here. Name one: ${exportPlatformKeys().join(", ")}.`, + undefined, + undefined, + exportPlatforms.map((entry) => ({ + command: `clerk migrate export ${entry.key}`, + description: entry.description, + })), + ); + } + + const platform = await select({ + message: "Which platform are you exporting from?", + choices: exportPlatforms.map((entry) => ({ + name: entry.label, + value: entry.key, + description: entry.description, + })), + }); + + const entry = getExportPlatform(platform); + // Unreachable via the picker; a guard so a registry edit cannot silently + // produce a choice with nothing behind it. + if (!entry) throwUsageError(`Unknown export platform "${platform}".`); + + await entry.run(options); +} + +const handlers = { + picker: exportPicker, + clerk: exportClerk, + auth0: exportAuth0, + supabase: exportSupabase, + authjs: exportAuthJs, + betterauth: exportBetterAuth, + firebase: exportFirebase, + workos: exportWorkOs, +}; + +/** The three platforms that read a database, which share `--db-url`. */ +const DB_PLATFORMS = [ + { + key: "supabase", + summary: "Export users from a Supabase Postgres database", + envVar: "SUPABASE_DB_URL", + example: "postgres://postgres:password@db.xxx.supabase.co:5432/postgres", + }, + { + key: "authjs", + summary: "Export users from an Auth.js database", + envVar: "AUTHJS_DB_URL", + example: "mysql://user:password@127.0.0.1:3306/authjs", + }, + { + key: "betterauth", + summary: "Export users from a Better Auth database", + envVar: "BETTERAUTH_DB_URL", + example: "./db.sqlite", + }, +] as const; + +/** Registers `export [platform]` under the `migrate` group. */ +export function registerMigrateExport(migrateCommand: Command<[], Record>): void { + const exportCommand = migrateCommand + .command("export") + .description("Export users from a source platform, ready for `clerk migrate import`") + .setExamples([ + { command: "clerk migrate export", description: "Pick a platform interactively" }, + { + command: "clerk migrate export clerk --output users.json", + description: "Export from a Clerk instance", + }, + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export from an Auth0 tenant", + }, + ]) + .action(async (_opts, cmd) => + handlers.picker(cmd.optsWithGlobals() as Record), + ); + + exportCommand + .command("clerk") + .description( + "Export users from a Clerk instance (default: ./exports/clerk-export-.json)", + ) + .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .option("--secret-key ", "Backend API secret key to use") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate export clerk", + description: "Prompts for the source instance and where to save the file", + }, + { + command: "clerk migrate export clerk --secret-key sk_live_… --output prod-users.json", + description: "Name the source instance outright, skipping the picker", + }, + ]) + .action(async (_opts, cmd) => + handlers.clerk(cmd.optsWithGlobals() as Parameters[0]), + ); + + exportCommand + .command("auth0") + .description( + "Export users from an Auth0 tenant (default: ./exports/auth0-export-.json)", + ) + .option("--domain ", "Auth0 tenant domain, e.g. my-tenant.us.auth0.com") + .option("--client-id ", "Machine-to-machine application client ID") + .option("--client-secret ", "Machine-to-machine application client secret") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .setExamples([ + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export with explicit credentials", + }, + { + command: "clerk migrate export auth0", + description: "Read AUTH0_DOMAIN, AUTH0_CLIENT_ID and AUTH0_CLIENT_SECRET, or prompt", + }, + ]) + .action(async (_opts, cmd) => + handlers.auth0(cmd.optsWithGlobals() as Parameters[0]), + ); + + exportCommand + .command("firebase") + .description( + "Export users from a Firebase project (default: ./exports/firebase-export-.json)", + ) + .option("--service-account ", "Path to a service account key JSON file") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .setExamples([ + { + command: "clerk migrate export firebase --service-account ./service-account.json", + description: "Export using a downloaded service account key", + }, + ]) + .action(async (_opts, cmd) => + handlers.firebase(cmd.optsWithGlobals() as Parameters[0]), + ); + + exportCommand + .command("workos") + .description( + "Export users from a WorkOS tenant (default: ./exports/workos-export-.json)", + ) + .option("--api-key ", "WorkOS secret API key, the one starting `sk_`") + .option( + "--with-identities", + "Also record each user's OAuth providers — one extra request per user", + ) + .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .setExamples([ + { + command: "clerk migrate export workos --api-key sk_…", + description: "Export with an explicit API key", + }, + { + command: "clerk migrate export workos", + description: "Read WORKOS_API_KEY, or prompt", + }, + ]) + .action(async (_opts, cmd) => + handlers.workos(cmd.optsWithGlobals() as Parameters[0]), + ); + + // All three take exactly one connection string, so they are registered from + // a table rather than three near-identical blocks. + for (const platform of DB_PLATFORMS) { + exportCommand + .command(platform.key) + .description( + `${platform.summary} (default: ./exports/${platform.key}-export-.json)`, + ) + .option("--db-url ", "Postgres, MySQL, libsql/Turso or SQLite connection string") + .option("-o, --output ", "Where to write the export, relative to the current directory") + .option("-y, --yes", "Do not prompt: require --output, and fail on a rejected credential") + .setExamples([ + { + command: `clerk migrate export ${platform.key} --db-url "${platform.example}"`, + description: "Export from an explicit database", + }, + { + command: `clerk migrate export ${platform.key}`, + description: `Read ${platform.envVar}, or prompt`, + }, + ]) + .action(async (_opts, cmd) => + handlers[platform.key](cmd.optsWithGlobals() as DbExportOptions), + ); + } +} diff --git a/packages/cli-core/src/commands/migrate/export/registry.test.ts b/packages/cli-core/src/commands/migrate/export/registry.test.ts new file mode 100644 index 000000000..f8471aa8b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/registry.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; +import { transformerKeys } from "../transformers/registry.ts"; +import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; + +describe("export registry", () => { + test("registers every source platform", () => { + expect(exportPlatformKeys()).toEqual([ + "clerk", + "auth0", + "supabase", + "authjs", + "firebase", + "betterauth", + "workos", + ]); + }); + + test.each([...exportPlatforms])("$key carries a label and description", (entry) => { + expect(entry.label.length).toBeGreaterThan(0); + expect(entry.description.length).toBeGreaterThan(0); + }); + + // The picker, the docs and the "what next" line all read this, so a typo + // would send someone to a transformer that does not exist. + test.each([...exportPlatforms])("$key names a real transformer", (entry) => { + expect(transformerKeys()).toContain(entry.transformerKey); + }); + + test.each([...exportPlatforms])("$key has something to run", (entry) => { + expect(typeof entry.run).toBe("function"); + }); + + test("looks a platform up by key", () => { + expect(getExportPlatform("auth0")?.label).toBe("Auth0"); + }); + + test("returns nothing for a platform that is not registered", () => { + expect(getExportPlatform("okta")).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/registry.ts b/packages/cli-core/src/commands/migrate/export/registry.ts new file mode 100644 index 000000000..a5d5a6c15 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/registry.ts @@ -0,0 +1,88 @@ +/** + * Export registry. + * + * The picker behind a bare `clerk migrate export` is built from this array, so + * adding a platform is one file plus one entry — the same shape the transformer + * registry uses. + * + * `run` takes no arguments on purpose: each platform resolves its own flags, + * environment variables and prompts, because what Auth0 needs (a tenant domain + * and M2M credentials) has nothing in common with what a database export needs. + */ + +import { exportAuth0 } from "./auth0.ts"; +import { exportAuthJs } from "./authjs.ts"; +import { exportBetterAuth } from "./betterauth.ts"; +import { exportClerk } from "./clerk.ts"; +import { exportFirebase } from "./firebase.ts"; +import { exportSupabase } from "./supabase.ts"; +import { exportWorkOs } from "./workos.ts"; + +export type ExportRegistryEntry = { + key: string; + label: string; + description: string; + /** Which `--transformer` reads the file this export writes. */ + transformerKey: string; + run: (options: Record) => Promise; +}; + +export const exportPlatforms: ExportRegistryEntry[] = [ + { + key: "clerk", + label: "Clerk", + description: "Another Clerk instance, e.g. development → production", + transformerKey: "clerk", + run: async (options) => exportClerk(options), + }, + { + key: "auth0", + label: "Auth0", + description: "An Auth0 tenant, via the Management API", + transformerKey: "auth0", + run: async (options) => exportAuth0(options), + }, + { + key: "supabase", + label: "Supabase", + description: "A Supabase Postgres database — includes password hashes", + transformerKey: "supabase", + run: async (options) => exportSupabase(options), + }, + { + key: "authjs", + label: "Auth.js (NextAuth)", + description: "An Auth.js database — Postgres, MySQL or SQLite", + transformerKey: "authjs", + run: async (options) => exportAuthJs(options), + }, + { + key: "firebase", + label: "Firebase", + description: "A Firebase project, via Identity Toolkit", + transformerKey: "firebase", + run: async (options) => exportFirebase(options), + }, + { + key: "betterauth", + label: "Better Auth", + description: "A Better Auth database — plugin columns detected automatically", + transformerKey: "betterauth", + run: async (options) => exportBetterAuth(options), + }, + { + key: "workos", + label: "WorkOS", + description: "A WorkOS tenant, via the User Management API — no password hashes", + transformerKey: "workos", + run: async (options) => exportWorkOs(options), + }, +]; + +export function exportPlatformKeys(): string[] { + return exportPlatforms.map((entry) => entry.key); +} + +export function getExportPlatform(key: string): ExportRegistryEntry | undefined { + return exportPlatforms.find((entry) => entry.key === key); +} diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts new file mode 100644 index 000000000..1c505ef81 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -0,0 +1,136 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import { type CliError, ERROR_CODE, EXIT_CODE } from "../../../lib/errors.ts"; + +const mockText = mock(); +mock.module("../../../lib/prompts.ts", () => ({ + text: (...args: unknown[]) => mockText(...args), +})); + +let human = true; +mock.module("../../../mode.ts", () => ({ + isHuman: () => human, + isAgent: () => !human, + getMode: () => (human ? "human" : "agent"), + setMode: () => {}, +})); + +const { defaultOutputPath, outputStamp, resolveOutputPath } = await import("./shared.ts"); +const { setAssumeYes } = await import("../lib/assume-yes.ts"); + +beforeEach(() => { + human = true; + setAssumeYes(false); + mockText.mockReset(); +}); + +describe("outputStamp", () => { + // Local time, and no seconds: this ends up in a filename someone reads off + // the screen and types back. + test("stamps to the minute", () => { + expect(outputStamp(new Date(2026, 7, 17, 14, 32, 59))).toBe("20260817-1432"); + }); + + test("pads single-digit months, days, hours and minutes", () => { + expect(outputStamp(new Date(2026, 0, 3, 9, 5, 0))).toBe("20260103-0905"); + }); +}); + +describe("defaultOutputPath", () => { + test("names the platform and the stamp, under exports/", () => { + expect(defaultOutputPath("clerk", new Date(2026, 7, 17, 14, 32))).toBe( + "exports/clerk-export-20260817-1432.json", + ); + }); + + // Two exports of the same platform an hour apart must not collide. + test("gives two runs different names", () => { + expect(defaultOutputPath("auth0", new Date(2026, 7, 17, 14, 32))).not.toBe( + defaultOutputPath("auth0", new Date(2026, 7, 17, 15, 32)), + ); + }); +}); + +describe("resolveOutputPath", () => { + test("--output is an answer already given", async () => { + expect(await resolveOutputPath("clerk", "somewhere/mine.json")).toBe("somewhere/mine.json"); + expect(mockText).not.toHaveBeenCalled(); + }); + + // One prompt, not a confirm plus a path question: the proposal is prefilled, + // so enter accepts it and typing replaces it. + test("prefills the proposed path so enter accepts it", async () => { + mockText.mockImplementation(async (config: { default: string }) => config.default); + + const chosen = await resolveOutputPath("clerk"); + + expect(chosen).toMatch(/^exports\/clerk-export-\d{8}-\d{4}\.json$/); + expect(mockText).toHaveBeenCalledTimes(1); + expect(mockText.mock.calls[0]?.[0]).toMatchObject({ message: "Save the export to:" }); + }); + + test("takes a path typed over the proposal, trimmed", async () => { + mockText.mockResolvedValue(" ../elsewhere/users.json "); + + expect(await resolveOutputPath("firebase")).toBe("../elsewhere/users.json"); + }); + + test("agent mode takes the proposed path without asking", async () => { + human = false; + + expect(await resolveOutputPath("supabase")).toMatch( + /^exports\/supabase-export-\d{8}-\d{4}\.json$/, + ); + expect(mockText).not.toHaveBeenCalled(); + }); + + // The one prompt whose default cannot be undone by running the command + // again: a file at a path nobody chose has to be found and moved, and a + // second run writes a second copy. So `-y` fails here rather than guessing. + describe("with -y", () => { + beforeEach(() => setAssumeYes(true)); + + test("fails rather than prompting or defaulting", async () => { + await expect(resolveOutputPath("supabase")).rejects.toThrow( + /needs an export location and will not prompt for one with -y/, + ); + expect(mockText).not.toHaveBeenCalled(); + }); + + test("is a usage error, so the exit code says what to fix", async () => { + const error = (await resolveOutputPath("supabase").catch((e: unknown) => e)) as CliError; + + expect(error.code).toBe(ERROR_CODE.USAGE_ERROR); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + }); + + // The whole point of failing instead of defaulting: the error has to hand + // back a line that runs, or it has cost the operator the run for nothing. + test("hands back the command to re-run, proposed path and all", async () => { + const error = (await resolveOutputPath("supabase").catch((e: unknown) => e)) as CliError; + + expect(error.examples?.[0]?.command).toMatch( + /^clerk migrate export supabase -y --output exports\/supabase-export-\d{8}-\d{4}\.json$/, + ); + }); + + test("names the platform that was actually run", async () => { + const error = (await resolveOutputPath("firebase").catch((e: unknown) => e)) as CliError; + + expect(error.message).toContain("`clerk migrate export firebase`"); + }); + + test("stays quiet when --output already answered it", async () => { + expect(await resolveOutputPath("clerk", "somewhere/mine.json")).toBe("somewhere/mine.json"); + }); + + // An agent passes `-y` reflexively and has no prompt to suppress, so the + // flag must not turn a working export into a usage error there. + test("still defaults in agent mode", async () => { + human = false; + + expect(await resolveOutputPath("supabase")).toMatch( + /^exports\/supabase-export-\d{8}-\d{4}\.json$/, + ); + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts new file mode 100644 index 000000000..ed8370475 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -0,0 +1,185 @@ +/** + * Shared plumbing for the export modules: where the file lands, and what the + * user is told about it. + * + * Ported from the standalone migration-tool's `src/lib/export.ts`, with one + * behavioural change: `--output` resolves against the **current working + * directory**, the way every other path flag in this CLI does. The original + * resolved a relative `--output` inside `exports/`, so `--output ./here.json` + * silently wrote to `exports/here.json`. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { dim, green, yellow } from "../../../lib/color.ts"; +import { throwUsageError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { NEXT_STEPS } from "../../../lib/next-steps.ts"; +import { text } from "../../../lib/prompts.ts"; +import { isHuman } from "../../../mode.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; + +/** + * `YYYYMMDD-HHmm`, local time — ISO 8601 basic format, minus seconds. + * + * Basic throughout rather than `2026-08-17-1954`, which mixes the extended + * date form with the basic time form and leaves the trailing group looking + * like a fourth date component. One separator, and it sorts lexically. + * + * Seconds are dropped on purpose. This lands in a filename people read off the + * screen, type back and tab-complete, and two exports of the same platform + * inside one minute is not an accident anyone has by surprise. + * + * Local rather than UTC because the only reader is the person who just ran the + * command, deciding which of two files is the one they meant. + */ +export function outputStamp(now: Date = new Date()): string { + const pad = (value: number) => String(value).padStart(2, "0"); + const date = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}`; + return `${date}-${pad(now.getHours())}${pad(now.getMinutes())}`; +} + +/** Where an export lands when `--output` is not given. */ +export function defaultOutputPath(platform: string, now?: Date): string { + return path.join("exports", `${platform}-export-${outputStamp(now)}.json`); +} + +/** + * Settles where the file lands, before the export runs. + * + * Asked up front rather than at write time so a long export can be left + * unattended — coming back to a stalled prompt with every user held in memory + * and nothing on disk is the worse half of that trade. + * + * One prompt, not a confirm followed by a path prompt: the proposed path is + * prefilled, so Enter accepts it and typing replaces it. + * + * `--output` is an answer already given, and agent mode has nobody to ask, so + * it takes the proposal. + * + * `-y` is neither: somebody is there, and they said not to ask. It fails + * instead of defaulting, because this is the one prompt whose default cannot + * be undone by running the command again — a file written to a path nobody + * chose has to be found and moved, and a second run writes a second copy. + * Silencing that question is what `--output` is for, so the error hands over + * the exact line, proposed path and all. (The log-directory question does take + * its default under `-y`: `./logs` is where the reader would look anyway, and + * nothing is saved.) + */ +export async function resolveOutputPath(platform: string, output?: string): Promise { + if (output) return output; + + const proposed = defaultOutputPath(platform); + // Ordered so agent mode keeps defaulting even when it also passes `-y`: + // there was never a prompt on that path to suppress. + if (!isHuman()) return proposed; + + if (isAssumeYes()) { + throwUsageError( + `\`clerk migrate export ${platform}\` needs an export location and will not prompt for one with -y.\nPass --output, then run it again.`, + undefined, + undefined, + [ + { + command: `clerk migrate export ${platform} -y --output ${proposed}`, + description: "Re-run with the proposed path", + }, + ], + ); + } + + const chosen = await text({ + message: "Save the export to:", + default: proposed, + validate: (value) => (value?.trim() ? undefined : "A path is required"), + }); + return chosen.trim(); +} + +/** + * Writes the export, creating any missing parent directories. + * + * @returns The absolute path written, for reporting. + */ +export function writeExportOutput(users: unknown[], outputFile: string): string { + const resolved = path.resolve(process.cwd(), outputFile); + fs.mkdirSync(path.dirname(resolved), { recursive: true }); + fs.writeFileSync(resolved, JSON.stringify(users, null, 2)); + return resolved; +} + +export type CoverageField = { label: string; count: number }; + +/** + * How complete an export is, per field. + * + * ✓ every user, ! some, dim ✗ none. The point is to see *before* importing + * that, say, only 3 of 400 users have a password — which changes what the + * migration means. + */ +export function formatFieldCoverage(fields: CoverageField[], total: number): string[] { + return fields.map(({ label, count }) => { + const icon = count === total ? green("✓") : count > 0 ? yellow("!") : dim("✗"); + return ` ${icon} ${dim(`${count}/${total} ${label}`)}`; + }); +} + +/** + * An extra block printed under the coverage table. + * + * For a breakdown that is not "how many users have this field" — WorkOS's + * OAuth providers, where one user can appear in two rows and the denominator + * is not the user count. Folding that into the coverage table would put rows + * of two different kinds under one heading. + */ +export type ExportSection = { title: string; rows: string[] }; + +export type ExportSummary = { + platform: string; + userCount: number; + outputPath: string; + coverage: CoverageField[]; + /** Extra blocks, printed under the coverage table in order. */ + sections?: ExportSection[]; + /** The transformer that reads this file, for the "what next" line. */ + transformerKey: string; +}; + +/** + * Reports the coverage table. + * + * @returns The next steps for the caller to hand to `setNextSteps`, so the + * suggested import command closes the gutter like every other command's. + * Empty when nothing was exported — there is nothing to import. + */ +export function reportExport(summary: ExportSummary): readonly string[] { + log.blank(); + if (summary.userCount === 0) { + log.warn(`No users found to export. Wrote an empty file to ${summary.outputPath}.`); + return []; + } + + log.info("Field coverage"); + for (const line of formatFieldCoverage(summary.coverage, summary.userCount)) { + log.info(line); + } + + for (const section of summary.sections ?? []) { + log.blank(); + log.info(section.title); + for (const row of section.rows) log.info(row); + } + + log.blank(); + log.success( + `Exported ${summary.userCount} user${summary.userCount === 1 ? "" : "s"} to ${summary.outputPath}`, + ); + + return NEXT_STEPS.MIGRATE_EXPORT(summary.transformerKey, relativeIfInside(summary.outputPath)); +} + +/** Shortens a path for display when it sits under the working directory. */ +function relativeIfInside(absolute: string): string { + const relative = path.relative(process.cwd(), absolute); + return relative.startsWith("..") ? absolute : relative; +} diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts new file mode 100644 index 000000000..3ab72ff78 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -0,0 +1,155 @@ +/** + * `clerk migrate export supabase` — read users straight out of `auth.users`. + * + * Ported from the standalone migration-tool's `src/export/supabase.ts`, on + * `Bun.sql` instead of `pg`. + * + * The database rather than the Admin API because **`encrypted_password` only + * exists here**. Supabase's API does not return password hashes, so an + * API-based export forces every user to reset their password; this one carries + * the bcrypt digests across. + */ + +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { reportExport, resolveOutputPath, writeExportOutput } from "./shared.ts"; +import { + promptDbUrl, + resolveDbUrl, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; + +/** + * `display_name` is coalesced into `first_name` here rather than in the + * transformer so a user who writes their own SQL sees the shape the + * transformer expects. + */ +const EXPORT_QUERY = ` + SELECT + id, + email, + email_confirmed_at, + encrypted_password, + phone, + phone_confirmed_at, + COALESCE( + raw_user_meta_data->>'display_name', + raw_user_meta_data->>'first_name', + raw_user_meta_data->>'name' + ) AS first_name, + raw_user_meta_data->>'last_name' AS last_name, + raw_user_meta_data, + raw_app_meta_data, + created_at + FROM auth.users + ORDER BY created_at +`; + +type SupabaseRow = Record & { + id?: unknown; + email?: string | null; + encrypted_password?: string | null; + raw_app_meta_data?: unknown; +}; + +/** Serializes values the JSON export cannot carry as-is. */ +function normalizeRow(row: SupabaseRow): Record { + const normalized: Record = {}; + + for (const [key, value] of Object.entries(row)) { + if (value === null || value === undefined) continue; + // Postgres returns timestamps as Date objects; the transformer parses + // strings, and JSON.stringify would otherwise bury the format difference. + normalized[key] = value instanceof Date ? value.toISOString() : value; + } + + return normalized; +} + +export async function fetchSupabaseUsers(client: DbClient): Promise { + return client.query(EXPORT_QUERY); +} + +export function buildSupabaseExport(rows: SupabaseRow[], dateTime: string) { + const users: Record[] = []; + const counts = { email: 0, emailConfirmed: 0, password: 0, phone: 0, firstName: 0, lastName: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + try { + users.push(normalizeRow(row)); + + if (row.email) counts.email++; + if (row.email_confirmed_at) counts.emailConfirmed++; + if (row.encrypted_password) counts.password++; + if (row.phone) counts.phone++; + if (row.first_name) counts.firstName++; + if (row.last_name) counts.lastName++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a confirmed email", count: counts.emailConfirmed }, + { label: "have a password hash", count: counts.password }, + { label: "have a phone number", count: counts.phone }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + ], + }; +} + +const SUPABASE_DB = { + platform: "supabase", + envVar: "SUPABASE_DB_URL", + prompt: "Supabase Postgres connection string", + hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", +} as const satisfies ResolveConfig; + +export async function exportSupabase(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, SUPABASE_DB); + + const destination = await resolveOutputPath("supabase", options.output); + + await withGutter("Exporting users from Supabase", async ({ setNextSteps }) => { + const dateTime = await startLogging(); + + const { value: rows } = await withInputRetry( + dbUrl, + async () => promptDbUrl(SUPABASE_DB), + async (connectionString) => + withSpinner("Reading auth.users...", async () => + withDbClient(connectionString, "supabase", fetchSupabaseUsers), + ), + ); + + const { users, coverage } = buildSupabaseExport(rows, dateTime); + const outputPath = writeExportOutput(users, destination); + + setNextSteps( + reportExport({ + platform: "supabase", + userCount: users.length, + outputPath, + coverage, + transformerKey: "supabase", + }), + ); + + if (users.length > 0) { + log.info( + "Password hashes are included — this is why the export reads the database rather than the Admin API.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts new file mode 100644 index 000000000..e61103bf3 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -0,0 +1,456 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog, useMigrateLogDir } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { + buildIdentityReport, + buildWorkOsExport, + exportWorkOs, + fetchAllWorkOsIdentities, + fetchAllWorkOsUsers, + fetchWorkOsIdentities, + fetchWorkOsPage, + mapWorkOsUserToExport, + resolveWithIdentities, + resolveWorkOsApiKey, + type WorkOsIdentity, +} from "./workos.ts"; + +/** A cwd with no `.env` files, so these tests exercise only the injected env. */ +const NO_ENV_FILES = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-no-env-")); + +const captured = useCaptureLog(); +useMigrateLogDir(); + +const API_KEY = "sk_test"; + +/** Colour is on or off depending on the runner, so rows are compared bare. */ +const stripAnsi = (value: string): string => value.replace(/\u001b\[[0-9;]*m/g, ""); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: string[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expworkos-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a leaked + // "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); + requests = []; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +const workosUser = (i: number, overrides: Record = {}) => ({ + id: `user_0${i}`, + email: `a${i}@x.dev`, + email_verified: true, + first_name: `Given${i}`, + last_name: `Family${i}`, + ...overrides, +}); + +/** + * Stubs one users page per entry in `pages`, chaining the cursor, plus an + * identities response per user id in `identities`. + */ +function stubWorkOs( + pages: Record[][], + identities: Record = {}, +) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = input.toString(); + requests.push(url); + + const identityMatch = /\/users\/([^/]+)\/identities/.exec(url); + if (identityMatch) { + const entry = identities[identityMatch[1] as string]; + if (entry === "fail") return new Response("nope", { status: 500 }); + return Response.json(entry ?? []); + } + + const after = new URL(url).searchParams.get("after"); + const index = after ? Number(after.replace("cursor", "")) : 0; + const isLast = index >= pages.length - 1; + return Response.json({ + data: pages[index] ?? [], + list_metadata: { after: isLast ? null : `cursor${index + 1}` }, + }); + }) as unknown as typeof fetch; +} + +describe("resolveWorkOsApiKey", () => { + test("prefers the flag", async () => { + expect(await resolveWorkOsApiKey({ apiKey: "sk_flag" }, NO_ENV_FILES, {})).toBe("sk_flag"); + }); + + test("falls back to the environment", async () => { + expect(await resolveWorkOsApiKey({}, NO_ENV_FILES, { WORKOS_API_KEY: "sk_env" })).toBe( + "sk_env", + ); + }); + + // Tests run non-TTY, the same signal an agent gives. + test("names the flag and the variable when neither supplied one", async () => { + await expect(resolveWorkOsApiKey({}, NO_ENV_FILES, {})).rejects.toThrow( + /Missing: --api-key \(or WORKOS_API_KEY\)\./, + ); + }); +}); + +describe("fetchWorkOsPage", () => { + test("asks for the documented page size", async () => { + stubWorkOs([[]]); + await fetchWorkOsPage(API_KEY); + expect(requests[0]).toContain("limit=100"); + }); + + test("explains a rejection instead of surfacing a raw status", async () => { + globalThis.fetch = (async () => + Response.json({ message: "Unauthorized" }, { status: 401 })) as unknown as typeof fetch; + + await expect(fetchWorkOsPage(API_KEY)).rejects.toThrow( + /WorkOS returned 401 listing users: Unauthorized/, + ); + }); + + // The usual cause is a publishable key, or a key from the other environment. + test("points at the key itself", async () => { + globalThis.fetch = (async () => new Response("{}", { status: 401 })) as unknown as typeof fetch; + await expect(fetchWorkOsPage(API_KEY)).rejects.toThrow(/secret key/); + }); +}); + +describe("fetchAllWorkOsUsers", () => { + test("follows the cursor until it comes back null", async () => { + stubWorkOs([ + Array.from({ length: 100 }, (_, i) => workosUser(i)), + Array.from({ length: 4 }, (_, i) => workosUser(100 + i)), + ]); + + const all = await fetchAllWorkOsUsers({ apiKey: API_KEY }); + + expect(all).toHaveLength(104); + expect(requests).toHaveLength(2); + expect(requests[1]).toContain("after=cursor1"); + }); + + // Cursor pagination has no record ceiling, so a full page that happens to be + // the last one must not read as "there is more". + test("stops on a full final page", async () => { + stubWorkOs([Array.from({ length: 100 }, (_, i) => workosUser(i))]); + expect(await fetchAllWorkOsUsers({ apiKey: API_KEY })).toHaveLength(100); + expect(requests).toHaveLength(1); + }); + + test("reuses a page already fetched rather than asking twice", async () => { + stubWorkOs([[workosUser(0)]]); + const firstPage = await fetchWorkOsPage(API_KEY); + requests = []; + + expect(await fetchAllWorkOsUsers({ apiKey: API_KEY, firstPage })).toHaveLength(1); + expect(requests).toHaveLength(0); + }); +}); + +describe("resolveWithIdentities", () => { + test("is on when the flag asked for it", async () => { + expect(await resolveWithIdentities({ withIdentities: true }, 10)).toBe(true); + }); + + // The fan-out is one request per user and nothing it returns can be + // imported, so it is never the default. + test("is off without the flag when there is nobody to ask", async () => { + expect(await resolveWithIdentities({}, 10)).toBe(false); + }); + + test("does not ask when there are no users to ask about", async () => { + const originalMode = getMode(); + setMode("human"); + try { + expect(await resolveWithIdentities({}, 0)).toBe(false); + } finally { + setMode(originalMode); + } + }); +}); + +describe("fetchAllWorkOsIdentities", () => { + test("collects each user's providers", async () => { + stubWorkOs([[]], { + user_00: [{ provider: "GoogleOAuth", idp_id: "g1", type: "OAuth" }], + user_01: [], + }); + + const { identities, failed } = await fetchAllWorkOsIdentities({ + apiKey: API_KEY, + users: [workosUser(0), workosUser(1)], + }); + + expect(identities.get("user_00")).toEqual([ + { provider: "GoogleOAuth", idp_id: "g1", type: "OAuth" }, + ]); + expect(identities.get("user_01")).toEqual([]); + expect(failed).toBe(0); + }); + + // "Lookup failed" and "has no providers" are different facts, and flattening + // the first into the second would put a wrong number in the report. + test("leaves a failed lookup absent rather than empty, and counts it", async () => { + stubWorkOs([[]], { user_00: "fail", user_01: [] }); + + const { identities, failed } = await fetchAllWorkOsIdentities({ + apiKey: API_KEY, + users: [workosUser(0), workosUser(1)], + }); + + expect(identities.has("user_00")).toBe(false); + expect(identities.get("user_01")).toEqual([]); + expect(failed).toBe(1); + }); +}); + +describe("buildIdentityReport", () => { + const rowsOf = (section: { rows: string[] }) => + section.rows.map((row) => stripAnsi(row).trimEnd()); + + test("ranks providers by use, and counts users with none", () => { + const section = buildIdentityReport( + [workosUser(0), workosUser(1), workosUser(2), workosUser(3)], + new Map([ + ["user_00", [{ provider: "GoogleOAuth" }]], + ["user_01", [{ provider: "GoogleOAuth" }, { provider: "MicrosoftOAuth" }]], + ["user_02", []], + ["user_03", []], + ]), + 0, + ); + + expect(section.title).toBe("OAuth providers"); + expect(rowsOf(section)).toEqual([ + " GoogleOAuth 2 users", + " MicrosoftOAuth 1 user", + " no OAuth provider 2 users", + ]); + }); + + // Counting a failed lookup as "no provider" would understate social sign-in. + test("reports unreadable lookups on their own row, with the caveat", () => { + const section = buildIdentityReport( + [workosUser(0), workosUser(1)], + new Map([["user_01", []]]), + 1, + ); + + const rows = rowsOf(section); + expect(rows).toContain(" not readable 1 user"); + expect(rows).toContain(" no OAuth provider 1 user"); + expect(rows.at(-1)).toContain("no `identities` field in the export, rather than an empty one"); + }); + + test("says nothing about unreadable lookups when there were none", () => { + const section = buildIdentityReport([workosUser(0)], new Map([["user_00", []]]), 0); + expect(rowsOf(section)).toEqual([" no OAuth provider 1 user"]); + }); +}); + +describe("mapWorkOsUserToExport", () => { + test("keeps the fields the workos transformer maps from", () => { + expect(mapWorkOsUserToExport(workosUser(0, { created_at: "2025-01-01" }))).toEqual({ + id: "user_00", + email: "a0@x.dev", + first_name: "Given0", + last_name: "Family0", + created_at: "2025-01-01", + email_verified: true, + }); + }); + + // Dropping a false flag would import an unconfirmed address as verified. + test("keeps email_verified when it is false", () => { + expect(mapWorkOsUserToExport(workosUser(0, { email_verified: false })).email_verified).toBe( + false, + ); + }); + + test("drops tenant fields the import has no use for", () => { + const mapped = mapWorkOsUserToExport( + workosUser(0, { + locale: "en-GB", + profile_picture_url: "https://x.dev/a.png", + last_sign_in_at: "2026-01-01", + updated_at: "2026-01-01", + external_id: "cust_1", + }), + ); + for (const noise of [ + "locale", + "profile_picture_url", + "last_sign_in_at", + "updated_at", + "external_id", + ]) { + expect(noise in mapped).toBe(false); + } + }); + + test("omits empty metadata", () => { + expect("metadata" in mapWorkOsUserToExport(workosUser(0, { metadata: {} }))).toBe(false); + expect(mapWorkOsUserToExport(workosUser(0, { metadata: { plan: "pro" } })).metadata).toEqual({ + plan: "pro", + }); + }); + + test("carries identities only when they were fetched", () => { + expect("identities" in mapWorkOsUserToExport(workosUser(0))).toBe(false); + expect(mapWorkOsUserToExport(workosUser(0), [{ provider: "GoogleOAuth" }]).identities).toEqual([ + { provider: "GoogleOAuth" }, + ]); + }); +}); + +describe("buildWorkOsExport", () => { + test("counts coverage and logs each user", () => { + const { users, coverage } = buildWorkOsExport( + [workosUser(0), workosUser(1, { first_name: undefined })], + "2026-01-01T00:00:00", + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + + const logged = fs.readdirSync(getLogDir()); + expect(logged[0]).toMatch(/^export-/); + }); + + // Always shown, always zero: seeing it before the import is the point. + test("reports the password row even though it can only ever be zero", () => { + const { coverage } = buildWorkOsExport([workosUser(0)], "2026-01-01T00:00:00"); + expect(coverage.at(-1)).toEqual({ + label: "have a password (WorkOS returns none)", + count: 0, + }); + }); + + // Providers get their own block: a coverage row means "N of M users have + // this field", and a provider count can exceed M. + test("keeps providers out of the coverage table", () => { + const { coverage } = buildWorkOsExport( + [workosUser(0)], + "2026-01-01T00:00:00", + new Map([["user_00", [{ provider: "GoogleOAuth" }]]]), + ); + expect(coverage.some((row) => row.label.toLowerCase().includes("oauth"))).toBe(false); + }); +}); + +/** The one file the export just wrote into `exports/`, whatever it stamped it. */ +function onlyExportFile(): string { + const entries = fs.readdirSync(path.join(workDir, "exports")); + expect(entries).toHaveLength(1); + return path.join(workDir, "exports", entries[0] as string); +} + +describe("exportWorkOs", () => { + test("writes the default path and reports coverage", async () => { + stubWorkOs([[workosUser(0)]]); + + await exportWorkOs({ apiKey: API_KEY }); + + // Stamped to the minute, so a second export does not overwrite the first. + expect(path.basename(onlyExportFile())).toMatch(/^workos-export-\d{8}-\d{4}\.json$/); + const written = JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as Record< + string, + unknown + >[]; + expect(written[0]?.id).toBe("user_00"); + expect(captured.err).toContain("Field coverage"); + }); + + test("names the command that consumes the file", async () => { + stubWorkOs([[workosUser(0)]]); + // The suggestion rides the gutter's Next steps block, which only renders + // in human mode. + const originalMode = getMode(); + setMode("human"); + try { + // --output answers the destination prompt, and --with-identities answers + // the providers question, so human mode stops on neither. + await exportWorkOs({ apiKey: API_KEY, output: "exports/mine.json", withIdentities: true }); + } finally { + setMode(originalMode); + } + expect(captured.err).toContain("migrate import --transformer workos --file exports/mine.json"); + }); + + test("--output controls the destination", async () => { + stubWorkOs([[workosUser(0)]]); + + await exportWorkOs({ apiKey: API_KEY, output: "tenant.json" }); + + expect(fs.existsSync(path.join(workDir, "tenant.json"))).toBe(true); + }); + + test("skips the per-user identity fan-out unless asked", async () => { + stubWorkOs([[workosUser(0), workosUser(1)]]); + + await exportWorkOs({ apiKey: API_KEY, output: "plain.json" }); + + expect(requests.filter((url) => url.includes("/identities"))).toHaveLength(0); + }); + + test("--with-identities records each user's providers in the file", async () => { + stubWorkOs([[workosUser(0)]], { user_00: [{ provider: "GoogleOAuth", idp_id: "g1" }] }); + + await exportWorkOs({ apiKey: API_KEY, output: "rich.json", withIdentities: true }); + + const written = JSON.parse(fs.readFileSync(path.join(workDir, "rich.json"), "utf-8")) as Record< + string, + unknown + >[]; + expect(written[0]?.identities).toEqual([{ provider: "GoogleOAuth", idp_id: "g1" }]); + }); + + // Finding this out after the import means nobody can sign in. + test("says plainly that no credentials are in the file", async () => { + stubWorkOs([[workosUser(0)]]); + await exportWorkOs({ apiKey: API_KEY, output: "warned.json" }); + expect(captured.err).toContain("does not return password hashes or TOTP secrets"); + }); +}); + +describe("fetchWorkOsIdentities", () => { + test("accepts the bare array the endpoint returns", async () => { + stubWorkOs([[]], { user_00: [{ provider: "GoogleOAuth" }] }); + expect(await fetchWorkOsIdentities(API_KEY, "user_00")).toEqual([{ provider: "GoogleOAuth" }]); + }); + + // A move to WorkOS's usual envelope must not read as "no providers". + test("accepts a { data } envelope too", async () => { + globalThis.fetch = (async () => + Response.json({ data: [{ provider: "AppleOAuth" }] })) as unknown as typeof fetch; + expect(await fetchWorkOsIdentities(API_KEY, "user_00")).toEqual([{ provider: "AppleOAuth" }]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts new file mode 100644 index 000000000..f8ccf5df6 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -0,0 +1,488 @@ +/** + * `clerk migrate export workos` — pull users out of a WorkOS tenant. + * + * Structurally the Auth0 case, and written against the same shape: two REST + * calls through `loggedFetch` rather than the `@workos-inc/node` SDK, so + * everything the command sends shows up under `--verbose`. See the header of + * `auth0.ts` and `.claude/rules/debug-logging.md`. + * + * **WorkOS is API-only, and no credential leaves it.** There is no + * bring-your-own-database option — apps mirror WorkOS users into their own + * store through webhooks, but that mirror is a derived copy holding no secrets, + * which is why this has no `--db-url` sibling. Password hashes are accepted on + * import and never returned; TOTP secrets come back on enrol only, never on + * list or get. So a WorkOS migration moves identities, and every password user + * signs in again by reset. The run says so rather than leaving it to be + * discovered when nobody can sign in. + */ + +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { confirm, password as passwordPrompt } from "../../../lib/prompts.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; +import { exportLogger, startLogging } from "../lib/logger.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; +import { createApiScheduler } from "../lib/scheduler.ts"; +import { + reportExport, + resolveOutputPath, + writeExportOutput, + type ExportSection, +} from "./shared.ts"; + +const API_BASE = "https://api.workos.com/user_management"; + +/** WorkOS caps `limit` at 100. */ +const PAGE_SIZE = 100; + +/** + * Pacing for the per-user identity fan-out. + * + * WorkOS allows 6,000 requests a minute, so this is nowhere near the ceiling — + * it is here so a 50,000-user tenant does not open 50,000 sockets at once. + */ +const IDENTITY_CONCURRENCY = 10; +const IDENTITY_RATE_PER_SECOND = 20; + +/** + * How often a non-interactive run says where it has got to. + * + * `withSpinner` hands a no-op `update` to anything that is not a TTY, so an + * agent exporting 50,000 users with `--with-identities` would otherwise see + * nothing at all for the ten minutes the fan-out takes. These two print + * through `log.info` instead, which a non-TTY does get. + */ +const IDENTITY_PROGRESS_EVERY = 500; +const USER_PROGRESS_EVERY_PAGES = 10; + +const NO_PROVIDER_LABEL = "no OAuth provider"; +const NOT_READABLE_LABEL = "not readable"; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; + +export type ExportWorkOsOptions = { + apiKey?: string; + withIdentities?: boolean; + output?: string; +}; + +export type WorkOsUser = Record & { id?: string }; + +export type WorkOsIdentity = { idp_id?: string; type?: string; provider?: string }; + +/** + * Resolves the API key: flag, then environment, then a prompt. + * + * One value rather than Auth0's three, so there is no "name everything that is + * missing" pass — there is only ever the one thing. + * + * @throws CliError in agent mode when nothing supplied it. + */ +export async function resolveWorkOsApiKey( + options: ExportWorkOsOptions, + cwd: string = process.cwd(), + env: Record = process.env, +): Promise { + const resolved = + options.apiKey ?? (await findMigrateEnvValue(["WORKOS_API_KEY"], cwd, env))?.value; + + if (resolved) return resolved.trim(); + + if (isAgent() || !isHuman()) { + throwUsageError( + "`clerk migrate export workos` needs a WorkOS API key and cannot prompt here.\n" + + "Missing: --api-key (or WORKOS_API_KEY).", + DOCS_URL, + undefined, + [ + { + command: "clerk migrate export workos --api-key sk_…", + description: "Export with an explicit API key", + }, + ], + ); + } + + log.info( + "WorkOS needs a secret API key, the one starting `sk_`. Find it in the WorkOS dashboard under API Keys.", + ); + + return promptWorkOsApiKey(); +} + +export async function promptWorkOsApiKey(): Promise { + const key = await passwordPrompt({ + message: "WorkOS secret API key (sk_…)", + validate: (value) => (value?.trim() ? undefined : "An API key is required"), + }); + return key.trim(); +} + +/** + * Whatever WorkOS put in an error body, in one string. + * + * Read as text and parsed from that, rather than `response.json()` with a text + * fallback: the failed parse disturbs the stream, so the fallback could never + * actually run. A non-JSON body — a proxy's HTML error page — is the case worth + * surfacing verbatim. + */ +async function describeFailure(response: Response): Promise { + const raw = (await response.text().catch(() => "")).trim(); + if (!raw) return "no detail returned"; + + try { + const body = JSON.parse(raw) as { + message?: string; + error_description?: string; + error?: string; + }; + return body.message ?? body.error_description ?? body.error ?? raw; + } catch { + return raw; + } +} + +export type WorkOsPage = { users: WorkOsUser[]; after?: string }; + +/** + * Fetches one page of users. + * + * Cursor pagination, so unlike Auth0's offset endpoint there is no record + * ceiling to warn about — `after` runs to the end of the tenant. + */ +export async function fetchWorkOsPage(apiKey: string, after?: string): Promise { + const url = new URL(`${API_BASE}/users`); + url.searchParams.set("limit", String(PAGE_SIZE)); + url.searchParams.set("order", "asc"); + if (after) url.searchParams.set("after", after); + + const response = await loggedFetch(url, { + tag: "workos", + method: "GET", + headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, + }); + + if (!response.ok) { + throw new CliError( + `WorkOS returned ${response.status} listing users: ${await describeFailure(response)}\n` + + "Check that the key is a secret key (`sk_…`) for the right environment, and that it has not been revoked.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + const body = (await response.json()) as { + data?: WorkOsUser[]; + list_metadata?: { after?: string | null }; + }; + + return { users: body.data ?? [], after: body.list_metadata?.after ?? undefined }; +} + +/** + * Pages through the tenant's users. + * + * @param firstPage - A page already fetched, so the request that proved the API + * key is not sent twice. + */ +export async function fetchAllWorkOsUsers(options: { + apiKey: string; + firstPage?: WorkOsPage; + spinner?: SpinnerControls; +}): Promise { + let page = options.firstPage ?? (await fetchWorkOsPage(options.apiKey)); + const all = [...page.users]; + options.spinner?.update(`Fetching users from WorkOS: ${all.length} so far...`); + + // Counted in pages rather than users: a short page would knock a + // `users % N` check off its multiple and silence every later one. + for (let pages = 1; page.after; pages++) { + page = await fetchWorkOsPage(options.apiKey, page.after); + all.push(...page.users); + options.spinner?.update(`Fetching users from WorkOS: ${all.length} so far...`); + if (!isHuman() && pages % USER_PROGRESS_EVERY_PAGES === 0) { + log.info(`Fetched ${all.length} users from WorkOS so far...`); + } + } + + return all; +} + +/** + * Whether to spend one request per user on OAuth providers. + * + * Off unless asked for, both times. WorkOS has no bulk identities endpoint, so + * this is the difference between ten requests and one per user — and nothing it + * returns can be imported, because `POST /v1/users` has no external-accounts + * field. It is a line in the coverage report, and a record kept in the file. + * + * Agent mode gets the flag's answer and no question: there is nobody to ask. + */ +export async function resolveWithIdentities( + options: ExportWorkOsOptions, + userCount: number, +): Promise { + if (options.withIdentities) return true; + if (userCount === 0 || isAgent() || !isHuman()) return false; + + return confirm({ + message: `Also fetch each user's OAuth providers? That is ${userCount} extra request${userCount === 1 ? "" : "s"}, and the result is report-only — Clerk's import cannot take external accounts.`, + default: false, + }); +} + +/** Fetches one user's OAuth identities. */ +export async function fetchWorkOsIdentities( + apiKey: string, + userId: string, +): Promise { + const response = await loggedFetch(new URL(`${API_BASE}/users/${userId}/identities`), { + tag: "workos", + method: "GET", + headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, + }); + + if (!response.ok) { + throw new CliError( + `WorkOS returned ${response.status} listing identities for ${userId}: ${await describeFailure(response)}`, + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + // The endpoint returns a bare array; the envelope is handled too so a future + // move to WorkOS's usual `{ data }` shape does not read as "no providers". + const body = (await response.json()) as WorkOsIdentity[] | { data?: WorkOsIdentity[] }; + return Array.isArray(body) ? body : (body.data ?? []); +} + +/** + * Fetches identities for every user. + * + * A user missing from the returned map is one whose lookup **failed**, which is + * not the same as one with no providers — so failures are counted and returned + * separately rather than flattened into an empty list. + */ +export async function fetchAllWorkOsIdentities(options: { + apiKey: string; + users: WorkOsUser[]; + spinner?: SpinnerControls; +}): Promise<{ identities: Map; failed: number }> { + const schedule = createApiScheduler(IDENTITY_CONCURRENCY, IDENTITY_RATE_PER_SECOND); + const total = options.users.length; + const identities = new Map(); + let failed = 0; + let done = 0; + + await Promise.all( + options.users.map(async (user) => + schedule(async () => { + const userId = String(user.id ?? ""); + try { + if (userId) identities.set(userId, await fetchWorkOsIdentities(options.apiKey, userId)); + } catch { + failed++; + } + done++; + options.spinner?.update(`Fetching OAuth providers: ${done}/${total}...`); + if (!isHuman() && done % IDENTITY_PROGRESS_EVERY === 0) { + log.info(`Fetched OAuth providers for ${done}/${total} users...`); + } + }), + ), + ); + + return { identities, failed }; +} + +/** + * The OAuth provider breakdown, as its own block under the coverage table. + * + * Kept out of coverage on purpose: a coverage row means "N of the M users have + * this field", and these rows do not. One user can hold two providers, so the + * counts can sum past the user count, and "not readable" is not a property of + * the user at all. Two kinds of row under one heading would make both harder + * to read. + */ +export function buildIdentityReport( + users: WorkOsUser[], + identities: Map, + failed: number, +): ExportSection { + const byProvider = new Map(); + let none = 0; + + for (const user of users) { + const found = identities.get(String(user.id ?? "")); + // Absent means the lookup failed; `failed` already counts it. + if (!found) continue; + if (found.length === 0) { + none++; + continue; + } + for (const identity of found) { + const provider = identity.provider ?? "unknown"; + byProvider.set(provider, (byProvider.get(provider) ?? 0) + 1); + } + } + + // Busiest provider first; alphabetical within a tie so two runs of the same + // tenant print the same order. + const entries = [...byProvider].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); + + const labels = [ + ...entries.map(([provider]) => provider), + NO_PROVIDER_LABEL, + ...(failed > 0 ? [NOT_READABLE_LABEL] : []), + ]; + const width = Math.max(...labels.map((label) => label.length)); + const row = (label: string, count: number) => + ` ${label.padEnd(width)} ${dim(`${count} user${count === 1 ? "" : "s"}`)}`; + + const rows = [ + ...entries.map(([provider, count]) => row(provider, count)), + row(NO_PROVIDER_LABEL, none), + ]; + + if (failed > 0) { + rows.push(row(NOT_READABLE_LABEL, failed)); + rows.push( + dim(" Those users have no `identities` field in the export, rather than an empty one."), + ); + } + + return { title: "OAuth providers", rows }; +} + +/** + * Keeps the fields the `workos` transformer maps from, plus `identities` when + * they were fetched. + * + * A copy rather than the raw record: WorkOS also returns `locale`, + * `profile_picture_url`, `last_sign_in_at` and `updated_at`, none of which + * `POST /v1/users` accepts. `identities` has no target field either and is + * dropped at validation, so it rides along purely as a record for whoever runs + * the migration. + */ +export function mapWorkOsUserToExport( + user: WorkOsUser, + identities?: WorkOsIdentity[], +): Record { + const exported: Record = {}; + + for (const field of ["id", "email", "first_name", "last_name", "created_at"] as const) { + if (user[field]) exported[field] = user[field]; + } + + // Meaningful when false: dropping it would import an address WorkOS never + // confirmed as a verified one. + if (user.email_verified !== undefined) exported.email_verified = user.email_verified; + + const metadata = user.metadata; + if (metadata && typeof metadata === "object" && Object.keys(metadata).length > 0) { + exported.metadata = metadata; + } + + if (identities && identities.length > 0) exported.identities = identities; + + return exported; +} + +export type WorkOsExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +export function buildWorkOsExport( + users: WorkOsUser[], + dateTime: string, + identities?: Map, +): WorkOsExportResult { + const exported: Record[] = []; + const counts = { email: 0, firstName: 0, lastName: 0, metadata: 0 }; + + for (const user of users) { + const userId = String(user.id ?? ""); + try { + const mapped = mapWorkOsUserToExport(user, identities?.get(userId)); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.first_name) counts.firstName++; + if (mapped.last_name) counts.lastName++; + if (mapped.metadata) counts.metadata++; + + exportLogger({ userId, status: "success" }, dateTime); + } catch (error) { + exportLogger({ userId, status: "error", error: (error as Error).message }, dateTime); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + { label: "have metadata", count: counts.metadata }, + // Always present, always zero. WorkOS returns no digest for anyone, and + // seeing that before the import is the whole reason the row is here. + { label: "have a password (WorkOS returns none)", count: 0 }, + ], + }; +} + +export async function exportWorkOs(options: ExportWorkOsOptions): Promise { + const resolved = await resolveWorkOsApiKey(options); + + const destination = await resolveOutputPath("workos", options.output); + + await withGutter("Exporting users from WorkOS", async ({ setNextSteps }) => { + const dateTime = await startLogging(); + + // Only WorkOS can say whether the key is live, for the right environment, + // and not revoked — so a rejected key is asked for again here. The page it + // fetches is kept and reused, so proving the key costs no extra request. + const { value: firstPage, input: apiKey } = await withInputRetry( + resolved, + async () => promptWorkOsApiKey(), + async (candidate) => + withSpinner("Authenticating with WorkOS...", async () => fetchWorkOsPage(candidate)), + ); + + const users = await withSpinner("Fetching users from WorkOS...", async (spinner) => + fetchAllWorkOsUsers({ apiKey, firstPage, spinner }), + ); + + const providers = (await resolveWithIdentities(options, users.length)) + ? await withSpinner("Fetching OAuth providers...", async (spinner) => + fetchAllWorkOsIdentities({ apiKey, users, spinner }), + ) + : undefined; + + const { users: exported, coverage } = buildWorkOsExport(users, dateTime, providers?.identities); + const outputPath = writeExportOutput(exported, destination); + + setNextSteps( + reportExport({ + platform: "workos", + userCount: exported.length, + outputPath, + coverage, + sections: providers + ? [buildIdentityReport(users, providers.identities, providers.failed)] + : [], + transformerKey: "workos", + }), + ); + + if (exported.length > 0) { + log.warn( + "WorkOS does not return password hashes or TOTP secrets, and there is no export that does. Imported users who signed in with a password must reset it on their first Clerk sign-in, and anyone using an authenticator app has to re-enrol. Users on SSO or social sign-in are unaffected.", + ); + log.info(dim(`See ${DOCS_URL}`)); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts new file mode 100644 index 000000000..68ed77a0e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -0,0 +1,335 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { BapiError } from "../../lib/errors.ts"; +import { + buildCreateUserBody, + importUsers, + normalizeErrorMessage, + readRetryAfter, + splitIdentifiers, +} from "./import-users.ts"; +import { getLogFilePath } from "./lib/logger.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import type { User } from "./types.ts"; + +const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurrencyLimit: 8 }; +const DATE_TIME = "2026-01-01T00:00:00"; + +const user = (overrides: Partial = {}): User => + ({ userId: "u1", email: "a@x.dev", ...overrides }) as User; + +describe("splitIdentifiers", () => { + test("promotes the first verified email and phone to primary", () => { + const result = splitIdentifiers( + user({ email: ["a@x.dev", "b@x.dev"], phone: ["+15555550100", "+15555550101"] }), + ); + expect(result.primaryEmail).toBe("a@x.dev"); + expect(result.additionalEmails).toEqual(["b@x.dev"]); + expect(result.primaryPhone).toBe("+15555550100"); + expect(result.additionalPhones).toEqual(["+15555550101"]); + }); + + test("merges the email and emailAddresses fields, deduping", () => { + const result = splitIdentifiers( + user({ email: "a@x.dev", emailAddresses: ["a@x.dev", "b@x.dev"] }), + ); + expect(result.primaryEmail).toBe("a@x.dev"); + expect(result.additionalEmails).toEqual(["b@x.dev"]); + }); + + test("drops an unverified identifier that is already verified", () => { + const result = splitIdentifiers( + user({ email: ["a@x.dev"], unverifiedEmailAddresses: ["a@x.dev", "c@x.dev"] }), + ); + expect(result.unverifiedEmails).toEqual(["c@x.dev"]); + }); + + test("copes with a user identified only by username", () => { + const result = splitIdentifiers({ userId: "u1", username: "alice" } as User); + expect(result.primaryEmail).toBeUndefined(); + expect(result.additionalEmails).toEqual([]); + }); +}); + +describe("buildCreateUserBody", () => { + test("maps the schema onto BAPI's snake_case body", () => { + const target = user({ + firstName: "Alice", + lastName: "Smith", + username: "alice", + createdAt: "2024-01-01T00:00:00.000Z", + publicMetadata: { plan: "pro" }, + createOrganizationsLimit: 3, + banned: true, + }); + const body = buildCreateUserBody(target, splitIdentifiers(target), true); + + expect(body).toMatchObject({ + external_id: "u1", + email_address: ["a@x.dev"], + first_name: "Alice", + last_name: "Smith", + username: "alice", + created_at: "2024-01-01T00:00:00.000Z", + public_metadata: { plan: "pro" }, + create_organizations_limit: 3, + banned: true, + }); + }); + + test("sends only the primary identifier; the rest are attached separately", () => { + const target = user({ email: ["a@x.dev", "b@x.dev"] }); + expect(buildCreateUserBody(target, splitIdentifiers(target), true).email_address).toEqual([ + "a@x.dev", + ]); + }); + + test("omits fields the source platform never recorded", () => { + const body = buildCreateUserBody(user(), splitIdentifiers(user()), true); + expect("first_name" in body).toBe(false); + expect("banned" in body).toBe(false); + expect("created_at" in body).toBe(false); + }); + + test("sends the password digest and hasher together", () => { + const target = user({ password: "digest", passwordHasher: "bcrypt" }); + const body = buildCreateUserBody(target, splitIdentifiers(target), true); + expect(body).toMatchObject({ password_digest: "digest", password_hasher: "bcrypt" }); + expect("skip_password_requirement" in body).toBe(false); + }); + + test.each([ + [true, true], + [false, false], + ])("skipPasswordRequirement=%p on a passwordless user -> flag present: %p", (skip, present) => { + const body = buildCreateUserBody(user(), splitIdentifiers(user()), skip); + expect("skip_password_requirement" in body).toBe(present); + }); +}); + +describe("readRetryAfter", () => { + const withHeader = (value: string) => + new BapiError(429, "{}", new Headers({ "retry-after": value })); + + test.each([ + ["12", 12], + ["0", undefined], + ["soon", undefined], + ])("Retry-After: %s -> %p", (header, expected) => { + expect(readRetryAfter(withHeader(header))).toBe(expected as number | undefined); + }); + + test("falls back to the error body's retryAfter meta", () => { + const error = new BapiError( + 429, + JSON.stringify({ + errors: [{ code: "rate_limit", message: "slow down", meta: { retryAfter: 7 } }], + }), + new Headers(), + ); + expect(readRetryAfter(error)).toBe(7); + }); + + test("returns undefined when neither source carries a value", () => { + expect(readRetryAfter(new BapiError(429, "{}", new Headers()))).toBeUndefined(); + }); +}); + +describe("normalizeErrorMessage", () => { + test("sorts field arrays so equivalent errors group together", () => { + const a = normalizeErrorMessage('["last_name" "first_name"] data does not match'); + const b = normalizeErrorMessage('["first_name" "last_name"] data does not match'); + expect(a).toBe(b); + expect(a).toBe('["first_name" "last_name"] data does not match'); + }); + + test("leaves messages without field arrays untouched", () => { + expect(normalizeErrorMessage("that email is taken")).toBe("that email is taken"); + }); +}); + +describe("importUsers", () => { + let workDir: string; + let originalCwd: string; + let originalFetch: typeof globalThis.fetch; + let requests: { method: string; url: string; body: unknown }[]; + + beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-import-")); + process.chdir(workDir); + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + }); + + beforeEach(() => { + requests = []; + fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + /** Installs a fetch that records every request and replies per `respond`. */ + function stub(respond: (url: string, attempt: number) => Response): void { + const attempts = new Map(); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + const attempt = (attempts.get(url) ?? 0) + 1; + attempts.set(url, attempt); + return respond(url, attempt); + }) as typeof fetch; + } + + const ok = (id: string) => new Response(JSON.stringify({ id }), { status: 200 }); + + const clerkError = (status: number, message: string, headers?: Record) => + new Response(JSON.stringify({ errors: [{ code: "err", message, long_message: message }] }), { + status, + headers, + }); + + const logEntries = () => + fs + .readFileSync(getLogFilePath("import", DATE_TIME), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + + test("creates each user and reports them as successful", async () => { + stub(() => ok("user_created")); + + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ totalProcessed: 2, successful: 2, failed: 0 }); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + expect(logEntries().filter((e) => e.status === "success")).toHaveLength(2); + }); + + test("attaches additional and unverified identifiers after the user exists", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [ + user({ + email: ["a@x.dev", "b@x.dev"], + unverifiedEmailAddresses: ["c@x.dev"], + phone: ["+15555550100", "+15555550101"], + }), + ], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + const emails = requests.filter((r) => r.url.endsWith("/v1/email_addresses")); + expect(emails.map((r) => r.body)).toEqual([ + { user_id: "user_created", email_address: "b@x.dev", primary: false, verified: true }, + { user_id: "user_created", email_address: "c@x.dev", primary: false, verified: false }, + ]); + expect(requests.filter((r) => r.url.endsWith("/v1/phone_numbers"))).toHaveLength(1); + }); + + test("logs a failed additional identifier without failing the user", async () => { + stub((url) => + url.endsWith("/v1/email_addresses") + ? clerkError(422, "that email is taken") + : ok("user_created"), + ); + + const summary = await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + expect(logEntries().some((e) => e.status === "additional_email_error")).toBe(true); + }); + + test("records a failed user and keeps going", async () => { + stub((_url, attempt) => + attempt === 1 ? clerkError(422, "that email is taken") : ok("user_ok"), + ); + + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary.successful + summary.failed).toBe(2); + expect(summary.failed).toBe(1); + expect([...summary.errorBreakdown.values()]).toEqual([1]); + expect(logEntries().some((e) => e.status === "error" && e.code === "422")).toBe(true); + }); + + test("retries a 429 after the interval the server asked for", async () => { + stub((_url, attempt) => + attempt === 1 ? clerkError(429, "slow down", { "retry-after": "1" }) : ok("user_ok"), + ); + + const started = performance.now(); + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + expect(performance.now() - started).toBeGreaterThanOrEqual(900); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + expect(logEntries().some((e) => e.status === "429_retry")).toBe(true); + }); + + test("gives up after the retry ceiling and records the user as failed", async () => { + stub(() => clerkError(429, "slow down", { "retry-after": "1" })); + + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + }); + + expect(summary).toMatchObject({ successful: 0, failed: 1 }); + // One initial attempt plus MAX_RETRIES retries. + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(6); + expect(logEntries().some((e) => e.code === "429")).toBe(true); + }, 20_000); + + test("carries the validation failure count into the summary", async () => { + stub(() => ok("user_ok")); + + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + dateTime: DATE_TIME, + validationFailed: 4, + }); + + expect(summary.validationFailed).toBe(4); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts new file mode 100644 index 000000000..4676a7565 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -0,0 +1,350 @@ +/** + * Creates users in Clerk from a validated batch. + * + * Ported from the standalone migration-tool's `src/migrate/import-users.ts`, + * rewritten onto `bapiRequest` instead of `@clerk/backend`. Two things fall out + * of that move: + * + * - The request body is BAPI's snake_case shape directly, so `created_at` and + * `legal_accepted_at` stay RFC3339 strings rather than round-tripping + * through `Date`. + * - `banned`, `delete_self_enabled` and the organization limits are all + * accepted by `POST /v1/users`, so the follow-up `updateUser`/`banUser` + * calls the SDK version needed are gone. + * + * Run state is local to {@link importUsers} rather than module-level, so two + * runs in one process (or one test file) cannot see each other's counters. + */ + +import { bapiRequest } from "../../lib/bapi.ts"; +import { BapiError } from "../../lib/errors.ts"; +import type { SpinnerControls } from "../../lib/spinner.ts"; +import { errorLogger, importLogger } from "./lib/logger.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; +import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; +import type { ImportSummary, User } from "./types.ts"; + +// Re-exported for the tests and callers that grew up against this module. +export { readRetryAfter } from "./lib/retry.ts"; + +/** + * Groups error messages that differ only in field ordering, so the summary + * reports "12 users: [\"first_name\" \"last_name\"] ..." once instead of twice. + */ +export function normalizeErrorMessage(errorMessage: string): string { + let normalized = ""; + let lastCopiedIndex = 0; + let arrayStartIndex = -1; + + for (let i = 0; i < errorMessage.length; i++) { + const char = errorMessage[i]; + + if (arrayStartIndex === -1) { + if (char === "[") arrayStartIndex = i; + continue; + } + if (char !== "]") continue; + + normalized += errorMessage.slice(lastCopiedIndex, arrayStartIndex); + normalized += normalizeFieldArray(errorMessage.slice(arrayStartIndex + 1, i)); + lastCopiedIndex = i + 1; + arrayStartIndex = -1; + } + + return normalized + errorMessage.slice(lastCopiedIndex); +} + +function normalizeFieldArray(fields: string): string { + const fieldNames: string[] = []; + let current = ""; + + for (const char of fields) { + if (char === '"' || char === "'" || char.trim() === "") { + if (current.length > 0) { + fieldNames.push(current); + current = ""; + } + continue; + } + current += char; + } + if (current.length > 0) fieldNames.push(current); + + fieldNames.sort(); + return `[${fieldNames.map((name) => `"${name}"`).join(" ")}]`; +} + +function toArray(value: string | string[] | undefined): string[] { + if (!value) return []; + return Array.isArray(value) ? value : [value]; +} + +function dedupe(values: string[]): string[] { + const seen: string[] = []; + for (const value of values) { + if (value && !seen.includes(value)) seen.push(value); + } + return seen; +} + +type Identifiers = { + primaryEmail: string | undefined; + additionalEmails: string[]; + unverifiedEmails: string[]; + primaryPhone: string | undefined; + additionalPhones: string[]; + unverifiedPhones: string[]; +}; + +/** + * Splits a user's identifiers into the one that goes on `POST /v1/users` and + * the rest, which are attached afterwards. + */ +export function splitIdentifiers(user: User): Identifiers { + const verifiedEmails = dedupe([...toArray(user.email), ...toArray(user.emailAddresses)]); + const verifiedPhones = dedupe([...toArray(user.phone), ...toArray(user.phoneNumbers)]); + + return { + primaryEmail: verifiedEmails[0], + additionalEmails: verifiedEmails.slice(1), + unverifiedEmails: dedupe( + toArray(user.unverifiedEmailAddresses).filter((email) => !verifiedEmails.includes(email)), + ), + primaryPhone: verifiedPhones[0], + additionalPhones: verifiedPhones.slice(1), + unverifiedPhones: dedupe( + toArray(user.unverifiedPhoneNumbers).filter((phone) => !verifiedPhones.includes(phone)), + ), + }; +} + +/** + * Builds the `POST /v1/users` request body. + * + * Optional fields are omitted rather than sent as null so Clerk applies its own + * defaults for anything the source platform did not record. + */ +export function buildCreateUserBody( + user: User, + identifiers: Identifiers, + skipPasswordRequirement: boolean, +): Record { + const body: Record = { external_id: user.userId }; + + if (identifiers.primaryEmail) body.email_address = [identifiers.primaryEmail]; + if (identifiers.primaryPhone) body.phone_number = [identifiers.primaryPhone]; + if (user.firstName) body.first_name = user.firstName; + if (user.lastName) body.last_name = user.lastName; + if (user.username) body.username = user.username; + if (user.totpSecret) body.totp_secret = user.totpSecret; + if (user.backupCodes) body.backup_codes = user.backupCodes; + if (user.unsafeMetadata) body.unsafe_metadata = user.unsafeMetadata; + if (user.privateMetadata) body.private_metadata = user.privateMetadata; + if (user.publicMetadata) body.public_metadata = user.publicMetadata; + if (user.createdAt) body.created_at = user.createdAt; + if (user.legalAcceptedAt) body.legal_accepted_at = user.legalAcceptedAt; + if (user.skipLegalChecks !== undefined) body.skip_legal_checks = user.skipLegalChecks; + if (user.skipPasswordChecks !== undefined) body.skip_password_checks = user.skipPasswordChecks; + if (user.banned !== undefined) body.banned = user.banned; + if (user.bypassClientTrust !== undefined) body.bypass_client_trust = user.bypassClientTrust; + if (user.deleteSelfEnabled !== undefined) body.delete_self_enabled = user.deleteSelfEnabled; + if (user.createOrganizationEnabled !== undefined) { + body.create_organization_enabled = user.createOrganizationEnabled; + } + if (user.createOrganizationsLimit !== undefined) { + body.create_organizations_limit = user.createOrganizationsLimit; + } + + if (user.password && user.passwordHasher) { + body.password_digest = user.password; + body.password_hasher = user.passwordHasher; + } else if (skipPasswordRequirement) { + body.skip_password_requirement = true; + } + // Without a password and without skipPasswordRequirement, Clerk rejects the + // user — which is exactly what --require-password is asking for. + + return body; +} + +type CreateContext = { + secretKey: string; + schedule: ApiScheduler; + dateTime: string; +}; + +/** Attaches one extra identifier, logging (but not rethrowing) any failure. */ +async function attachIdentifier( + ctx: CreateContext, + userId: string, + clerkUserId: string, + kind: "email" | "phone", + value: string, + verified: boolean, +): Promise { + const path = kind === "email" ? "/v1/email_addresses" : "/v1/phone_numbers"; + const body = + kind === "email" + ? { user_id: clerkUserId, email_address: value, primary: false, verified } + : { user_id: clerkUserId, phone_number: value, primary: false, verified }; + + try { + await ctx.schedule(async () => + bapiRequest({ + method: "POST", + path, + secretKey: ctx.secretKey, + body: JSON.stringify(body), + }), + ); + } catch (error) { + const label = `${verified ? "additional" : "unverified"} ${kind} ${value}`; + errorLogger( + { + userId, + status: `additional_${kind}_error`, + errors: [ + { + code: `additional_${kind}_failed`, + message: `Failed to add ${label}`, + longMessage: `Failed to add ${label}: ${(error as Error).message}`, + }, + ], + }, + ctx.dateTime, + ); + } +} + +/** Creates one user, then attaches any additional identifiers it carries. */ +async function createUser( + ctx: CreateContext, + user: User, + skipPasswordRequirement: boolean, +): Promise { + const identifiers = splitIdentifiers(user); + + const response = await ctx.schedule(async () => + bapiRequest({ + method: "POST", + path: "/v1/users", + secretKey: ctx.secretKey, + body: JSON.stringify(buildCreateUserBody(user, identifiers, skipPasswordRequirement)), + }), + ); + + const clerkUserId = (response.body as { id?: string })?.id ?? ""; + + // Extra identifiers are best-effort: a duplicate secondary email should not + // undo a user who was otherwise imported successfully. + await Promise.all([ + ...identifiers.additionalEmails.map(async (email) => + attachIdentifier(ctx, user.userId, clerkUserId, "email", email, true), + ), + ...identifiers.unverifiedEmails.map(async (email) => + attachIdentifier(ctx, user.userId, clerkUserId, "email", email, false), + ), + ...identifiers.additionalPhones.map(async (phone) => + attachIdentifier(ctx, user.userId, clerkUserId, "phone", phone, true), + ), + ...identifiers.unverifiedPhones.map(async (phone) => + attachIdentifier(ctx, user.userId, clerkUserId, "phone", phone, false), + ), + ]); + + return clerkUserId; +} + +export type ImportUsersOptions = { + users: User[]; + secretKey: string; + limits: ResolvedLimits; + dateTime: string; + /** Allow users that carry no password. */ + skipPasswordRequirement?: boolean; + /** Carried into the summary so the report covers the whole file. */ + validationFailed?: number; + spinner?: SpinnerControls; +}; + +/** + * Imports every user, concurrently and within the instance's rate limit. + * + * A failed user is recorded and the run continues; a 429 backs off (honouring + * `Retry-After`) and retries up to {@link MAX_RETRIES} times. + */ +export async function importUsers(options: ImportUsersOptions): Promise { + const { + users, + secretKey, + limits, + dateTime, + skipPasswordRequirement = true, + validationFailed = 0, + spinner, + } = options; + + const total = users.length; + const errorBreakdown = new Map(); + let processed = 0; + let successful = 0; + let failed = 0; + + const ctx: CreateContext = { + secretKey, + dateTime, + schedule: createApiScheduler(limits.concurrencyLimit, limits.rateLimit), + }; + + const progress = () => + spinner?.update( + `Importing users: [${processed}/${total}] (${successful} succeeded, ${failed} failed)...`, + ); + + const recordFailure = (userId: string, message: string, code: string) => { + failed++; + processed++; + const normalized = normalizeErrorMessage(message); + errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); + importLogger({ userId, status: "error", error: message, code }, dateTime); + progress(); + }; + + const processUser = async (user: User): Promise => { + try { + const clerkUserId = await retryOn429( + async () => createUser(ctx, user, skipPasswordRequirement), + { + onRetry: ({ message }) => + errorLogger( + { + userId: user.userId, + status: "429_retry", + errors: [{ code: "rate_limit_retry", message, longMessage: message }], + }, + dateTime, + ), + }, + ); + successful++; + processed++; + importLogger({ userId: user.userId, status: "success", clerkUserId }, dateTime); + progress(); + } catch (error) { + if (error instanceof RateLimitExceededError) { + recordFailure(user.userId, error.message, "429"); + return; + } + + const apiError = error as BapiError; + const message = apiError.longMessage ?? apiError.message ?? "Unknown error"; + recordFailure(user.userId, message, String(apiError.status ?? "unknown")); + } + }; + + progress(); + await Promise.all(users.map(async (user) => processUser(user))); + + return { totalProcessed: total, successful, failed, validationFailed, errorBreakdown }; +} diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts new file mode 100644 index 000000000..81c636f72 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -0,0 +1,241 @@ +import { describe, expect, test } from "bun:test"; +import { createProgram } from "../../cli-program.ts"; +import { exportPlatformKeys } from "./export/registry.ts"; +import { isAssumeYes, setAssumeYes } from "./lib/assume-yes.ts"; +import { transformerKeys } from "./transformers/registry.ts"; + +function findCommand(names: string[]) { + let current = createProgram().commands.find((cmd) => cmd.name() === names[0]); + for (const name of names.slice(1)) { + current = current?.commands.find((cmd) => cmd.name() === name); + } + return current; +} + +describe("registerMigrate", () => { + test("registers migrate as a top-level command group", () => { + const migrate = findCommand(["migrate"]); + expect(migrate).toBeDefined(); + expect(migrate?.description()).toContain("Migrate users"); + }); + + test("registers the import subcommand", () => { + expect(findCommand(["migrate", "import"])).toBeDefined(); + }); + + // The direction is never implied: `import` and `export` are siblings, so a + // default would make one of them the meaning of the bare group name. + test("leaves migrate with no default subcommand", () => { + const migrate = findCommand(["migrate"]) as unknown as { _defaultCommandName?: string }; + expect(migrate._defaultCommandName).toBeFalsy(); + }); + + test.each([ + "--transformer", + "--file", + "--resume-after", + "--require-password", + "--skip-unsupported-providers", + "--firebase-signer-key", + "--firebase-salt-separator", + "--firebase-rounds", + "--firebase-mem-cost", + "--yes", + "--secret-key", + "--app", + "--instance", + ])("migrate import accepts %s", (flag) => { + const flags = findCommand(["migrate", "import"])?.options.map((option) => option.long); + expect(flags).toContain(flag); + }); + + test.each([[["transformers"]], [["transformers", "list"]]])("registers migrate %p", (names) => { + expect(findCommand(["migrate", ...names])).toBeDefined(); + }); + + test.each([ + [["export"]], + [["export", "clerk"]], + [["export", "auth0"]], + [["export", "supabase"]], + [["export", "authjs"]], + [["export", "betterauth"]], + [["export", "firebase"]], + ])("registers migrate %p", (names) => { + expect(findCommand(["migrate", ...names])).toBeDefined(); + }); + + // Bare `migrate export` runs the picker rather than defaulting to a + // platform, so nobody exports from the wrong place by pressing enter. + test("leaves export with no default subcommand", () => { + const group = findCommand(["migrate", "export"]) as unknown as { + _defaultCommandName?: string; + }; + expect(group._defaultCommandName).toBeFalsy(); + }); + + test("registers an export subcommand per registered platform", () => { + const registered = findCommand(["migrate", "export"])?.commands.map((c) => c.name()); + for (const key of exportPlatformKeys()) expect(registered).toContain(key); + }); + + test.each(["--output", "--secret-key", "--app", "--instance"])( + "export clerk accepts %s", + (flag) => { + expect(findCommand(["migrate", "export", "clerk"])?.options.map((o) => o.long)).toContain( + flag, + ); + }, + ); + + test.each(["--domain", "--client-id", "--client-secret", "--output"])( + "export auth0 accepts %s", + (flag) => { + expect(findCommand(["migrate", "export", "auth0"])?.options.map((o) => o.long)).toContain( + flag, + ); + }, + ); + + test.each(["supabase", "authjs", "betterauth"])("export %s accepts --db-url", (platform) => { + expect(findCommand(["migrate", "export", platform])?.options.map((o) => o.long)).toContain( + "--db-url", + ); + }); + + test("export firebase accepts --service-account", () => { + expect(findCommand(["migrate", "export", "firebase"])?.options.map((o) => o.long)).toContain( + "--service-account", + ); + }); + + test("documents the default output location in help", () => { + expect(findCommand(["migrate", "export", "clerk"])?.description()).toContain( + "./exports/clerk-export-.json", + ); + expect(findCommand(["migrate", "export", "auth0"])?.description()).toContain( + "./exports/auth0-export-.json", + ); + }); + + test("makes list the default transformers subcommand", () => { + const group = findCommand(["migrate", "transformers"]) as unknown as { + _defaultCommandName?: string; + }; + expect(group._defaultCommandName).toBe("list"); + }); + + test.each(["--json", "--transformer-file"])("transformers list accepts %s", (flag) => { + expect(findCommand(["migrate", "transformers", "list"])?.options.map((o) => o.long)).toContain( + flag, + ); + }); + + test("migrate import accepts --transformer-file", () => { + expect(findCommand(["migrate", "import"])?.options.map((o) => o.long)).toContain( + "--transformer-file", + ); + }); + + // Flat rather than under a noun group: it is the one command in this tree + // that destroys data in Clerk. + test("registers delete as a direct subcommand of migrate", () => { + expect(findCommand(["migrate", "delete"])).toBeDefined(); + expect(findCommand(["migrate", "delete"])?.description()).toContain("last migration"); + }); + + test.each(["--yes", "--secret-key", "--app", "--instance"])( + "migrate delete accepts %s", + (flag) => { + expect(findCommand(["migrate", "delete"])?.options.map((o) => o.long)).toContain(flag); + }, + ); + + test.each([[["logs"]], [["logs", "list"]], [["logs", "clean"]], [["logs", "convert"]]])( + "registers migrate %p", + (names) => { + expect(findCommand(["migrate", ...names])).toBeDefined(); + }, + ); + + // Listing is read-only, so it is safe as the default for a bare + // `clerk migrate logs`. + test("makes list the default logs subcommand", () => { + const logs = findCommand(["migrate", "logs"]) as unknown as { _defaultCommandName?: string }; + expect(logs._defaultCommandName).toBe("list"); + }); + + test.each([ + [["logs", "list"], "--json"], + [["logs", "clean"], "--yes"], + [["logs", "convert"], "--all"], + ])("%s accepts %s", (names, flag) => { + expect(findCommand(["migrate", ...names])?.options.map((option) => option.long)).toContain( + flag, + ); + }); + + test("logs convert takes variadic file positionals", () => { + const args = findCommand(["migrate", "logs", "convert"])?.registeredArguments; + expect(args?.[0]?.variadic).toBe(true); + expect(args?.[0]?.required).toBe(false); + }); + + test("constrains --transformer to the registered transformers, for validation and completion", () => { + const option = findCommand(["migrate", "import"])?.options.find( + (o) => o.long === "--transformer", + ); + // Tracks the registry so adding a platform needs no edit here. + expect(option?.argChoices).toEqual(transformerKeys()); + }); + + test.each([ + ["-t", "--transformer"], + ["-f", "--file"], + ["-r", "--resume-after"], + ["-y", "--yes"], + ])("exposes %s as the short form of %s", (short, long) => { + const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === long); + expect(option?.short).toBe(short); + }); + + // Every export takes `-y`: it is what turns the credential-retry loop off, + // and the loop is on every one of them. + test.each(exportPlatformKeys())("migrate export %s accepts --yes", (platform) => { + const flags = findCommand(["migrate", "export", platform])?.options.map((o) => o.long); + expect(flags).toContain("--yes"); + }); +}); + +/** + * The hook is the only link between the parsed flag and the two places that + * read it, three layers down. If it stopped firing — a Commander upgrade that + * dropped hook inheritance, an action registered outside the group — both + * behaviours would silently revert and every unit test around them would still + * pass, because they set the flag directly. + */ +describe("the migrate group's -y hook", () => { + async function parse(argv: string[]) { + const program = createProgram(); + // `exitOverride` so a usage error inside the action throws here instead of + // taking the test runner down with it; the hook has already run by then. + program.exitOverride(); + try { + await program.parseAsync(["node", "clerk", ...argv]); + } catch { + // The action is allowed to fail — only the hook's effect is under test. + } + return isAssumeYes(); + } + + test("records -y on an export", async () => { + expect(await parse(["migrate", "export", "supabase", "-y", "--db-url", "./none.sqlite"])).toBe( + true, + ); + }); + + test("records its absence, so a previous run cannot leak into this one", async () => { + setAssumeYes(true); + expect(await parse(["migrate", "export", "supabase", "--db-url", "./none.sqlite"])).toBe(false); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts new file mode 100644 index 000000000..e49dcd3ec --- /dev/null +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -0,0 +1,172 @@ +import { createOption } from "@commander-js/extra-typings"; +import type { Program } from "../../cli-program.ts"; +import { parseIntegerOption } from "../../lib/option-parsers.ts"; +import { deleteMigration } from "./delete.ts"; +import { setAssumeYes } from "./lib/assume-yes.ts"; +import { registerMigrateExport } from "./export/index.ts"; +import { registerMigrateLogs } from "./logs/index.ts"; +import { registerMigrateSettings } from "./settings/index.ts"; +import { run } from "./run.ts"; +import { list as transformersList } from "./transformers/list.ts"; +import { transformerKeys } from "./transformers/registry.ts"; + +const migrate = { run, delete: deleteMigration, transformersList }; + +export function registerMigrate(program: Program): void { + const migrateCommand = program + .command("migrate") + .description("Migrate users into Clerk from another auth provider or another Clerk instance") + .setExamples([ + { command: "clerk migrate import", description: "Walk through an import interactively" }, + { + command: "clerk migrate import -y --transformer clerk --file users.json", + description: "Import users from a Clerk export", + }, + { + command: "clerk migrate import -y -t supabase -f users.json --skip-unsupported-providers", + description: "Skip Supabase users whose provider is not enabled", + }, + { + command: "clerk migrate export supabase", + description: "Export users from Supabase, ready to import", + }, + { command: "clerk migrate settings", description: "Show what a run here would pick up" }, + { + command: "clerk migrate settings set firebase-signer-key abc123", + description: "Save a credential to .env.clerk-migrate", + }, + { command: "clerk migrate logs", description: "List the local migration logs" }, + { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { command: "clerk migrate delete", description: "Undo the last migration" }, + ]); + + // `-y` is read three layers down — by the log-directory question and by the + // credential-retry loop — so it is resolved once here rather than threaded + // through every export handler. Hooks are inherited, so this fires for every + // subcommand under `migrate`; one that declares no `-y` resolves to false. + migrateCommand.hook("preAction", (_thisCommand, actionCommand) => { + setAssumeYes(Boolean(actionCommand.opts().yes)); + }); + + // Named, not `isDefault`. `import` and `export` are the two directions this + // group moves users in, and neither is implied by the bare group name — a + // default would make `clerk migrate --file users.json` mean "import" while + // its sibling has to be spelled out. Bare `clerk migrate` prints help. + // + // The flags stay here rather than on `migrate`, matching how `config` keeps + // its own on `pull`/`patch`/`put` — a group's help is a list of subcommands + // and examples, not a merge of everything underneath it. + migrateCommand + .command("import") + .description("Import users from an exported JSON or CSV file") + .addOption( + createOption( + "-t, --transformer ", + "Source platform the file was exported from", + ).choices(transformerKeys()), + ) + .option( + "--transformer-file ", + "Path to a transformer you wrote, for a platform with no built-in", + ) + .option("-f, --file ", "Path to the exported user data (JSON or CSV)") + .option("-r, --resume-after ", "Skip every user up to and including this source ID") + .option("--require-password", "Import only users that have a password") + .option( + "--skip-unsupported-providers", + "Supabase: skip users whose only social provider is not enabled in Clerk", + ) + .option("--firebase-signer-key ", "Firebase base64 signer key") + .option("--firebase-salt-separator ", "Firebase base64 salt separator") + .option("--firebase-rounds ", "Firebase scrypt rounds", (value: string) => + parseIntegerOption(value, "--firebase-rounds", { min: 1 }), + ) + .option("--firebase-mem-cost ", "Firebase scrypt memory cost", (value: string) => + parseIntegerOption(value, "--firebase-mem-cost", { min: 1 }), + ) + .option("-y, --yes", "Skip the confirmation prompt") + .option("--secret-key ", "Backend API secret key to use") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate import -y --transformer clerk --file users.json", + description: "Import a Clerk Dashboard export", + }, + { + command: "clerk migrate import -y -t clerk -f users.csv --require-password", + description: "Import only the users that carry a password digest", + }, + { + command: "clerk migrate import -y -t clerk -f users.json -r user_2x9k", + description: "Resume a partial migration after the last imported user", + }, + { + command: "clerk migrate import -y -t supabase -f users.json --skip-unsupported-providers", + description: "Skip Supabase users whose only provider is not enabled in Clerk", + }, + ]) + .action(async (_opts, cmd) => + migrate.run(cmd.optsWithGlobals() as Parameters[0]), + ); + + // Flat, not under a noun group: this is the one command in the tree that + // destroys data in Clerk, and it is worth keeping short and prominent. + migrateCommand + .command("delete") + .description("Delete the users created by the last migration for this project") + .option("-y, --yes", "Skip the confirmation prompt") + .option("--secret-key ", "Backend API secret key to use") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate delete", + description: "Undo the last migration after confirming", + }, + { command: "clerk migrate delete -y", description: "Undo without prompting" }, + ]) + .action(async (_opts, cmd) => + migrate.delete(cmd.optsWithGlobals() as Parameters[0]), + ); + + registerMigrateExport(migrateCommand); + + // A compiled binary has no source tree to grep, so the available mappings + // need a command rather than only appearing in the interactive picker. + const transformersCommand = migrateCommand + .command("transformers") + .description("Inspect the available source-platform transformers") + .setExamples([ + { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { + command: "clerk migrate transformers list --json", + description: "Machine-readable, including each one's ID field", + }, + { + command: "clerk migrate transformers list --transformer-file ./my-transformer.ts", + description: "Include one you wrote", + }, + ]); + + transformersCommand + .command("list", { isDefault: true }) + .description("List the built-in transformers, and any loaded from a file") + .option("--json", "Output as JSON") + .option("--transformer-file ", "Also list a transformer you wrote") + .setExamples([ + { command: "clerk migrate transformers list", description: "Show the built-in transformers" }, + { + command: "clerk migrate transformers list --transformer-file ./my-transformer.ts", + description: "Include one you wrote", + }, + ]) + .action(async (_opts, cmd) => + migrate.transformersList( + cmd.optsWithGlobals() as Parameters[0], + ), + ); + + registerMigrateLogs(migrateCommand); + registerMigrateSettings(migrateCommand); +} diff --git a/packages/cli-core/src/commands/migrate/lib/analysis.test.ts b/packages/cli-core/src/commands/migrate/lib/analysis.test.ts new file mode 100644 index 000000000..322c2d753 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/analysis.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "bun:test"; +import { analyzeFields, hasValue } from "./analysis.ts"; + +describe("hasValue", () => { + test.each([ + ["a string", "x", true], + ["zero", 0, true], + ["false", false, true], + ["a populated array", ["a"], true], + ["an object", {}, true], + ["an empty string", "", false], + ["an empty array", [], false], + ["null", null, false], + ["undefined", undefined, false], + ])("treats %s as present: %p", (_label, value, expected) => { + expect(hasValue(value)).toBe(expected); + }); +}); + +describe("analyzeFields", () => { + test("returns zeroed counts for an empty file", () => { + const result = analyzeFields([]); + expect(result.totalUsers).toBe(0); + expect(result.identifiers.hasAnyIdentifier).toBe(0); + expect(result.fieldCounts).toEqual({}); + }); + + test("counts each identifier kind separately", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev" }, + { userId: "2", unverifiedEmailAddresses: ["b@x.dev"] }, + { userId: "3", phone: "+15555550100" }, + { userId: "4", unverifiedPhoneNumbers: ["+15555550101"] }, + { userId: "5", username: "carol" }, + ]); + + expect(result.identifiers).toMatchObject({ + verifiedEmails: 1, + unverifiedEmails: 1, + verifiedPhones: 1, + unverifiedPhones: 1, + username: 1, + hasAnyIdentifier: 5, + }); + }); + + test("counts emailAddresses towards verified emails", () => { + const result = analyzeFields([{ userId: "1", emailAddresses: ["a@x.dev"] }]); + expect(result.identifiers.verifiedEmails).toBe(1); + }); + + test("counts a user with several identifiers once", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev", phone: "+15555550100", username: "ada" }, + ]); + expect(result.identifiers.hasAnyIdentifier).toBe(1); + expect(result.identifiers.verifiedEmails).toBe(1); + expect(result.identifiers.verifiedPhones).toBe(1); + }); + + // These users cannot be imported under any instance configuration, which is + // what makes the count worth surfacing separately. + test("counts users carrying no identifier at all", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev" }, + { userId: "2", firstName: "Nobody" }, + { userId: "3" }, + ]); + expect(result.totalUsers).toBe(3); + expect(result.identifiers.hasAnyIdentifier).toBe(1); + }); + + test("counts the analyzed non-identifier fields", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev", firstName: "Ada", password: "d", totpSecret: "s" }, + { userId: "2", email: "b@x.dev", firstName: "Grace" }, + { userId: "3", email: "c@x.dev", lastName: "Hopper" }, + ]); + expect(result.fieldCounts).toEqual({ + firstName: 2, + lastName: 1, + password: 1, + totpSecret: 1, + }); + }); + + test("omits fields no user carries, rather than reporting them as zero", () => { + const result = analyzeFields([{ userId: "1", email: "a@x.dev" }]); + expect("password" in result.fieldCounts).toBe(false); + }); + + test("does not count an empty value as present", () => { + const result = analyzeFields([{ userId: "1", email: "a@x.dev", firstName: "", lastName: [] }]); + expect(result.fieldCounts.firstName).toBeUndefined(); + expect(result.fieldCounts.lastName).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/analysis.ts b/packages/cli-core/src/commands/migrate/lib/analysis.ts new file mode 100644 index 000000000..848ec1d76 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/analysis.ts @@ -0,0 +1,80 @@ +/** + * Counts what an import file actually contains, per field. + * + * Ported from the standalone migration-tool's `src/lib/analysis.ts`. Runs on + * transformed-but-unvalidated users so the report describes the whole file, + * including the rows that will be skipped. + */ + +import type { User } from "../types.ts"; + +/** Non-identifier fields the readiness report reports coverage for. */ +export const ANALYZED_FIELDS = [ + { key: "firstName", label: "First name" }, + { key: "lastName", label: "Last name" }, + { key: "password", label: "Password" }, + { key: "totpSecret", label: "TOTP secret" }, +] as const; + +export type IdentifierCounts = { + verifiedEmails: number; + unverifiedEmails: number; + verifiedPhones: number; + unverifiedPhones: number; + username: number; + /** Users with at least one identifier — the rest cannot be imported at all. */ + hasAnyIdentifier: number; +}; + +export type FieldAnalysis = { + identifiers: IdentifierCounts; + totalUsers: number; + fieldCounts: Record; +}; + +/** True for anything with real content — `0` and `false` count, `""` and `[]` do not. */ +export function hasValue(value: unknown): boolean { + if (value === undefined || value === null || value === "") return false; + if (Array.isArray(value)) return value.length > 0; + return true; +} + +export function analyzeFields(users: (User | Record)[]): FieldAnalysis { + const identifiers: IdentifierCounts = { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: 0, + }; + const fieldCounts: Record = {}; + + for (const entry of users) { + const user = entry as Record; + + for (const field of ANALYZED_FIELDS) { + if (hasValue(user[field.key])) { + fieldCounts[field.key] = (fieldCounts[field.key] ?? 0) + 1; + } + } + + const verifiedEmail = hasValue(user.email) || hasValue(user.emailAddresses); + const unverifiedEmail = hasValue(user.unverifiedEmailAddresses); + const verifiedPhone = hasValue(user.phone) || hasValue(user.phoneNumbers); + const unverifiedPhone = hasValue(user.unverifiedPhoneNumbers); + const username = hasValue(user.username); + + if (verifiedEmail) identifiers.verifiedEmails++; + if (unverifiedEmail) identifiers.unverifiedEmails++; + if (verifiedPhone) identifiers.verifiedPhones++; + if (unverifiedPhone) identifiers.unverifiedPhones++; + if (username) identifiers.username++; + + if (verifiedEmail || unverifiedEmail || verifiedPhone || unverifiedPhone || username) { + identifiers.hasAnyIdentifier++; + } + } + + return { identifiers, totalUsers: users.length, fieldCounts }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/assume-yes.ts b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts new file mode 100644 index 000000000..6315520f2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts @@ -0,0 +1,32 @@ +/** + * Whether this run was given `-y`. + * + * `-y` is not the same question as {@link isAgent}. Agent mode says the CLI + * *cannot* prompt; `-y` says the operator does not want it to. Most prompts + * only care about the first — a confirm is skipped by either — but the two + * places that take a default instead of asking need to know a human chose it, + * so the two cannot be collapsed into one flag. + * + * Held per-run rather than threaded through, because the readers are three + * layers below the command that parses it: `ensureLogDir` runs inside the + * gutter of seven different commands, and `withInputRetry` sits under every + * credential prompt. Passing it down would put a `yes` parameter on every + * export handler signature on the way. This mirrors `mode.ts`, which resolves + * `--mode` once in a `preAction` hook and is read the same way. + * + * Set by the `migrate` group's `preAction` hook, so every subcommand under it + * is covered whether or not it declares the flag — one that does not simply + * resolves to `false`. + */ + +let assumeYes = false; + +/** Records this run's `-y`. Called once per invocation, before the action. */ +export function setAssumeYes(value: boolean): void { + assumeYes = value; +} + +/** Whether `-y` was passed to the command now running. */ +export function isAssumeYes(): boolean { + return assumeYes; +} diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts new file mode 100644 index 000000000..a94479cd9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts @@ -0,0 +1,85 @@ +import { test, expect, describe, mock, beforeEach, afterAll } from "bun:test"; +import { stubFetch, useCaptureLog } from "../../../test/lib/stubs.ts"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { fetchInstanceSettings } from "./clerk-config.ts"; + +const USER_SETTINGS = { + attributes: { email_address: { enabled: true, required: true } }, +} as unknown as UserSettingsJSON; + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +describe("fetchInstanceSettings", () => { + const originalFetch = globalThis.fetch; + useCaptureLog(); + const mockFetch = mock(); + + beforeEach(() => { + mockFetch.mockReset(); + stubFetch(mockFetch); + }); + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + /** Routes the three hops: BAPI domains → FAPI dev browser → FAPI environment. */ + function route(domains: unknown): void { + mockFetch.mockImplementation((input: string | URL) => { + const url = String(input); + if (url.includes("/v1/domains")) return Promise.resolve(json({ data: domains })); + if (url.includes("/v1/dev_browser")) return Promise.resolve(json({ token: "jwt" })); + if (url.includes("/v1/environment")) { + return Promise.resolve(json({ user_settings: USER_SETTINGS })); + } + throw new Error(`unexpected request: ${url}`); + }); + } + + test("reads settings off the primary domain's Frontend API", async () => { + route([{ is_satellite: false, frontend_api_url: "https://clerk.example.com" }]); + + expect(await fetchInstanceSettings("sk_test_abc")).toEqual(USER_SETTINGS); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.some((url) => url.includes("clerk.example.com/v1/dev_browser"))).toBe(true); + expect(urls.some((url) => url.includes("clerk.example.com/v1/environment"))).toBe(true); + }); + + test("prefers the primary domain over a satellite", async () => { + route([ + { is_satellite: true, frontend_api_url: "https://satellite.example.com" }, + { is_satellite: false, frontend_api_url: "https://clerk.example.com" }, + ]); + + await fetchInstanceSettings("sk_test_abc"); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.every((url) => !url.includes("satellite.example.com"))).toBe(true); + }); + + test("skips the dev browser bootstrap for a production key", async () => { + route([{ is_satellite: false, frontend_api_url: "https://clerk.example.com" }]); + + expect(await fetchInstanceSettings("sk_live_abc")).toEqual(USER_SETTINGS); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.some((url) => url.includes("/v1/dev_browser"))).toBe(false); + }); + + // `null` means "unknown", so callers degrade rather than treating a failed + // lookup as "nothing is enabled". + test("returns null when no domain names a Frontend API URL", async () => { + route([{ is_satellite: false }]); + expect(await fetchInstanceSettings("sk_test_abc")).toBeNull(); + }); + + test("returns null when the domains lookup fails", async () => { + mockFetch.mockResolvedValue(new Response("nope", { status: 401 })); + expect(await fetchInstanceSettings("sk_test_abc")).toBeNull(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts new file mode 100644 index 000000000..f69ae7fab --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -0,0 +1,144 @@ +/** + * The destination instance's live user settings: which identifiers it accepts + * and requires, and which social providers it has enabled. + * + * Ported from the standalone migration-tool's `src/lib/clerk.ts`, rewritten + * onto the CLI's own primitives: the FAPI host comes from BAPI `/v1/domains` + * — a secret key is all `migrate import` is given — and the settings come from + * `lib/fapi.ts` rather than a bespoke fetch. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import { + bootstrapDevBrowser, + fetchUserSettings, + type UserSettingsJSON, +} from "../../../lib/fapi.ts"; +import { log } from "../../../lib/log.ts"; +import { detectInstanceType } from "./instance.ts"; + +/** + * Supabase provider keys whose Clerk strategy is not simply `oauth_`. + * + * Everything not listed here maps by prefix, which covers google, github, + * discord, spotify, twitch, notion, figma, gitlab, bitbucket and the rest. + */ +const CLERK_STRATEGY_ALIASES: Record = { + azure: "oauth_microsoft", + twitter: "oauth_x", + slack_oidc: "oauth_slack", + fly: "oauth_fly", +}; + +/** Supabase's provider key as Clerk's OAuth strategy name. */ +export function toClerkStrategy(provider: string): string { + return CLERK_STRATEGY_ALIASES[provider] ?? `oauth_${provider}`; +} + +/** Human label for a provider key, for report output. */ +export function providerLabel(provider: string): string { + const special: Record = { + github: "GitHub", + gitlab: "GitLab", + linkedin_oidc: "LinkedIn (OIDC)", + slack_oidc: "Slack (OIDC)", + twitter: "Twitter (X)", + azure: "Microsoft (Azure)", + workos: "WorkOS", + fly: "Fly.io", + }; + return special[provider] ?? provider.charAt(0).toUpperCase() + provider.slice(1); +} + +/** + * The Frontend API host of the instance a secret key addresses. + * + * `/v1/instance` carries no publishable key — for any instance, linked or not + * — so the primary domain's `frontend_api_url` is the only route from a secret + * key to the host its settings live behind. Every instance has at least one + * domain; satellites share the primary's Frontend API, so ordering only + * matters for tidiness. + */ +async function fetchFapiHost(secretKey: string): Promise { + const response = await bapiRequest({ method: "GET", path: "/v1/domains", secretKey }); + const domains = (response.body as { data?: unknown })?.data; + if (!Array.isArray(domains)) return null; + + const primary = + domains.find((domain) => !(domain as { is_satellite?: boolean }).is_satellite) ?? domains[0]; + const frontendApiUrl = (primary as { frontend_api_url?: unknown })?.frontend_api_url; + if (typeof frontendApiUrl !== "string" || !frontendApiUrl) return null; + + return new URL(frontendApiUrl).host; +} + +/** + * Fetches the user settings for the instance a secret key addresses. + * + * @returns The settings, or `null` when they could not be read. Callers must + * treat `null` as "unknown" rather than as "nothing is enabled" — the + * readiness report degrades to a note, and provider skipping stands down. + */ +export async function fetchInstanceSettings(secretKey: string): Promise { + try { + const fapiHost = await fetchFapiHost(secretKey); + if (!fapiHost) { + log.debug("migrate: no domain on this instance named a Frontend API URL"); + return null; + } + + // Development FAPI rejects an environment request without a dev browser JWT. + const jwt = + detectInstanceType(secretKey) === "dev" ? await bootstrapDevBrowser(fapiHost) : undefined; + return await fetchUserSettings(fapiHost, jwt ? { jwt } : {}); + } catch (error) { + log.debug( + `migrate: could not read instance settings: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return null; + } +} + +/** The enabled social strategies (`oauth_google`, …) in a settings payload. */ +export function enabledSocialProviders(settings: UserSettingsJSON): string[] { + return Object.entries(settings.social ?? {}) + .filter(([, value]) => value?.enabled) + .map(([strategy]) => strategy); +} + +/** + * Convenience wrapper for callers that only need the enabled strategies. + * + * @returns `null` when the instance settings could not be read. + */ +export async function fetchEnabledSocialProviders(secretKey: string): Promise { + const settings = await fetchInstanceSettings(secretKey); + return settings ? enabledSocialProviders(settings) : null; +} + +/** + * How many users the destination instance already holds. + * + * The closest thing to a live quota check the CLI has: `max_allowed_users` is + * not exposed by any public API, so headroom on a development instance can + * only be estimated from the count and {@link DEV_USER_LIMIT}. + * + * @returns `null` when the count could not be read — an unknown count must not + * be reported as zero. + */ +export async function fetchUserCount(secretKey: string): Promise { + try { + const response = await bapiRequest({ method: "GET", path: "/v1/users/count", secretKey }); + const total = (response.body as { total_count?: unknown })?.total_count; + return typeof total === "number" ? total : null; + } catch (error) { + log.debug( + `migrate: could not read the instance's user count: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return null; + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts new file mode 100644 index 000000000..05ae45928 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -0,0 +1,308 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { + createDbClient, + describeDbError, + detectDbType, + redactConnectionString, + sqlitePath, + withDbClient, +} from "./db.ts"; + +let workDir: string; +let dbPath: string; + +beforeAll(() => { + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-db-"))); + dbPath = path.join(workDir, "test.sqlite"); + + const db = new Database(dbPath, { create: true }); + db.run(`CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER)`); + db.run(`INSERT INTO "user" VALUES (?, ?, ?)`, ["u1", "a@x.dev", 1]); + db.run(`INSERT INTO "user" VALUES (?, ?, ?)`, ["u2", "b@x.dev", 0]); + db.close(); +}); + +afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("detectDbType", () => { + test.each([ + ["postgres://u:p@h/db", "postgres"], + ["postgresql://u:p@h/db", "postgres"], + ["POSTGRES://u:p@h/db", "postgres"], + ["mysql://u:p@h/db", "mysql"], + ["mysql2://u:p@h/db", "mysql"], + ["./db.sqlite", "sqlite"], + ["libsql://app-org.turso.io", "sqlite"], + ["file:./db.sqlite", "sqlite"], + ["/abs/path.db", "sqlite"], + [" postgres://u:p@h/db ", "postgres"], + ])("%s -> %s", (input, expected) => { + expect(detectDbType(input)).toBe(expected as never); + }); +}); + +describe("redactConnectionString", () => { + test.each([ + ["postgres://user:secret@host:5432/db", "postgres://***@host:5432/db"], + ["mysql://root:hunter2@127.0.0.1:3306/app", "mysql://***@127.0.0.1:3306/app"], + ["postgres://host/db", "postgres://host/db"], + ["libsql://app.turso.io?authToken=secret", "libsql://app.turso.io?authToken=***"], + ])("%s -> %s", (input, expected) => { + expect(redactConnectionString(input)).toBe(expected); + }); + + // An unencoded `@` in the password is the most common mistake, and it is + // exactly when the string ends up in an error message. Matching the first + // `@` would leave the rest of the password visible. + test("redacts a password containing an unencoded @", () => { + const redacted = redactConnectionString("postgres://user:pa@ss@host/db"); + expect(redacted).toBe("postgres://***@host/db"); + expect(redacted).not.toContain("ss"); + }); + + test("redacts a password containing a colon", () => { + expect(redactConnectionString("postgres://user:a:b:c@host/db")).toBe("postgres://***@host/db"); + }); + + test.each([["./db.sqlite"], ["/var/data/app.db"], ["file:./local.sqlite"]])( + "leaves the credential-free path %s alone", + (input) => { + expect(redactConnectionString(input)).toBe(input); + }, + ); +}); + +describe("sqlitePath", () => { + test.each([ + ["./db.sqlite", "./db.sqlite"], + ["file:./db.sqlite", "./db.sqlite"], + ["file:/abs/db.sqlite", "/abs/db.sqlite"], + ["./db.sqlite?mode=ro", "./db.sqlite"], + [" ./db.sqlite ", "./db.sqlite"], + ])("%s -> %s", (input, expected) => { + expect(sqlitePath(input)).toBe(expected); + }); +}); + +describe("a libsql client", () => { + const originalFetch = globalThis.fetch; + let requests: { url: string; token?: string; body: any }[] = []; + + function stubFetch(result: unknown) { + requests = []; + globalThis.fetch = (async (url: string, init: RequestInit) => { + requests.push({ + url: String(url), + token: (init.headers as Record).authorization, + body: JSON.parse(String(init.body)), + }); + return new Response(JSON.stringify({ results: [result, { type: "ok" }] }), { + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + } + + const okRows = (cols: string[], rows: unknown[][]) => ({ + type: "ok", + response: { type: "execute", result: { cols: cols.map((name) => ({ name })), rows } }, + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + test("posts to the pipeline endpoint with the URL's token and decodes rows", async () => { + stubFetch( + okRows( + ["id", "count", "verified", "missing", "hash"], + [ + [ + { type: "text", value: "u1" }, + { type: "integer", value: "12" }, + { type: "float", value: 1.5 }, + { type: "null" }, + { type: "blob", base64: Buffer.from("hash").toString("base64") }, + ], + ], + ), + ); + + const client = await createDbClient("libsql://app-org.turso.io?authToken=t0ken"); + const rows = await client.query('SELECT * FROM "user" WHERE id = ?', ["u1"]); + await client.close(); + + expect(requests[0]?.url).toBe("https://app-org.turso.io/v2/pipeline"); + expect(requests[0]?.token).toBe("Bearer t0ken"); + expect(requests.at(-1)?.body.requests[0].stmt.args).toEqual([{ type: "text", value: "u1" }]); + expect(rows).toEqual([ + { + id: "u1", + count: 12, + verified: 1.5, + missing: null, + hash: Buffer.from("hash"), + }, + ] as never); + expect(client.dbType).toBe("sqlite"); + }); + + test("reports a server-side error", async () => { + stubFetch({ type: "error", error: { message: "no such table: user" } }); + + await expect(createDbClient("libsql://app-org.turso.io")).rejects.toThrow(CliError); + }); +}); + +describe("a sqlite client", () => { + test("connects and queries", async () => { + const client = await createDbClient(dbPath); + try { + const rows = await client.query<{ id: string }>(`SELECT id FROM "user" ORDER BY id`); + expect(rows.map((row) => row.id)).toEqual(["u1", "u2"]); + } finally { + await client.close(); + } + }); + + test("binds parameters", async () => { + const client = await createDbClient(dbPath); + try { + const rows = await client.query<{ email: string }>(`SELECT email FROM "user" WHERE id = ?`, [ + "u2", + ]); + expect(rows[0]?.email).toBe("b@x.dev"); + } finally { + await client.close(); + } + }); + + test("reports its dialect's placeholder and quoting", async () => { + const client = await createDbClient(dbPath); + try { + expect(client.dbType).toBe("sqlite"); + expect(client.placeholder(1)).toBe("?"); + expect(client.quote("user")).toBe('"user"'); + } finally { + await client.close(); + } + }); + + test("accepts a file: URL", async () => { + const client = await createDbClient(`file:${dbPath}`); + try { + expect(await client.query(`SELECT 1 AS n`)).toHaveLength(1); + } finally { + await client.close(); + } + }); + + // bun:sqlite opens lazily, so without an explicit probe a missing file would + // surface at the first real query, long after "connecting" finished. + test("fails at connect time when the file is missing, not mid-export", async () => { + await expect(createDbClient(path.join(workDir, "nope.sqlite"))).rejects.toThrow(CliError); + }); + + test("names the file in the failure", async () => { + await expect(createDbClient(path.join(workDir, "nope.sqlite"))).rejects.toThrow( + /Could not open the SQLite file/, + ); + }); +}); + +describe("withDbClient", () => { + test("returns the work's value", async () => { + expect(await withDbClient(dbPath, undefined, async () => "done")).toBe("done"); + }); + + test("closes the client even when the work throws", async () => { + // A leaked handle keeps the process alive after the export has written its + // file, which reads as a hang. + await expect( + withDbClient(dbPath, undefined, async () => { + throw new Error("boom"); + }), + ).rejects.toThrow(/boom/); + + // The file is still usable, so nothing is holding it open. + expect(await withDbClient(dbPath, undefined, async () => "reopened")).toBe("reopened"); + }); + + test("attaches a hint to a query failure, not just a connection failure", async () => { + await expect( + withDbClient(dbPath, undefined, (client) => client.query(`SELECT * FROM missing_table`)), + ).rejects.toThrow(/expected table was not found/); + }); + + test("passes a CliError through unchanged", async () => { + await expect( + withDbClient(dbPath, undefined, async () => { + throw new CliError("already explained"); + }), + ).rejects.toThrow(/already explained/); + }); +}); + +describe("describeDbError", () => { + const withCode = (code: string, message = "") => Object.assign(new Error(message), { code }); + + // Bun reports an unreachable host and a closed port identically, as + // "Connection closed" — precisely where a bare driver error helps least. + test.each([ + ["ERR_POSTGRES_CONNECTION_CLOSED", "Connection closed"], + ["ERR_MYSQL_CONNECTION_CLOSED", "Connection closed"], + ])("turns %s into host/port guidance", (code, message) => { + expect(describeDbError(withCode(code, message))).toMatch(/Check the host and port/); + }); + + test("gives Supabase the IPv4 add-on hint, which is the usual cause there", () => { + const hint = describeDbError( + withCode("ERR_POSTGRES_CONNECTION_CLOSED", "Connection closed"), + "supabase", + ); + expect(hint).toMatch(/pooler connection string/); + expect(hint).toMatch(/IPv4/); + }); + + test.each([ + ['password authentication failed for user "postgres"'], + ["Access denied for user 'root'@'localhost' (using password: YES)"], + ])("recognizes the rejected credentials in %p", (message) => { + expect(describeDbError(new Error(message))).toMatch(/rejected those credentials/); + }); + + test.each([ + ['relation "auth.users" does not exist'], + ["no such table: user"], + ["permission denied for table users"], + ])("recognizes the missing table in %p", (message) => { + expect(describeDbError(new Error(message))).toMatch(/table was not found|cannot read it/); + }); + + test("points Supabase at Auth being enabled and the postgres role", () => { + const hint = describeDbError(new Error('relation "auth.users" does not exist'), "supabase"); + expect(hint).toMatch(/Supabase Auth is enabled/); + expect(hint).toMatch(/postgres` role/); + }); + + test("recognizes an unopenable SQLite file", () => { + expect(describeDbError(new Error("unable to open database file"))).toMatch( + /Could not open the SQLite file/, + ); + }); + + test("still says something useful for an error it does not recognize", () => { + expect(describeDbError(new Error("something odd"))).toMatch(/Check the connection string/); + }); + + test("never echoes the error's own text, which could carry a connection string", () => { + const hint = describeDbError(new Error("failed for postgres://user:secret@host/db")); + expect(hint).not.toContain("secret"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts new file mode 100644 index 000000000..0f2dd4e8b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -0,0 +1,373 @@ +/** + * One query interface over Postgres, MySQL and SQLite. + * + * Rewritten from the standalone migration-tool's `src/lib/db.ts`, which used + * `pg`, `mysql2` and `better-sqlite3`. None of those belong in a statically + * compiled binary — `better-sqlite3` is a native addon outright — so this runs + * on `Bun.sql` (Postgres and MySQL) and `bun:sqlite`, both built into the + * runtime. That swap is the entire reason the `engines.bun` floor exists. + * + * **Placeholders are not unified.** `Bun.sql` passes the query through to each + * server as written, so Postgres wants `$1` and MySQL wants `?` — verified + * against both. Rather than rewrite SQL strings (Postgres uses `?` as a JSONB + * operator, so a naive rewriter would corrupt real queries), callers ask the + * client for the placeholder and the identifier quoting they need. They already + * build per-dialect SQL for table casing, so this adds no new branching. + */ + +import { Database } from "bun:sqlite"; +import { SQL } from "bun"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; + +export type DbType = "postgres" | "mysql" | "sqlite"; + +export interface DbClient { + dbType: DbType; + query>(sql: string, params?: unknown[]): Promise; + /** The bind placeholder for the 1-indexed `position`. */ + placeholder(position: number): string; + /** Quotes an identifier for this dialect. */ + quote(identifier: string): string; + close(): Promise; +} + +/** + * Reads the database type from a connection string. + * + * Anything that is not a recognized URL scheme is treated as a SQLite path, + * matching how the standalone tool behaved and how users actually pass + * `./db.sqlite`. `libsql://` (Turso) is SQLite too — it only differs in how + * the rows are fetched, so callers that branch on the dialect want "sqlite". + */ +export function detectDbType(connectionString: string): DbType { + const lower = connectionString.trim().toLowerCase(); + if (lower.startsWith("postgresql://") || lower.startsWith("postgres://")) return "postgres"; + if (lower.startsWith("mysql://") || lower.startsWith("mysql2://")) return "mysql"; + return "sqlite"; +} + +/** True for a remote libsql/Turso URL, which is read over HTTP rather than opened. */ +export function isLibsqlUrl(connectionString: string): boolean { + return /^libsql:\/\//i.test(connectionString.trim()); +} + +/** + * Replaces any credentials in a connection string with `***`. + * + * Connection strings reach the CLI on the command line and end up in error + * messages and `--verbose` output. Bun's own errors do not echo them, and + * nothing here should either. + */ +export function redactConnectionString(connectionString: string): string { + // Greedy up to the LAST `@`: an unencoded `@` in the password is the most + // common connection-string mistake, and matching the first one would leave + // the rest of the password in the message. Everything before the final `@` + // is userinfo, so redacting all of it is always safe. + // Non-URL forms (SQLite paths) have no `://` and are left alone. + // Turso carries its credential as `?authToken=`, not as userinfo. + return connectionString + .replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@") + .replace(/([?&]authToken=)[^&]*/gi, "$1***"); +} + +/** Strips a `file:` prefix and any URL query, leaving a filesystem path. */ +export function sqlitePath(connectionString: string): string { + const trimmed = connectionString.trim(); + const withoutScheme = trimmed.startsWith("file:") ? trimmed.slice("file:".length) : trimmed; + return withoutScheme.split("?")[0] ?? withoutScheme; +} + +const QUOTING: Record string> = { + // Doubling the delimiter is the escape in every dialect here, so an + // identifier containing one cannot break out of the quotes. + postgres: (identifier) => `"${identifier.replace(/"/g, '""')}"`, + sqlite: (identifier) => `"${identifier.replace(/"/g, '""')}"`, + mysql: (identifier) => `\`${identifier.replace(/`/g, "``")}\``, +}; + +function bunSqlClient(connectionString: string, dbType: "postgres" | "mysql"): DbClient { + const sql = new SQL(connectionString); + + return { + dbType, + async query>(query: string, params: unknown[] = []) { + const rows = await sql.unsafe(query, params); + return (Array.isArray(rows) ? rows : []) as T[]; + }, + placeholder: dbType === "postgres" ? (position) => `$${position}` : () => "?", + quote: QUOTING[dbType], + async close() { + await sql.close(); + }, + }; +} + +/** + * One value in Hrana's wire format, the protocol libsql servers speak. + * + * Integers arrive as strings so 64-bit values survive JSON. + */ +type HranaValue = { type: string; value?: string | number; base64?: string }; + +function decodeHrana(value: HranaValue): unknown { + switch (value.type) { + case "null": + return null; + case "integer": { + const raw = String(value.value ?? "0"); + const asNumber = Number(raw); + // Past 2^53 a number would silently lose digits; ids can get that big. + return Number.isSafeInteger(asNumber) ? asNumber : BigInt(raw); + } + case "float": + return Number(value.value); + case "blob": + // bun:sqlite hands back bytes for a BLOB, so this does too. + return Buffer.from(value.base64 ?? "", "base64"); + default: + return value.value ?? null; + } +} + +function encodeHrana(param: unknown): HranaValue { + if (param === null || param === undefined) return { type: "null" }; + if (typeof param === "bigint") return { type: "integer", value: param.toString() }; + if (typeof param === "boolean") return { type: "integer", value: param ? "1" : "0" }; + if (typeof param === "number") { + return Number.isInteger(param) + ? { type: "integer", value: String(param) } + : { type: "float", value: param }; + } + if (param instanceof Uint8Array) { + return { type: "blob", base64: Buffer.from(param).toString("base64") }; + } + return { type: "text", value: String(param) }; +} + +/** + * Talks to a libsql server (Turso) over its HTTP pipeline endpoint. + * + * `bun:sqlite` opens local files and cannot reach a remote database, and + * `@libsql/client` ships native optional dependencies that do not survive + * `bun build --compile`. The protocol is one POST per statement, so it is + * fewer lines to speak it directly than to carry the dependency. + * + * The token comes from `?authToken=` on the URL — the form the Turso CLI + * prints — or from `TURSO_AUTH_TOKEN`/`LIBSQL_AUTH_TOKEN`. A self-hosted sqld + * with auth disabled needs neither, so a missing token is not an error here. + */ +function libsqlClient( + connectionString: string, + env: Record = process.env, +): DbClient { + const url = new URL(connectionString.trim()); + const token = url.searchParams.get("authToken") || env.TURSO_AUTH_TOKEN || env.LIBSQL_AUTH_TOKEN; + const endpoint = `https://${url.host}/v2/pipeline`; + + return { + dbType: "sqlite", + async query>(query: string, params: unknown[] = []) { + const response = await fetch(endpoint, { + method: "POST", + headers: { + "content-type": "application/json", + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + // `close` keeps every request stateless: no baton to carry forward. + body: JSON.stringify({ + requests: [ + { type: "execute", stmt: { sql: query, args: params.map(encodeHrana) } }, + { type: "close" }, + ], + }), + }); + + if (!response.ok) { + throw new Error(`libsql request failed: ${response.status} ${response.statusText}`.trim()); + } + + const body = (await response.json()) as { + results?: { + type: string; + error?: { message?: string }; + response?: { result?: { cols?: { name?: string }[]; rows?: HranaValue[][] } }; + }[]; + }; + + const first = body.results?.[0]; + if (!first || first.type === "error") { + throw new Error(first?.error?.message ?? "libsql returned no result"); + } + + const cols = first.response?.result?.cols ?? []; + const rows = first.response?.result?.rows ?? []; + return rows.map( + (row) => + Object.fromEntries( + row.map((value, index) => [cols[index]?.name ?? String(index), decodeHrana(value)]), + ) as T, + ); + }, + placeholder: () => "?", + quote: QUOTING.sqlite, + async close() { + return Promise.resolve(); + }, + }; +} + +function sqliteClient(connectionString: string): DbClient { + const database = new Database(sqlitePath(connectionString), { readonly: true }); + + return { + dbType: "sqlite", + async query>(query: string, params: unknown[] = []) { + // bun:sqlite is synchronous; the Promise keeps one interface for callers. + return Promise.resolve(database.query(query).all(...(params as never[])) as T[]); + }, + placeholder: () => "?", + quote: QUOTING.sqlite, + async close() { + database.close(); + return Promise.resolve(); + }, + }; +} + +/** + * Connects to the database a connection string names. + * + * @param platform - Tailors the failure hint; the same "Connection closed" + * means something different on Supabase than on a local SQLite file. + */ +export async function createDbClient( + connectionString: string, + platform?: DbPlatform, +): Promise { + const dbType = detectDbType(connectionString); + + try { + if (isLibsqlUrl(connectionString)) { + const client = libsqlClient(connectionString); + await client.query("SELECT 1"); + return client; + } + + if (dbType === "sqlite") { + const client = sqliteClient(connectionString); + // bun:sqlite opens lazily, so a missing file would not surface until the + // first real query — long after the "connecting" spinner has stopped. + await client.query("SELECT 1"); + return client; + } + + const client = bunSqlClient(connectionString, dbType); + await client.query("SELECT 1"); + return client; + } catch (error) { + throw connectionError(error, connectionString, platform); + } +} + +export type DbPlatform = "supabase" | "betterauth" | "authjs"; + +/** + * Turns a driver error into something a user can act on. + * + * Rewritten rather than ported: the standalone tool matched on `pg`'s + * `ENOTFOUND`/`ETIMEDOUT`, which `Bun.sql` never emits. Bun reports both an + * unreachable host and a closed port as `ERR_*_CONNECTION_CLOSED` with the + * message "Connection closed" — exactly the case where a bare driver error + * helps least. + */ +export function describeDbError(error: unknown, platform?: DbPlatform): string { + const message = error instanceof Error ? error.message : String(error); + const code = (error as { code?: string })?.code ?? ""; + + if (code.includes("CONNECTION_CLOSED") || /connection closed|econnrefused/i.test(message)) { + if (platform === "supabase") { + return ( + "Could not reach the database. Check the host and port in the connection string.\n" + + "Supabase direct connections need the IPv4 add-on — use the pooler connection string\n" + + "(Dashboard → Connect → Session pooler), or enable IPv4 under Settings → Add-Ons." + ); + } + return "Could not reach the database. Check the host and port, and that the server accepts connections from here."; + } + + // Turso resolves every `*.turso.io` name, so a typo'd database does not fail + // to connect — it answers 404. "Check the host" would send the reader after + // the half that is right. + if (/\b404\b/.test(message)) { + return ( + "No database at that libsql host. Check the database name in the URL —\n" + + "`turso db show ` prints the URL to use." + ); + } + + if (/\b401\b|unauthorized|not authorized/i.test(message)) { + return ( + "The libsql server rejected that token.\n" + + "Append ?authToken=… to the URL, or set TURSO_AUTH_TOKEN (`turso db tokens create `)." + ); + } + + if (/password authentication failed|access denied/i.test(message)) { + return "The database rejected those credentials. Check the user and password in the connection string."; + } + + if (/does not exist|unknown database|no such table|permission denied/i.test(message)) { + if (platform === "supabase") { + return ( + "The auth.users table was not readable. It is created automatically when Supabase Auth is enabled.\n" + + "Check Authentication is enabled, and connect as the `postgres` role rather than an application role." + ); + } + return "The expected table was not found, or the user cannot read it. Check the database name and the user's SELECT permission."; + } + + if (/unable to open database|sqlitecantopen|no such file/i.test(message)) { + return "Could not open the SQLite file. Check the path, and that the file exists and is readable."; + } + + return "Check the connection string, that the server is running, and that it is reachable from here."; +} + +function connectionError( + error: unknown, + connectionString: string, + platform?: DbPlatform, +): CliError { + const message = error instanceof Error ? error.message : String(error); + return new CliError( + `Could not connect to ${redactConnectionString(connectionString)}: ${message}\n\n${describeDbError(error, platform)}`, + { code: ERROR_CODE.USAGE_ERROR }, + ); +} + +/** + * Runs `work` against a fresh client and always closes it. + * + * A leaked connection keeps the process alive after the export has written its + * file, which looks like a hang. + */ +export async function withDbClient( + connectionString: string, + platform: DbPlatform | undefined, + work: (client: DbClient) => Promise, +): Promise { + const client = await createDbClient(connectionString, platform); + try { + return await work(client); + } catch (error) { + // A query failure carries the same actionable hints as a connection one: + // a missing table is the most common thing that goes wrong here. + if (error instanceof CliError) throw error; + throw new CliError( + `${error instanceof Error ? error.message : String(error)}\n\n${describeDbError(error, platform)}`, + { code: ERROR_CODE.USAGE_ERROR }, + ); + } finally { + await client.close().catch(() => {}); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.test.ts b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts new file mode 100644 index 000000000..855cc9274 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/env-file.test.ts @@ -0,0 +1,171 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + clearMigrateEnvValues, + findMigrateEnvValue, + MIGRATE_ENV_FILE, + writeMigrateEnvValues, +} from "./env-file.ts"; + +let workDir: string; + +const envFile = () => path.join(workDir, MIGRATE_ENV_FILE); +const read = (file: string) => fs.readFileSync(path.join(workDir, file), "utf-8"); + +beforeEach(() => { + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-envfile-"))); +}); + +afterEach(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("writeMigrateEnvValues", () => { + test("creates the file and gitignores it", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).toBe("CLERK_FIREBASE_ROUNDS=8\n"); + expect(read(".gitignore")).toContain(MIGRATE_ENV_FILE); + }); + + test("appends to an existing .gitignore without duplicating the entry", async () => { + fs.writeFileSync(path.join(workDir, ".gitignore"), "node_modules\n"); + + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_MEM_COST: "14" }, workDir); + + expect(read(".gitignore")).toBe(`node_modules\n${MIGRATE_ENV_FILE}\n`); + }); + + // The header `mergeEnvVars` adds is right for an app's shared .env and wrong + // here — one `settings set` per key would stack one header per call. + test("adds no section header, however many times it is called", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_MEM_COST: "14" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_SIGNER_KEY: "k" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).not.toContain("#"); + }); + + test("updates a key in place rather than appending a second copy", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "10" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).toBe("CLERK_FIREBASE_ROUNDS=10\n"); + }); + + // The file is meant to be hand-editable, so a write must not flatten it. + test("preserves hand-written comments and unrelated keys", async () => { + fs.writeFileSync(envFile(), "# my note\nOTHER=keep\n"); + + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + expect(read(MIGRATE_ENV_FILE)).toBe("# my note\nOTHER=keep\nCLERK_FIREBASE_ROUNDS=8\n"); + }); +}); + +describe("findMigrateEnvValue", () => { + test("reads a value out of the file", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_SIGNER_KEY: "from-file" }, workDir); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_SIGNER_KEY"], workDir, {}); + expect(located).toEqual({ + value: "from-file", + name: "CLERK_FIREBASE_SIGNER_KEY", + source: MIGRATE_ENV_FILE, + }); + }); + + test("beats the app's own .env.local", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=1\n"); + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, {}); + expect(located?.value).toBe("8"); + }); + + // An exported variable is the one thing an operator can change per-invocation. + test("loses to an exported environment variable", async () => { + await writeMigrateEnvValues({ CLERK_FIREBASE_ROUNDS: "8" }, workDir); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "99", + }); + expect(located).toEqual({ + value: "99", + name: "CLERK_FIREBASE_ROUNDS", + source: "CLERK_FIREBASE_ROUNDS env var", + }); + }); + + test("returns nothing when the setting is absent everywhere", async () => { + expect(await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, {})).toBeUndefined(); + }); + + // Bun loads `.env.local` into process.env before the CLI runs, so a value a + // developer put in a file arrives looking like an exported variable. Naming + // the variable answers nothing — the question is which file to edit. + describe("attributing an environment value to the file it came from", () => { + test("names the file when it holds the same value", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "8", + }); + expect(located?.source).toBe(".env.local"); + }); + + // The one case the source column exists for: the file lost, so naming it + // would point at the value that is not being used. + test("keeps the variable when the file holds a different value", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "99", + }); + expect(located?.source).toBe("CLERK_FIREBASE_ROUNDS env var"); + }); + + test("prefers the file the runtime would have loaded last", async () => { + fs.writeFileSync(path.join(workDir, ".env"), "CLERK_FIREBASE_ROUNDS=8\n"); + fs.writeFileSync(path.join(workDir, ".env.local"), "CLERK_FIREBASE_ROUNDS=8\n"); + + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "8", + }); + expect(located?.source).toBe(".env.local"); + }); + + test("keeps the variable when no file holds it at all", async () => { + const located = await findMigrateEnvValue(["CLERK_FIREBASE_ROUNDS"], workDir, { + CLERK_FIREBASE_ROUNDS: "8", + }); + expect(located?.source).toBe("CLERK_FIREBASE_ROUNDS env var"); + }); + }); +}); + +describe("clearMigrateEnvValues", () => { + test("removes only the named settings", async () => { + fs.writeFileSync(envFile(), "OTHER=keep\nCLERK_FIREBASE_ROUNDS=8\n"); + + expect(await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir)).toEqual([ + "CLERK_FIREBASE_ROUNDS", + ]); + expect(read(MIGRATE_ENV_FILE)).toBe("OTHER=keep\n"); + }); + + // Left behind, it reads as "there is config here" when there is not. + test("deletes the file when nothing but comments would remain", async () => { + fs.writeFileSync(envFile(), "# a note\nCLERK_FIREBASE_ROUNDS=8\n"); + + await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir); + expect(fs.existsSync(envFile())).toBe(false); + }); + + test("reports nothing dropped when there is no file", async () => { + expect(await clearMigrateEnvValues(["CLERK_FIREBASE_ROUNDS"], workDir)).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/env-file.ts b/packages/cli-core/src/commands/migrate/lib/env-file.ts new file mode 100644 index 000000000..2c15c9163 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/env-file.ts @@ -0,0 +1,162 @@ +/** + * `.env.clerk-migrate` — the migration's own env file. + * + * Migration credentials are a Firebase signer key, an Auth0 client secret, a + * database URL: things the app being migrated has no use for. Writing them into + * the app's `.env.local` mixes two unrelated sets of config in the file a + * developer reads every day, so they get their own. + * + * Read ahead of `.env`/`.env.local`, so a value set here wins over a stale one + * left in the app's file. An exported shell variable still beats both — that is + * {@link findEnvValue}'s contract for every value the CLI resolves. + * + * Always added to `.gitignore` on write. The CLI creating a credential-bearing + * file in someone's repository without that is how one ends up committed. + */ + +import { unlink } from "node:fs/promises"; +import { join } from "node:path"; +import { + findEnvValue, + parseEnvFile, + serializeEnvFile, + type EnvLine, + type LocatedEnvValue, +} from "../../../lib/dotenv.ts"; +import { ensureGitignoreEntry } from "../../../lib/git.ts"; +import { log } from "../../../lib/log.ts"; + +export const MIGRATE_ENV_FILE = ".env.clerk-migrate"; + +/** Lowest priority first: the migration's own file overrides the app's. */ +const MIGRATE_ENV_FILES = [".env", ".env.local", MIGRATE_ENV_FILE] as const; + +/** + * The project env file a value in the environment actually came from, if any. + * + * Bun loads `.env`, `.env.local` and friends into `process.env` before the CLI + * runs, so a variable a developer wrote into `.env.local` reaches + * {@link findEnvValue} as an environment variable and gets reported as one. + * That is true but useless: "`ROUNDS` env var" does not tell anyone which of + * their files to edit. + * + * Attribution is by value, not by presence. A file that holds the same key with + * a *different* value lost to something exported in the shell, and saying + * `.env.local` there would name the file that is not winning — the one case + * this column exists to catch. Highest-priority file first, matching the order + * the runtime loaded them in. + */ +async function fileHolding( + cwd: string, + { name, value }: LocatedEnvValue, +): Promise { + for (const envFile of [...MIGRATE_ENV_FILES].reverse()) { + const file = Bun.file(join(cwd, envFile)); + if (!(await file.exists())) continue; + + for (const line of parseEnvFile(await file.text())) { + if (line.type === "entry" && line.key === name && line.value === value) return envFile; + } + } + return undefined; +} + +/** Resolves a migration setting: environment first, then the project's env files. */ +export async function findMigrateEnvValue( + names: string[], + cwd: string = process.cwd(), + env: Record = process.env, +): Promise { + const located = await findEnvValue(cwd, names, { env, files: MIGRATE_ENV_FILES }); + if (!located) return undefined; + + // `findEnvValue` reports the environment before it reads a file, so a value + // the runtime loaded out of `.env.local` is credited to the variable rather + // than to the file the user would edit. Put the file back. + const source = located.source.endsWith(" env var") + ? ((await fileHolding(cwd, located)) ?? located.source) + : located.source; + + log.debug(`migrate: ${names[0]} from ${source}`); + return { ...located, source }; +} + +/** + * Merges `values` into the parsed file: existing keys update in place, new ones + * append. + * + * Deliberately not `mergeEnvVars` from `lib/dotenv.ts`. That one prepends a + * `# Clerk` section header when the file holds none of the keys being written, + * which is right for `env pull` dropping Clerk keys into an app's shared `.env` + * — and wrong here twice over: every key in this file is already Clerk's, and + * writing one setting at a time means the check fires again on every call, + * stacking a fresh header per `settings set`. + */ +function mergeMigrateEnv(lines: EnvLine[], values: Record): EnvLine[] { + const remaining = { ...values }; + + const merged = lines.map((line): EnvLine => { + if (line.type !== "entry" || !(line.key in remaining)) return line; + const value = remaining[line.key]!; + delete remaining[line.key]; + return { type: "entry", key: line.key, value, raw: `${line.key}=${value}` }; + }); + + for (const [key, value] of Object.entries(remaining)) { + merged.push({ type: "entry", key, value, raw: `${key}=${value}` }); + } + return merged; +} + +/** + * Writes settings into `.env.clerk-migrate`, creating and gitignoring it first. + * + * Existing comments, blank lines and key order survive — the file is meant to + * be hand-edited, so rewriting it wholesale would discard the user's notes. + */ +export async function writeMigrateEnvValues( + values: Record, + cwd: string = process.cwd(), +): Promise { + const target = join(cwd, MIGRATE_ENV_FILE); + const existing = await Bun.file(target) + .text() + .catch(() => ""); + + await Bun.write(target, serializeEnvFile(mergeMigrateEnv(parseEnvFile(existing), values))); + await ensureGitignoreEntry(cwd, MIGRATE_ENV_FILE); + + return MIGRATE_ENV_FILE; +} + +/** Removes the named settings from `.env.clerk-migrate`, leaving the rest. */ +export async function clearMigrateEnvValues( + names: string[], + cwd: string = process.cwd(), +): Promise { + const target = join(cwd, MIGRATE_ENV_FILE); + const existing = await Bun.file(target) + .text() + .catch(() => ""); + if (!existing) return []; + + const dropped: string[] = []; + const kept = parseEnvFile(existing).filter((line) => { + if (line.type !== "entry" || !names.includes(line.key)) return true; + dropped.push(line.key); + return false; + }); + + if (dropped.length === 0) return dropped; + + // A file holding nothing but the comments that described the settings it no + // longer has is worse than no file: it reads as "there is config here". + if (kept.some((line) => line.type === "entry")) { + await Bun.write(target, serializeEnvFile(kept)); + } else { + await unlink(target).catch(() => {}); + log.debug(`migrate: removed empty ${MIGRATE_ENV_FILE}`); + } + + return dropped; +} diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts new file mode 100644 index 000000000..ec5d89fb2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts @@ -0,0 +1,211 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { resolveFirebaseHashConfig } from "./firebase-hash.ts"; + +const captured = useCaptureLog(); + +const ALL_FLAGS = { + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, +}; + +const ENV = { + CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER", + CLERK_FIREBASE_SALT_SEPARATOR: "Bw==", + CLERK_FIREBASE_ROUNDS: "8", + CLERK_FIREBASE_MEM_COST: "14", +}; + +let workDir: string; +let originalCwd: string; + +const setEnv = (vars: Partial) => Object.assign(process.env, vars); + +beforeEach(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-fbhash-"))); + process.chdir(workDir); +}); + +afterEach(() => { + for (const name of Object.keys(ENV)) delete process.env[name]; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("gating on the transformer", () => { + // `migrate import` is one command for every platform, so a signer key left in + // .env.clerk-migrate after a Firebase migration is in scope for whatever runs + // next unless the transformer says otherwise. + test.each([["clerk"], ["supabase"], ["auth0"], ["authjs"], ["betterauth"]])( + "reads nothing for the %s transformer", + async (transformer) => { + setEnv(ENV); + expect(await resolveFirebaseHashConfig({}, transformer)).toBeUndefined(); + }, + ); + + test("stays silent about a complete set on another platform's run", async () => { + setEnv(ENV); + await resolveFirebaseHashConfig({}, "supabase"); + expect(captured.err).toBe(""); + }); + + // The case that regressed: half a set used to fail every later run. + test("stays silent about a partial set on another platform's run", async () => { + fs.writeFileSync(path.join(workDir, ".env.clerk-migrate"), "CLERK_FIREBASE_SIGNER_KEY=left\n"); + + expect(await resolveFirebaseHashConfig({}, "supabase")).toBeUndefined(); + expect(captured.err).toBe(""); + }); + + test("ignores even explicit flags when the platform is not firebase", async () => { + expect(await resolveFirebaseHashConfig(ALL_FLAGS, "supabase")).toBeUndefined(); + }); + + test("resolves nothing before the platform is known", async () => { + setEnv(ENV); + expect(await resolveFirebaseHashConfig({}, undefined)).toBeUndefined(); + }); +}); + +describe("on a firebase run", () => { + test("builds the config from flags", async () => { + expect(await resolveFirebaseHashConfig(ALL_FLAGS, "firebase")).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + test("falls back to the environment", async () => { + setEnv(ENV); + expect((await resolveFirebaseHashConfig({}, "firebase"))?.base64_signer_key).toBe("ENV_SIGNER"); + }); + + test("reads .env.clerk-migrate when the variable is not exported", async () => { + fs.writeFileSync( + path.join(workDir, ".env.clerk-migrate"), + Object.entries(ENV) + .map(([key, value]) => `${key}=${value}`) + .join("\n"), + ); + + expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); + }); + + test("prefers a flag over the environment", async () => { + setEnv(ENV); + expect((await resolveFirebaseHashConfig(ALL_FLAGS, "firebase"))?.base64_signer_key).toBe( + "SIGNER", + ); + }); + + test("fills only the gaps the flags left", async () => { + setEnv({ CLERK_FIREBASE_ROUNDS: "8", CLERK_FIREBASE_MEM_COST: "14" }); + + expect( + await resolveFirebaseHashConfig( + { firebaseSignerKey: "SIGNER", firebaseSaltSeparator: "Bw==" }, + "firebase", + ), + ).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + // A digest built from a partial set is well-formed but verifies against + // nothing, so every migrated user would silently fail to sign in. + test.each([ + ["firebaseSignerKey", "--firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost"], + ] as const)("rejects a flag set missing %s, naming it", async (omit, flag) => { + const partial = { ...ALL_FLAGS }; + delete (partial as Record)[omit]; + + await expect(resolveFirebaseHashConfig(partial, "firebase")).rejects.toThrow(new RegExp(flag)); + }); + + test("names every missing flag at once", async () => { + await expect( + resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" }, "firebase"), + ).rejects.toThrow(/--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/); + }); + + // Saved config is a leftover, not an instruction — but on a Firebase import + // it is the reason the passwords will not come across, so it is said aloud. + test("warns and continues when only saved config is partial", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); + + expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); + expect(captured.err).toContain("Ignoring an incomplete Firebase hash configuration"); + }); + + test("still fails when a flag supplied part of the set", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "ENV_SIGNER" }); + + await expect(resolveFirebaseHashConfig({ firebaseRounds: 8 }, "firebase")).rejects.toThrow( + /--firebase-salt-separator/, + ); + }); + + // An empty variable is how a shell spells "unset". + test("ignores an empty variable", async () => { + setEnv({ CLERK_FIREBASE_SIGNER_KEY: "" }); + expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); + }); + + test("returns nothing when neither flags nor the environment supply a config", async () => { + expect(await resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); + }); +}); + +// Firebase names these `base64_signer_key`, `rounds` and friends, and that is +// how every guide — Clerk's own standalone script included — tells you to write +// them into `.env`. A project that followed one has the values already. +describe("the names Firebase itself uses", () => { + const writeEnvLocal = (contents: string) => + fs.writeFileSync(path.join(workDir, ".env.local"), contents); + + test("reads a set written under the unprefixed names", async () => { + writeEnvLocal("BASE64_SIGNER_KEY=SIGNER\nBASE64_SALT_SEPARATOR=Bw==\nROUNDS=8\nMEM_COST=14\n"); + + expect(await resolveFirebaseHashConfig({}, "firebase")).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + test("reads a set written under the FIREBASE_ prefix", async () => { + writeEnvLocal( + "FIREBASE_BASE64_SIGNER_KEY=SIGNER\nFIREBASE_BASE64_SALT_SEPARATOR=Bw==\n" + + "FIREBASE_ROUNDS=8\nFIREBASE_MEM_COST=14\n", + ); + + expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); + }); + + // The alias is a fallback, not a synonym: `ROUNDS` in an app's own env file + // is not necessarily about Firebase at all. + test("prefers the prefixed variable in the same file", async () => { + writeEnvLocal( + "ROUNDS=99\nCLERK_FIREBASE_ROUNDS=8\nBASE64_SIGNER_KEY=SIGNER\n" + + "BASE64_SALT_SEPARATOR=Bw==\nMEM_COST=14\n", + ); + + expect((await resolveFirebaseHashConfig({}, "firebase"))?.rounds).toBe(8); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts new file mode 100644 index 000000000..d32fc3c58 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts @@ -0,0 +1,124 @@ +/** + * Firebase's four scrypt parameters: where they come from, and when they are + * looked for at all. + * + * **Only read when the transformer is `firebase`.** `migrate import` is one + * command serving every platform, so a `CLERK_FIREBASE_SIGNER_KEY` left in + * `.env.clerk-migrate` after a Firebase migration is in scope for the Supabase + * run that follows it unless something says otherwise. Nothing downstream would + * misuse it — only the Firebase transformer reads the config off + * {@link TransformContext} — but resolving it means a stale or partial set can + * warn, or fail, a run that never mentioned Firebase. So the gate is here, + * before the lookup, rather than a filter after it. + * + * The per-platform export commands need no such gate: `migrate export auth0` + * reads `AUTH0_*` and nothing else, because the command itself is the platform. + * This is the only place one command spans them all. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { envNames, findSetting } from "../settings/registry.ts"; +import { findMigrateEnvValue } from "./env-file.ts"; +import type { FirebaseHashConfig } from "../types.ts"; + +/** The `--firebase-*` flags, and the setting each falls back to. */ +export const FIREBASE_FLAGS = [ + ["firebaseSignerKey", "--firebase-signer-key", "firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator", "firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds", "firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost", "firebase-mem-cost"], +] as const; + +const FIREBASE_NUMERIC: ReadonlySet = new Set(["firebaseRounds", "firebaseMemCost"]); + +export type FirebaseHashFlags = { + firebaseSignerKey?: string; + firebaseSaltSeparator?: string; + firebaseRounds?: number; + firebaseMemCost?: number; +}; + +/** + * Overlays the saved environment values onto whichever flags were not passed. + * + * The variables come from the settings registry — `CLERK_FIREBASE_*` and the + * unprefixed names Firebase itself uses — so `clerk migrate settings` and the + * import read exactly the same set. Resolved through + * {@link findMigrateEnvValue}: the environment first, then + * `.env.clerk-migrate`, then the app's own `.env` files. The signer key is a + * Firebase secret, so it is never written to the CLI's config — + * `.env.clerk-migrate` is gitignored on creation. + */ +async function withFirebaseEnv(flags: FirebaseHashFlags): Promise { + const merged: FirebaseHashFlags = { ...flags }; + for (const [key, , settingName] of FIREBASE_FLAGS) { + if (merged[key] !== undefined) continue; + const setting = findSetting(settingName); + const located = setting && (await findMigrateEnvValue(envNames(setting))); + if (!located || located.value.trim() === "") continue; + // A non-numeric round count is left to fail the flag's own validation + // rather than silently becoming NaN. + (merged as Record)[key] = FIREBASE_NUMERIC.has(key) + ? Number(located.value) + : located.value; + } + return merged; +} + +/** + * Resolves the four parameters from flags, then the `CLERK_FIREBASE_*` + * variables, then the project's env files. + * + * The four are required as a set: a digest built from a partial set is + * well-formed but verifies against nothing, so every migrated user would fail + * to sign in with no error at import time. How a partial set is treated depends + * on where it came from — flags are an instruction, saved config is not. + * + * @param transformer - The platform being migrated. Anything but `firebase` + * returns immediately, without reading the environment. + * @returns The config, or `undefined` when none was supplied — which is fine + * for an export that carries no password hashes. + */ +export async function resolveFirebaseHashConfig( + flags: FirebaseHashFlags, + transformer: string | undefined, +): Promise { + if (transformer !== "firebase") return undefined; + + const fromFlags = FIREBASE_FLAGS.filter(([key]) => flags[key] !== undefined); + const resolved = await withFirebaseEnv(flags); + const provided = FIREBASE_FLAGS.filter(([key]) => resolved[key] !== undefined); + + if (provided.length === 0) return undefined; + + if (provided.length < FIREBASE_FLAGS.length) { + const missing = FIREBASE_FLAGS.filter(([key]) => resolved[key] === undefined).map( + ([, flag]) => flag, + ); + + // Saved config is a leftover, not an instruction: half a set in + // `.env.clerk-migrate` should not fail the run, but on a Firebase import it + // is the reason the passwords will not come across, so it is said out loud. + if (fromFlags.length === 0) { + log.warn( + `Ignoring an incomplete Firebase hash configuration (no ${missing.join(", ")}). ` + + "Run `clerk migrate settings` to see what is set.", + ); + return undefined; + } + + throwUsageError( + `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + + "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + "https://clerk.com/docs/guides/development/migrating/firebase", + ); + } + + return { + base64_signer_key: resolved.firebaseSignerKey as string, + base64_salt_separator: resolved.firebaseSaltSeparator as string, + rounds: resolved.firebaseRounds as number, + mem_cost: resolved.firebaseMemCost as number, + }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts new file mode 100644 index 000000000..c414d6d1b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -0,0 +1,212 @@ +/** + * `withInputRetry` — the loop that puts a credential prompt back up when the + * far end rejects what it was given. + * + * Its own file because `mock.module` registrations last for the process, and + * `bun test --parallel` puts several files in each worker — a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +let answers: string[] = []; +let cancelPrompt = false; + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../../lib/prompts.ts", () => ({ + password: async () => { + if (cancelPrompt) throw new UserAbortError(); + return answers.shift() ?? ""; + }, + text: async () => answers.shift() ?? "", + confirm: async () => true, + multiselect: async () => [], + select: async () => "", + editor: async () => "{}", + note: () => {}, +})); + +const { withInputRetry } = await import("./input-retry.ts"); +const { promptDbUrl } = await import("../export/db-options.ts"); +const { setAssumeYes } = await import("./assume-yes.ts"); + +const captured = useCaptureLog(); + +const CONFIG = { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", +} as const; + +const FIRST = "libsql://typo.turso.io?authToken=t"; +const SECOND = "libsql://right.turso.io?authToken=t"; + +const rejected = () => new CliError("Could not reach it", { code: ERROR_CODE.USAGE_ERROR }); + +let originalMode: Mode; + +beforeAll(() => { + originalMode = getMode(); +}); + +afterAll(() => { + setMode(originalMode); +}); + +beforeEach(() => { + setMode("human"); + setAssumeYes(false); + answers = []; + cancelPrompt = false; +}); + +describe("withInputRetry", () => { + test("returns the first result without prompting when the work succeeds", async () => { + const seen: string[] = []; + + const { value, input } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + seen.push(url); + return Promise.resolve("rows"); + }, + ); + + expect(value).toBe("rows"); + expect(input).toBe(FIRST); + expect(seen).toEqual([FIRST]); + }); + + test("asks again after a failure and runs with the new input", async () => { + answers = [SECOND]; + const seen: string[] = []; + + const { value } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + seen.push(url); + if (url === FIRST) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(value).toBe("rows"); + expect(seen).toEqual([FIRST, SECOND]); + // The operator has to be told what was wrong with a string they cannot see. + expect(captured.err).toContain("Could not reach it"); + }); + + // Later steps run against the credential that worked, not the one first tried + // — a Firebase export reads its project id off the key that Google accepted. + test("reports the input that finally worked", async () => { + answers = [SECOND]; + + const { input } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + if (url === FIRST) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(input).toBe(SECOND); + }); + + test("keeps asking until an input works", async () => { + answers = [FIRST, FIRST, SECOND]; + let attempts = 0; + + await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + attempts++; + if (url !== SECOND) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(attempts).toBe(4); + }); + + // Cancelling the prompt is an answer: it ends the command rather than looping + // on a question the operator has already declined. + test("lets a cancelled prompt out of the loop", async () => { + cancelPrompt = true; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + throw rejected(); + }, + ), + ).rejects.toThrow(UserAbortError); + }); + + test("throws without prompting when there is nobody to ask", async () => { + setMode("agent"); + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + + // `-y` is a human on a TTY who could be asked and said not to. Agent mode + // cannot reach the prompt at all; this one can and declines to, so it needs + // its own check rather than riding on the mode assertion above. + test("throws without prompting when `-y` said not to ask", async () => { + setAssumeYes(true); + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + + // A bug inside the work, or an interrupt, is not a wrong answer to a prompt. + test("does not retry an error the database layer did not raise", async () => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw new TypeError("undefined is not a function"); + }, + ), + ).rejects.toThrow(TypeError); + + expect(attempts).toBe(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts new file mode 100644 index 000000000..13a3604b7 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -0,0 +1,63 @@ +/** + * Retrying the *answer*, not the request. + * + * Distinct from `retry.ts`, which re-sends an identical request after a 429: + * there the request was right and the server was busy. Here the request was + * fine and the input was wrong, so nothing changes until the operator supplies + * something better. + * + * Every credential a migration takes — a connection string, a Firebase service + * account key, an Auth0 client secret — is long, pasted by hand, masked as it + * is typed, and wrong in ways nothing local can check: a typo'd host, an + * expired token, a key that was revoked, the right server but the wrong + * database. Only the remote end can say, and by then the operator has already + * answered every other question the command asked. Ending there charges them a + * full re-run for one line they could not see. + */ + +import { CliError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { isAssumeYes } from "./assume-yes.ts"; + +/** + * Runs `work`, and on failure asks for the input again and runs it once more. + * + * Keep `work` to the step that *proves* the input — the connection, the token + * exchange. Everything inside runs again on each attempt, so work that has + * already written a file, or a long fetch the credential has already been + * accepted for, does not belong in here. + * + * `-y`, agent mode and a non-TTY get the failure unchanged: there is nobody to + * ask, and a loop that cannot prompt is a loop that cannot end. A cancelled + * prompt throws {@link UserAbortError}, which is not a `CliError` and so leaves + * the loop — declining the question is an answer. + * + * @param input - What to try first: a flag, an environment value, or the + * answer to the prompt the caller has already put up. + * @param reprompt - Asks for a replacement. Called once per failure. + * @param work - The step the input has to survive. + * @returns The result, and the input that produced it — which is not `input` + * when it took a retry, and later steps need the one that worked. + */ +export async function withInputRetry( + input: I, + reprompt: () => Promise, + work: (input: I) => Promise, +): Promise<{ value: T; input: I }> { + let candidate = input; + + for (;;) { + try { + return { value: await work(candidate), input: candidate }; + } catch (error) { + // Everything these steps raise for a bad credential is a CliError + // carrying its own explanation; anything else (an interrupt, a bug) is + // not ours to retry. + if (!(error instanceof CliError) || !isHuman() || isAgent() || isAssumeYes()) throw error; + + log.error(error.message); + candidate = await reprompt(); + } + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/instance.test.ts b/packages/cli-core/src/commands/migrate/lib/instance.test.ts new file mode 100644 index 000000000..ef95b0b92 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/instance.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, test } from "bun:test"; +import { + DEV_USER_LIMIT, + detectInstanceType, + getDefaultConcurrencyLimit, + getDefaultRateLimit, + getRetryDelay, + resolveLimits, +} from "./instance.ts"; + +describe("detectInstanceType", () => { + test.each([ + ["sk_live_abc123", "prod"], + ["sk_test_abc123", "dev"], + ["sk_something_else", "dev"], + ["nonsense", "dev"], + ])("%s -> %s", (key, expected) => { + expect(detectInstanceType(key)).toBe(expected as "dev" | "prod"); + }); +}); + +describe("default limits", () => { + test.each([ + ["prod", 100], + ["dev", 10], + ])("%s instances get %i req/s", (instanceType, expected) => { + expect(getDefaultRateLimit(instanceType as "dev" | "prod")).toBe(expected); + }); + + test.each([ + [100, 9], + [10, 1], + [1, 1], + ])("a %i req/s limit yields %i concurrent calls", (rateLimit, expected) => { + expect(getDefaultConcurrencyLimit(rateLimit)).toBe(expected); + }); + + test("development instances default to 100 users", () => { + expect(DEV_USER_LIMIT).toBe(100); + }); +}); + +describe("resolveLimits", () => { + test("derives both limits from the key when nothing is overridden", () => { + expect(resolveLimits("sk_live_x", {})).toEqual({ + instanceType: "prod", + rateLimit: 100, + concurrencyLimit: 9, + }); + }); + + test("honours environment overrides", () => { + expect( + resolveLimits("sk_test_x", { + CLERK_MIGRATE_RATE_LIMIT: "50", + CLERK_MIGRATE_CONCURRENCY_LIMIT: "4", + }), + ).toEqual({ instanceType: "dev", rateLimit: 50, concurrencyLimit: 4 }); + }); + + test("derives concurrency from an overridden rate limit", () => { + expect(resolveLimits("sk_test_x", { CLERK_MIGRATE_RATE_LIMIT: "200" }).concurrencyLimit).toBe( + 19, + ); + }); + + test.each([["0"], ["-5"], ["fast"], [""]])( + "ignores the unusable override %p in favour of the default", + (value) => { + expect(resolveLimits("sk_test_x", { CLERK_MIGRATE_RATE_LIMIT: value }).rateLimit).toBe(10); + }, + ); +}); + +describe("getRetryDelay", () => { + test.each([ + [undefined, 10_000, 10_000, 10], + [15, 10_000, 15_000, 15], + [1, 10_000, 1000, 1], + ])("Retry-After %p -> %i ms", (retryAfter, fallback, delayMs, delaySeconds) => { + expect(getRetryDelay(retryAfter, fallback)).toEqual({ delayMs, delaySeconds }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/instance.ts b/packages/cli-core/src/commands/migrate/lib/instance.ts new file mode 100644 index 000000000..24f222a9d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/instance.ts @@ -0,0 +1,101 @@ +/** + * Instance-type detection and the throughput limits that follow from it. + * + * Ported from the standalone migration-tool's `src/envs-constants.ts`, minus + * its dotenv/Zod env bootstrap: the secret key arrives from + * `resolveBapiSecretKey`, and only the two override knobs read the environment. + */ + +/** + * The user limit a development instance is created with. + * + * Only a default: Clerk raises it per instance on request, and the real value + * (`max_allowed_users`) is not served by BAPI, DAPI or FAPI — only by Clerk's + * internal staff API. So this is a number to warn against, never one to refuse + * an import over; the instance in front of you may be allowed far more. + * Production instances have no limit at all. + */ +export const DEV_USER_LIMIT = 100; + +/** How many times a 429 is retried before the user is recorded as failed. */ +export const MAX_RETRIES = 5; + +/** Fallback backoff when a 429 response carries no `Retry-After`. */ +export const RETRY_DELAY_MS = 10_000; + +export type InstanceType = "dev" | "prod"; + +/** + * Derives the instance type from the secret key's prefix. + * + * @example detectInstanceType("sk_live_xxx") // "prod" + * @example detectInstanceType("sk_test_xxx") // "dev" + */ +export function detectInstanceType(secretKey: string): InstanceType { + return secretKey.split("_")[1] === "live" ? "prod" : "dev"; +} + +/** + * Clerk's documented `POST /v1/users` rate limits, as requests per second: + * 1000 per 10s for production, 100 per 10s for development. + */ +export function getDefaultRateLimit(instanceType: InstanceType): number { + return instanceType === "prod" ? 100 : 10; +} + +/** + * Concurrency that saturates ~95% of the rate limit, assuming ~100ms of API + * latency per call: N concurrent requests at 100ms each yield N * 10 req/s. + * + * Override with `CLERK_MIGRATE_CONCURRENCY_LIMIT` when actual latency differs. + */ +export function getDefaultConcurrencyLimit(rateLimit: number): number { + return Math.max(1, Math.floor(rateLimit * 0.095)); +} + +export type ResolvedLimits = { + instanceType: InstanceType; + rateLimit: number; + concurrencyLimit: number; +}; + +/** + * Resolves throughput limits for a run: defaults from the detected instance + * type, each overridable by an environment variable. + * + * Non-numeric or non-positive overrides are ignored in favour of the default + * rather than failing the run — an unusable limit would stall the import. + */ +export function resolveLimits( + secretKey: string, + env: Record = process.env, +): ResolvedLimits { + const instanceType = detectInstanceType(secretKey); + + const positive = (value: string | undefined): number | undefined => { + if (!value) return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; + }; + + const rateLimit = positive(env.CLERK_MIGRATE_RATE_LIMIT) ?? getDefaultRateLimit(instanceType); + const concurrencyLimit = + positive(env.CLERK_MIGRATE_CONCURRENCY_LIMIT) ?? getDefaultConcurrencyLimit(rateLimit); + + return { instanceType, rateLimit, concurrencyLimit }; +} + +/** + * Backoff for a 429, preferring the server's `Retry-After` over the default. + * + * @param retryAfterSeconds - `Retry-After` value from the response, if present. + * @param defaultDelayMs - Fallback delay in milliseconds. + */ +export function getRetryDelay( + retryAfterSeconds: number | undefined, + defaultDelayMs: number, +): { delayMs: number; delaySeconds: number } { + const delayMs = retryAfterSeconds ? retryAfterSeconds * 1000 : defaultDelayMs; + const delaySeconds = retryAfterSeconds || defaultDelayMs / 1000; + return { delayMs, delaySeconds }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts b/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts new file mode 100644 index 000000000..7aba346fe --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/log-dir-prompt.test.ts @@ -0,0 +1,141 @@ +/** + * The one path `logger.test.ts` cannot cover: `ensureLogDir` actually asking. + * + * Its own file because `mock.module` registrations last for the process, and + * `bun test --parallel` puts several files in each worker — a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +type TextConfig = { message: string; default?: string; placeholder?: string }; + +let answer = ""; +const mockText = mock(async (_config: TextConfig) => answer); + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../../lib/prompts.ts", () => ({ + text: (...args: unknown[]) => mockText(...(args as [TextConfig])), + confirm: async () => true, + multiselect: async () => [], + password: async () => "", + editor: async () => "{}", +})); + +const { _resetLogDir, ensureLogDir } = await import("./logger.ts"); +const { loadSettings, saveSettings } = await import("./settings.ts"); +const { setAssumeYes } = await import("./assume-yes.ts"); + +useCaptureLog(); + +let workDir: string; +let configDir: string; +let originalCwd: string; +let originalMode: Mode; +let originalEnv: string | undefined; + +beforeAll(() => { + originalCwd = process.cwd(); + originalMode = getMode(); + originalEnv = process.env.CLERK_MIGRATE_LOG_DIR; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-cfg-")); + _setConfigDir(configDir); + process.chdir(workDir); +}); + +afterAll(() => { + setMode(originalMode); + if (originalEnv === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; + else process.env.CLERK_MIGRATE_LOG_DIR = originalEnv; + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + _resetLogDir(); + delete process.env.CLERK_MIGRATE_LOG_DIR; + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + mockText.mockClear(); + answer = ""; + setMode("human"); + setAssumeYes(false); +}); + +afterEach(() => _resetLogDir()); + +describe("ensureLogDir asks once", () => { + test("saves the answer, so the next run does not ask", async () => { + answer = "./migration-logs"; + + expect(await ensureLogDir()).toBe(path.join(workDir, "migration-logs")); + expect(await loadSettings()).toMatchObject({ logDir: "./migration-logs" }); + + _resetLogDir(); + expect(await ensureLogDir()).toBe(path.join(workDir, "migration-logs")); + expect(mockText).toHaveBeenCalledTimes(1); + }); + + test("offers ./logs as the default", async () => { + await ensureLogDir(); + expect(mockText.mock.calls[0]?.[0]).toMatchObject({ default: "./logs" }); + }); + + // Enter on the prompt is an answer, not a skip: it settles the question so + // the next run goes straight to importing. + test("treats an empty answer as ./logs and remembers it", async () => { + answer = " "; + + expect(await ensureLogDir()).toBe(path.join(workDir, "logs")); + expect(await loadSettings()).toMatchObject({ logDir: "./logs" }); + }); + + test("leaves the project's other settings alone", async () => { + await saveSettings({ transformer: "firebase", file: "users.json" }); + answer = "./audit"; + + await ensureLogDir(); + + expect(await loadSettings()).toEqual({ + transformer: "firebase", + file: "users.json", + logDir: "./audit", + }); + }); +}); + +describe("ensureLogDir with `-y`", () => { + // Agent mode cannot reach this prompt; `-y` is a human on a TTY who could be + // asked and said not to be, so it needs its own check. + test("takes ./logs without asking", async () => { + setAssumeYes(true); + + expect(await ensureLogDir()).toBe(path.join(workDir, "logs")); + expect(mockText).not.toHaveBeenCalled(); + }); + + // Landing on a default is not a choice, and recording one would retire the + // question for a human who never saw it. + test("saves nothing, so the next interactive run still asks", async () => { + setAssumeYes(true); + await ensureLogDir(); + expect((await loadSettings()).logDir).toBeUndefined(); + + _resetLogDir(); + setAssumeYes(false); + answer = "./audit"; + + expect(await ensureLogDir()).toBe(path.join(workDir, "audit")); + expect(mockText).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.test.ts b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts new file mode 100644 index 000000000..cc7587453 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/log-files.test.ts @@ -0,0 +1,203 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { classifyLogFile, findLogFile, formatSize, listLogFiles, readNdjson } from "./log-files.ts"; +import { getLogDir } from "./logger.ts"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logfiles-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); +}); + +/** Writes a log file with one NDJSON line per entry. */ +function writeLog(name: string, entries: unknown[]): string { + fs.mkdirSync(getLogDir(), { recursive: true }); + const filePath = path.join(getLogDir(), name); + fs.writeFileSync(filePath, entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n"); + return filePath; +} + +describe("classifyLogFile", () => { + test.each([ + ["import-2026-01-01T12-00-00.log", "import", "2026-01-01T12-00-00"], + ["delete-2026-01-01T12-00-00.log", "delete", "2026-01-01T12-00-00"], + ["export-2026-01-01T12-00-00.log", "export", "2026-01-01T12-00-00"], + // Written by the standalone tool and by earlier CLI builds. + ["migration-2026-01-01T12-00-00.log", "import", "2026-01-01T12-00-00"], + ["user-deletion-2026-01-01T12-00-00.log", "delete", "2026-01-01T12-00-00"], + ])("%s is a %s log from %s", (name, kind, timestamp) => { + expect(classifyLogFile(name)).toEqual({ kind: kind as never, timestamp }); + }); + + test.each([["random.log"], ["import.log"], ["notes.txt"]])( + "%s is unrecognized rather than a parse failure", + (name) => { + expect(classifyLogFile(name)).toEqual({ kind: "unknown", timestamp: "" }); + }, + ); +}); + +describe("listLogFiles", () => { + test("returns nothing when the directory does not exist", () => { + expect(fs.existsSync(getLogDir())).toBe(false); + expect(listLogFiles()).toEqual([]); + }); + + test("returns nothing when the directory is empty", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + expect(listLogFiles()).toEqual([]); + }); + + test("reports kind, timestamp, size and entry count per file", () => { + writeLog("import-2026-01-01T12-00-00.log", [{ userId: "u1" }, { userId: "u2" }]); + + const [file] = listLogFiles(); + expect(file).toMatchObject({ + name: "import-2026-01-01T12-00-00.log", + kind: "import", + timestamp: "2026-01-01T12-00-00", + entryCount: 2, + }); + expect(file?.sizeBytes).toBeGreaterThan(0); + }); + + test("ignores files that are not logs", () => { + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); + fs.writeFileSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"), "[]"); + fs.writeFileSync(path.join(getLogDir(), "notes.txt"), "hi"); + + expect(listLogFiles().map((file) => file.name)).toEqual(["import-2026-01-01T12-00-00.log"]); + }); + + test("ignores subdirectories", () => { + fs.mkdirSync(path.join(getLogDir(), "nested.log"), { recursive: true }); + expect(listLogFiles()).toEqual([]); + }); + + test("returns the newest run first", () => { + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-03-01T12-00-00.log", [{ a: 1 }]); + writeLog("import-2026-02-01T12-00-00.log", [{ a: 1 }]); + + expect(listLogFiles().map((file) => file.timestamp)).toEqual([ + "2026-03-01T12-00-00", + "2026-02-01T12-00-00", + "2026-01-01T12-00-00", + ]); + }); + + // Sorting on the filename would put every "import-" ahead of every + // "migration-", regardless of when the runs actually happened. + test("orders by timestamp across log kinds, not by the name's prefix", () => { + writeLog("import-2026-01-30T17-02-51.log", [{ a: 1 }]); + writeLog("export-2026-02-01T09-14-22.log", [{ a: 1 }]); + + expect(listLogFiles().map((file) => file.kind)).toEqual(["export", "import"]); + }); + + test("sorts unrecognized names last", () => { + writeLog("something-else.log", [{ a: 1 }]); + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); + + expect(listLogFiles().map((file) => file.name)).toEqual([ + "import-2026-01-01T12-00-00.log", + "something-else.log", + ]); + }); + + test("does not count blank lines as entries", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync(path.join(getLogDir(), "migration-x.log"), '{"a":1}\n\n\n{"b":2}\n'); + expect(listLogFiles()[0]?.entryCount).toBe(2); + }); + + test("lists a log whose name does not match the convention", () => { + writeLog("something-else.log", [{ a: 1 }]); + expect(listLogFiles()[0]).toMatchObject({ kind: "unknown", timestamp: "", entryCount: 1 }); + }); +}); + +describe("findLogFile", () => { + beforeEach(() => { + writeLog("import-2026-01-01T12-00-00.log", [{ a: 1 }]); + }); + + test("finds a log by name", () => { + expect(findLogFile("import-2026-01-01T12-00-00.log")?.entryCount).toBe(1); + }); + + test("accepts a path and matches on the basename", () => { + expect(findLogFile("./logs/import-2026-01-01T12-00-00.log")?.entryCount).toBe(1); + }); + + test("returns nothing for a name that is not there", () => { + expect(findLogFile("migration-nope.log")).toBeUndefined(); + }); +}); + +describe("readNdjson", () => { + test("parses one entry per line", () => { + const file = writeLog("migration-a.log", [{ userId: "u1" }, { userId: "u2" }]); + const { entries, errors } = readNdjson(file); + + expect(entries).toEqual([{ userId: "u1" }, { userId: "u2" }]); + expect(errors).toEqual([]); + }); + + test("skips blank lines without reporting them", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + const file = path.join(getLogDir(), "migration-b.log"); + fs.writeFileSync(file, '\n{"a":1}\n \n{"b":2}\n\n'); + + const { entries, errors } = readNdjson(file); + expect(entries).toHaveLength(2); + expect(errors).toEqual([]); + }); + + // A run killed mid-write leaves one truncated line; the complete entries + // before it are still worth having, so the read reports rather than aborts. + test("reports a malformed line by number and keeps the rest", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + const file = path.join(getLogDir(), "migration-c.log"); + fs.writeFileSync(file, '{"a":1}\n{"b":\n{"c":3}\n'); + + const { entries, errors } = readNdjson(file); + expect(entries).toEqual([{ a: 1 }, { c: 3 }]); + expect(errors).toHaveLength(1); + expect(errors[0]?.line).toBe(2); + }); + + test("numbers lines from one, counting blanks", () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + const file = path.join(getLogDir(), "migration-d.log"); + fs.writeFileSync(file, '\n\n{"a":1}\nnot json\n'); + + expect(readNdjson(file).errors[0]?.line).toBe(4); + }); +}); + +describe("formatSize", () => { + test.each([ + [0, "0 B"], + [512, "512 B"], + [1024, "1.0 KB"], + [1536, "1.5 KB"], + [1024 * 1024, "1.0 MB"], + ])("%i bytes reads as %s", (bytes, expected) => { + expect(formatSize(bytes)).toBe(expected); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/log-files.ts b/packages/cli-core/src/commands/migrate/lib/log-files.ts new file mode 100644 index 000000000..af87c6712 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/log-files.ts @@ -0,0 +1,153 @@ +/** + * Enumerating and reading the cwd-relative `./logs/` directory. + * + * The standalone migration-tool re-read the directory inside both of its log + * commands to build their pickers. `list`, `clean` and `convert` all share + * this instead, which is also what makes `logs list` nearly free. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { getLogDir } from "./logger.ts"; + +/** + * The run that produced a log file, read from its filename prefix. + * + * The kinds are the command names — `migrate import` writes `import-*.log` — + * so a listing points straight at the command that produced each line. + */ +export type LogKind = "export" | "import" | "delete" | "unknown"; + +const FILENAME_PATTERN = /^(export|import|delete|migration|user-deletion)-(.+)\.log$/; + +/** + * `migration-` and `user-deletion-` are the names the standalone tool and + * earlier CLI builds wrote. They still classify, so a directory of older logs + * lists and converts rather than reading as "unknown". + */ +const KIND_BY_PREFIX: Record = { + export: "export", + import: "import", + delete: "delete", + migration: "import", + "user-deletion": "delete", +}; + +export type LogFile = { + name: string; + path: string; + kind: LogKind; + /** Timestamp as recorded in the filename, or `""` for an unrecognized name. */ + timestamp: string; + sizeBytes: number; + /** Non-empty NDJSON lines, malformed ones included. */ + entryCount: number; +}; + +export function classifyLogFile(name: string): { kind: LogKind; timestamp: string } { + const match = FILENAME_PATTERN.exec(name); + if (!match) return { kind: "unknown", timestamp: "" }; + return { kind: KIND_BY_PREFIX[match[1] as string] ?? "unknown", timestamp: match[2] as string }; +} + +function countEntries(filePath: string): number { + try { + return fs + .readFileSync(filePath, "utf-8") + .split("\n") + .filter((line) => line.trim().length > 0).length; + } catch { + // An unreadable file still belongs in the listing; its count is unknown. + return 0; + } +} + +/** + * Every `.log` file in `./logs/`, newest first. + * + * @returns An empty array when the directory is absent — "no logs yet" and "no + * logs directory" are the same thing to every caller. + */ +export function listLogFiles(): LogFile[] { + const dir = getLogDir(); + if (!fs.existsSync(dir)) return []; + + const files: LogFile[] = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith(".log")) continue; + + const filePath = path.join(dir, name); + let stats: fs.Stats; + try { + stats = fs.statSync(filePath); + } catch { + continue; + } + if (!stats.isFile()) continue; + + files.push({ + name, + path: filePath, + ...classifyLogFile(name), + sizeBytes: stats.size, + entryCount: countEntries(filePath), + }); + } + + // Sort on the timestamp, not the filename: the kind prefix sorts first in a + // filename comparison, which would interleave a run from January ahead of one + // from March purely because "import" > "export". Timestamps are + // ISO-ish and zero-padded, so lexical order is chronological. Names without + // one sort last, then alphabetically. + return files.sort( + (a, b) => b.timestamp.localeCompare(a.timestamp) || a.name.localeCompare(b.name), + ); +} + +/** Resolves a user-supplied name or path to a log file in `./logs/`. */ +export function findLogFile(nameOrPath: string): LogFile | undefined { + const wanted = path.basename(nameOrPath); + return listLogFiles().find((file) => file.name === wanted); +} + +export type NdjsonLineError = { + /** 1-indexed line number in the source file. */ + line: number; + message: string; +}; + +export type NdjsonReadResult = { + entries: unknown[]; + errors: NdjsonLineError[]; +}; + +/** + * Parses an NDJSON file line by line. + * + * Malformed lines are collected with their line numbers rather than aborting + * the read: a run killed mid-write leaves one truncated final line, and the + * hundreds of complete entries before it are still worth having. + */ +export function readNdjson(filePath: string): NdjsonReadResult { + const entries: unknown[] = []; + const errors: NdjsonLineError[] = []; + + const lines = fs.readFileSync(filePath, "utf-8").split("\n"); + for (const [index, line] of lines.entries()) { + if (line.trim().length === 0) continue; + try { + entries.push(JSON.parse(line)); + } catch (error) { + errors.push({ line: index + 1, message: (error as Error).message }); + } + } + + return { entries, errors }; +} + +/** Human-readable file size. */ +export function formatSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} diff --git a/packages/cli-core/src/commands/migrate/lib/logger.test.ts b/packages/cli-core/src/commands/migrate/lib/logger.test.ts new file mode 100644 index 000000000..e9176df9a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/logger.test.ts @@ -0,0 +1,205 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + _resetLogDir, + DEFAULT_LOG_DIR, + ensureLogDir, + errorLogger, + getDateTimeStamp, + getLogDir, + getLogFilePath, + importLogger, + resolveLogDir, + validationLogger, +} from "./logger.ts"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { MIGRATE_ENV_FILE } from "./env-file.ts"; +import { loadSettings, saveSettings } from "./settings.ts"; + +const DATE_TIME = "2026-01-01T12:00:00"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + // realpath so the comparison against process.cwd() survives macOS's + // /var -> /private/var symlink. + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logger-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + _resetLogDir(); + fs.rmSync(getLogDir(), { recursive: true, force: true }); +}); + +function readEntries(): Record[] { + return fs + .readFileSync(getLogFilePath("import", DATE_TIME), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); +} + +describe("log file paths", () => { + test("writes under the current working directory, not next to the binary", () => { + expect(getLogDir()).toBe(path.join(workDir, "logs")); + }); + + test("replaces the timestamp's colons so the name is valid on Windows", () => { + expect(path.basename(getLogFilePath("import", DATE_TIME))).toBe( + "import-2026-01-01T12-00-00.log", + ); + }); + + test("getDateTimeStamp drops milliseconds", () => { + expect(getDateTimeStamp()).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$/); + }); +}); + +describe("log writers", () => { + test("creates the logs directory on first write", () => { + expect(fs.existsSync(getLogDir())).toBe(false); + importLogger({ userId: "u1", status: "success", clerkUserId: "user_x" }, DATE_TIME); + expect(fs.existsSync(getLogDir())).toBe(true); + }); + + test("appends one NDJSON line per entry", () => { + importLogger({ userId: "u1", status: "success", clerkUserId: "user_x" }, DATE_TIME); + importLogger({ userId: "u2", status: "error", error: "boom", code: "422" }, DATE_TIME); + + const entries = readEntries(); + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ userId: "u1", status: "success", clerkUserId: "user_x" }); + expect(entries[1]).toEqual({ userId: "u2", status: "error", error: "boom", code: "422" }); + }); + + test("writes one line per error in a failed payload", () => { + errorLogger( + { + userId: "u1", + status: "422", + errors: [ + { code: "a", message: "short a", longMessage: "long a" }, + { code: "b", message: "short b" }, + ], + }, + DATE_TIME, + ); + + const entries = readEntries(); + expect(entries).toHaveLength(2); + expect(entries[0]).toMatchObject({ type: "User Creation Error", error: "long a" }); + // Falls back to `message` when the API omitted a long form. + expect(entries[1]).toMatchObject({ error: "short b" }); + }); + + test("records validation failures in the same run log", () => { + validationLogger( + { error: "missing identifier", path: ["email"], userId: "u3", row: 4 }, + DATE_TIME, + ); + expect(readEntries()[0]).toEqual({ + userId: "u3", + status: "fail", + error: "missing identifier", + path: ["email"], + row: 4, + }); + }); +}); + +describe("resolving the log directory", () => { + let configDir: string; + let originalMode: Mode; + let originalEnv: string | undefined; + + beforeAll(() => { + originalMode = getMode(); + originalEnv = process.env.CLERK_MIGRATE_LOG_DIR; + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logdir-cfg-")); + _setConfigDir(configDir); + }); + + afterAll(() => { + setMode(originalMode); + if (originalEnv === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; + else process.env.CLERK_MIGRATE_LOG_DIR = originalEnv; + _setConfigDir(undefined); + fs.rmSync(configDir, { recursive: true, force: true }); + }); + + beforeEach(() => { + delete process.env.CLERK_MIGRATE_LOG_DIR; + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.rmSync(path.join(workDir, MIGRATE_ENV_FILE), { force: true }); + setMode("agent"); + }); + + test("falls back to ./logs when nothing has chosen one", async () => { + expect(await resolveLogDir()).toBe(path.join(workDir, "logs")); + }); + + test("prefers the saved setting over the default", async () => { + await saveSettings({ logDir: "./audit" }); + expect(await resolveLogDir()).toBe(path.join(workDir, "audit")); + }); + + // A variable exported for one shell is the narrower statement of the two. + test("prefers the environment over the saved setting", async () => { + await saveSettings({ logDir: "./audit" }); + process.env.CLERK_MIGRATE_LOG_DIR = "./from-env"; + + expect(await resolveLogDir()).toBe(path.join(workDir, "from-env")); + }); + + test("reads the migration's own env file", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "CLERK_MIGRATE_LOG_DIR=./from-file\n"); + expect(await resolveLogDir()).toBe(path.join(workDir, "from-file")); + }); + + // Every synchronous write reads the settled value, so resolving is what makes + // the log files land anywhere but ./logs. + test("settles the directory the log writers use", async () => { + await saveSettings({ logDir: "./audit" }); + await resolveLogDir(); + + expect(getLogFilePath("import", DATE_TIME)).toBe( + path.join(workDir, "audit", `import-${DATE_TIME.replace(/:/g, "-")}.log`), + ); + }); + + describe("ensureLogDir", () => { + test("takes the default without saving it when nobody can be asked", async () => { + expect(await ensureLogDir()).toBe(path.join(workDir, DEFAULT_LOG_DIR)); + // Nothing saved: the question stays open for the first interactive run. + expect(await loadSettings()).toEqual({}); + }); + + test("does not ask once the setting is saved", async () => { + await saveSettings({ logDir: "./audit" }); + setMode("human"); + + // Reaching the prompt in a test without a TTY throws, so returning is the + // assertion. + expect(await ensureLogDir()).toBe(path.join(workDir, "audit")); + }); + + test("does not ask when the environment already answers", async () => { + process.env.CLERK_MIGRATE_LOG_DIR = "./from-env"; + setMode("human"); + + expect(await ensureLogDir()).toBe(path.join(workDir, "from-env")); + expect(await loadSettings()).toEqual({}); + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/logger.ts b/packages/cli-core/src/commands/migrate/lib/logger.ts new file mode 100644 index 000000000..3eca2cdce --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/logger.ts @@ -0,0 +1,235 @@ +/** + * NDJSON migration logs. + * + * Ported from the standalone migration-tool's `src/logger.ts`, with one + * behavioural fix: logs are written relative to the current working directory + * rather than to `__dirname/../logs`. In a `bun build --compile` binary there + * is no source tree next to the executable, so the original path would land + * logs inside wherever the binary happens to live. + * + * Writes are synchronous appends so a run interrupted with Ctrl-C still leaves + * a complete record of everything already processed. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { log } from "../../../lib/log.ts"; +import { text } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { envNames, findSetting } from "../settings/registry.ts"; +import { isAssumeYes } from "./assume-yes.ts"; +import { findMigrateEnvValue } from "./env-file.ts"; +import { loadSettings, saveSettings } from "./settings.ts"; +import type { + DeleteLogEntry, + ErrorLog, + ErrorPayload, + ExportLogEntry, + ImportLogEntry, + ValidationErrorPayload, +} from "../types.ts"; + +/** Where logs go when nobody has said otherwise. */ +export const DEFAULT_LOG_DIR = "./logs"; + +/** + * The directory settled for this process, once something has settled it. + * + * The log writers are synchronous — a run interrupted with Ctrl-C has to leave + * a complete record of what it already processed — but resolving the directory + * reads the config, the env files and possibly the operator. So resolution + * happens once, up front, and every synchronous write reads the answer from + * here. {@link resolveLogDir} and {@link ensureLogDir} are the only writers. + */ +let settled: string | undefined; + +function remember(dir: string): string { + settled = path.resolve(process.cwd(), dir); + return settled; +} + +/** Forgets the settled directory. Tests only — each one resolves its own. */ +export function _resetLogDir(): void { + settled = undefined; +} + +/** + * Absolute path of the log directory. + * + * Falls back to `./logs` when nothing has resolved yet, so a caller that + * forgets to is wrong about *where*, never broken. + */ +export function getLogDir(): string { + return settled ?? path.resolve(process.cwd(), DEFAULT_LOG_DIR); +} + +/** The `log-dir` setting, which owns both the env var and the config key. */ +const LOG_DIR = findSetting("log-dir") as NonNullable>; + +/** + * The directory the operator has already chosen, by either route. + * + * The environment wins over the remembered value, matching every other setting + * the CLI resolves: a variable exported for one shell is the narrower, more + * deliberate statement of the two. + */ +async function chosenLogDir(): Promise { + const located = await findMigrateEnvValue(envNames(LOG_DIR)); + if (located?.value) return located.value; + return (await loadSettings()).logDir; +} + +/** + * Settles the log directory without asking: environment, then the saved + * setting, then `./logs`. + * + * For the read-only log commands. Landing on the default here does not save it + * — an operator who has only ever *listed* logs has still made no choice, and + * recording one on their behalf would skip the question forever. + */ +export async function resolveLogDir(): Promise { + return remember((await chosenLogDir()) ?? DEFAULT_LOG_DIR); +} + +/** + * Settles the log directory, asking a human who has not chosen yet. + * + * Migration logs are the only record of which users landed and which failed, + * and `migrate delete` reads them to undo a run — so where they go is worth one + * question, once per project, before the first thing is written. The answer is + * saved, so it is asked once and never again. + * + * `-y`, agent mode and a non-TTY take the default rather than a prompt they + * cannot answer, and save nothing: the question stays open for the first + * interactive run. + */ +export async function ensureLogDir(): Promise { + const chosen = await chosenLogDir(); + if (chosen) return remember(chosen); + if (!isHuman() || isAgent() || isAssumeYes()) return remember(DEFAULT_LOG_DIR); + + const answer = await text({ + message: "Where should migration logs be saved?", + default: DEFAULT_LOG_DIR, + placeholder: DEFAULT_LOG_DIR, + }); + const dir = answer.trim() || DEFAULT_LOG_DIR; + + await saveSettings({ ...(await loadSettings()), logDir: dir }); + log.info( + `Saving migration logs to ${dir}. Change it with \`clerk migrate settings set log-dir \`.`, + ); + + return remember(dir); +} + +/** + * Settles where this run's logs go, and stamps it. + * + * Every command that writes a log starts here rather than calling + * {@link getDateTimeStamp} directly, so there is no path on which a log file is + * named before its directory has been resolved. + */ +export async function startLogging(): Promise { + await ensureLogDir(); + return getDateTimeStamp(); +} + +/** + * The log directory the way the user would type it from here. + * + * Relative (`./logs`) when it sits under the current directory, absolute when + * it does not — a path the reader can paste either way, without a home + * directory's worth of prefix on the common case. + */ +export function displayLogDir(): string { + const dir = getLogDir(); + const relative = path.relative(process.cwd(), dir); + if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) return dir; + return `.${path.sep}${relative}`; +} + +/** Absolute path of the log file a run with this timestamp writes to. */ +export function getLogFilePath(logFile: string, dateTime: string): string { + // Colons are illegal in Windows filenames, and the timestamp is an ISO string. + return path.join(getLogDir(), `${logFile}-${dateTime}.log`.replace(/:/g, "-")); +} + +/** ISO timestamp without milliseconds — the log-file name discriminator. */ +export function getDateTimeStamp(): string { + return new Date().toISOString().split(".")[0] ?? ""; +} + +function appendToLogFile(fullPath: string, entry: unknown): void { + try { + fs.mkdirSync(path.dirname(fullPath), { recursive: true }); + fs.appendFileSync(fullPath, `${JSON.stringify(entry)}\n`); + } catch (error) { + // A broken log destination must not abort an in-flight migration; the run + // is still making real progress against the API. + log.warn(`Could not write migration log: ${(error as Error).message}`); + } +} + +/** Writes each error in a failed API call as its own NDJSON line. */ +export function errorLogger(payload: ErrorPayload, dateTime: string): void { + for (const err of payload.errors) { + const entry: ErrorLog = { + type: "User Creation Error", + userId: payload.userId, + status: payload.status, + error: err.longMessage ?? err.message, + }; + appendToLogFile(getLogFilePath("import", dateTime), entry); + } +} + +/** Writes a user that failed schema validation before any API call. */ +export function validationLogger(payload: ValidationErrorPayload, dateTime: string): void { + appendToLogFile(getLogFilePath("import", dateTime), { + userId: payload.userId, + status: "fail" as const, + error: payload.error, + path: payload.path, + row: payload.row, + }); +} + +/** Writes the outcome of one import attempt. */ +export function importLogger(entry: ImportLogEntry, dateTime: string): void { + appendToLogFile(getLogFilePath("import", dateTime), entry); +} + +/** + * Writes the outcome of one deletion attempt. + * + * A separate `delete-` file rather than another line in the import log: undoing + * a migration is its own run, and mixing the two would make "what did this + * import do" unanswerable after an undo. + */ +export function deleteLogger(entry: DeleteLogEntry, dateTime: string): void { + appendToLogFile(getLogFilePath("delete", dateTime), entry); +} + +/** + * Writes the outcome of exporting one user. + * + * Its own `export-` file for the same reason deletes get theirs: an export is a + * distinct run, and `migrate logs list` reports each kind separately. + */ +export function exportLogger(entry: ExportLogEntry, dateTime: string): void { + appendToLogFile(getLogFilePath("export", dateTime), entry); +} + +/** Writes each error in a failed deletion as its own NDJSON line. */ +export function deleteErrorLogger(payload: ErrorPayload, dateTime: string): void { + for (const err of payload.errors) { + const entry: ErrorLog = { + type: "User Deletion Error", + userId: payload.userId, + status: payload.status, + error: err.longMessage ?? err.message, + }; + appendToLogFile(getLogFilePath("delete", dateTime), entry); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts new file mode 100644 index 000000000..e9b55e8e0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts @@ -0,0 +1,303 @@ +import { describe, expect, test } from "bun:test"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import type { FieldAnalysis } from "./analysis.ts"; +import { buildReadinessReport } from "./readiness.ts"; +import { applyChanges, buildChangePayload, buildSettingChanges } from "./modify-settings.ts"; + +/** Instance settings carrying only the attributes and providers a test names. */ +function settings(config: { + attributes?: Record; + social?: Record; +}): UserSettingsJSON { + return { + attributes: Object.fromEntries( + Object.entries(config.attributes ?? {}).map(([name, value]) => [ + name, + { enabled: value.enabled, required: value.required ?? false }, + ]), + ), + social: config.social ?? {}, + } as unknown as UserSettingsJSON; +} + +function analysis(overrides: Partial & { totalUsers: number }): FieldAnalysis { + return { + identifiers: { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: overrides.totalUsers, + ...overrides.identifiers, + }, + fieldCounts: overrides.fieldCounts ?? {}, + totalUsers: overrides.totalUsers, + }; +} + +/** + * Changes are built from a real report rather than hand-written rows, so a row + * whose `key` stops matching the path table fails here instead of silently + * dropping out of the offer. + */ +function changesFor(input: Parameters[0]) { + return buildSettingChanges(buildReadinessReport(input).blocking); +} + +describe("what gets offered", () => { + test("a required field not every user has is offered as a relaxation", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + + expect(changes).toEqual([ + { + id: "email_address", + label: "Make Email optional at sign-up", + section: "identifiers", + kind: "relax", + writes: [{ path: ["auth_email", "required_for_sign_up"], value: false }], + }, + ]); + }); + + test("a field the file carries but Clerk has switched off is offered as an enable", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { username: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { username: { enabled: false } } }), + }); + + expect(changes).toEqual([ + { + id: "username", + label: "Enable Username", + section: "identifiers", + kind: "enable", + writes: [{ path: ["auth_username", "used_for_sign_up"], value: true }], + }, + ]); + }); + + /** + * Clerk refuses a verifiable attribute that is on with no way to verify it — + * `422 phone_number: verifiable attributes need to have at least one + * verification` — and switching the attribute off empties the strategies, so + * every enable that turned one off has to put one back. + */ + test.each([ + ["phone_number", "auth_phone", "phone_code"], + ["email_address", "auth_email", "email_code"], + ])("enabling %s also restores its verification strategy", (attribute, group, strategy) => { + const carriesIt = attribute === "phone_number" ? { verifiedPhones: 2 } : { verifiedEmails: 2 }; + + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { ...carriesIt, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { [attribute]: { enabled: false } } }), + }); + + expect(changes[0]?.writes).toEqual([ + { path: [group, "used_for_sign_up"], value: true }, + { path: [group, "verification_strategies"], value: [strategy] }, + ]); + expect(buildChangePayload(changes)).toEqual({ + [group]: { used_for_sign_up: true, verification_strategies: [strategy] }, + }); + }); + + // Not verifiable, so no strategy to restore — one write is the whole change. + test("enabling username takes a single write", () => { + const changes = changesFor({ + analysis: analysis({ totalUsers: 2, identifiers: { username: 2 } as never }), + settings: settings({ attributes: { username: { enabled: false } } }), + }); + expect(changes[0]?.writes).toHaveLength(1); + }); + + test("a disabled social provider is offered under Clerk's own strategy name", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_x: { enabled: false } }, + }), + // Supabase calls it `twitter`; the config document calls it `oauth_x`. + providerCounts: { twitter: 2 }, + }); + + expect(changes).toEqual([ + { + id: "twitter", + label: "Enable Twitter (X) sign-in", + section: "social", + kind: "enable", + writes: [{ path: ["connection_oauth_x", "enabled"], value: true }], + }, + ]); + }); + + // "Could not read" is not "switched off", so nothing is flagged and nothing + // is offered — the report already degrades to a coverage-only listing. + test("nothing is offered when the instance settings could not be read", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 1, hasAnyIdentifier: 2 } as never, + }), + settings: null, + }); + + expect(changes).toEqual([]); + }); + + test("nothing is offered when every field is already configured", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + + expect(changes).toEqual([]); + }); +}); + +describe("the payload", () => { + test("collapses changes that share a parent into one branch", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 3 } as never, + fieldCounts: { firstName: 2, lastName: 1 }, + }), + settings: settings({ + attributes: { + email_address: { enabled: true }, + first_name: { enabled: true, required: true }, + last_name: { enabled: true, required: true }, + }, + }), + }); + + expect(buildChangePayload(changes)).toEqual({ + user_model: { first_name: { required: false }, last_name: { required: false } }, + }); + }); + + test("carries only the changes it is given", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 3 } as never, + fieldCounts: { password: 1 }, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + password: { enabled: true, required: true }, + }, + }), + }); + expect(changes.map((change) => change.id)).toEqual(["email_address", "password"]); + + expect(buildChangePayload(changes.filter((change) => change.id === "password"))).toEqual({ + auth_password: { required: false }, + }); + }); + + test("is empty when nothing was selected", () => { + expect(buildChangePayload([])).toEqual({}); + }); +}); + +/** + * The redraw after a write comes from `applyChanges`, not a second fetch: + * Clerk's Frontend API is eventually consistent, so re-reading straight after + * the patch returns the pre-write settings and redraws every row just cleared. + */ +describe("the settings after a write", () => { + /** The two fields a change touches, as `settings()` above builds them. */ + const attr = (value: { enabled: boolean; required: boolean }) => + value as unknown as UserSettingsJSON["attributes"]["email_address"]; + + test("drops the requirement a relaxation removed", () => { + const before = settings({ attributes: { email_address: { enabled: true, required: true } } }); + const input = { + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, + }), + settings: before, + }; + + const after = applyChanges(before, changesFor(input)); + + expect(after?.attributes.email_address).toEqual(attr({ enabled: true, required: false })); + // The report is rebuilt from this, so the row must stop being flagged. + expect(buildReadinessReport({ ...input, settings: after }).blocking).toEqual([]); + }); + + test("turns on what an enable switched on", () => { + const before = settings({ attributes: { username: { enabled: false } } }); + const changes = changesFor({ + analysis: analysis({ totalUsers: 2, identifiers: { username: 2 } as never }), + settings: before, + }); + + expect(applyChanges(before, changes)?.attributes.username).toEqual( + attr({ enabled: true, required: false }), + ); + }); + + test("enables a provider under Clerk's strategy name, not the source platform's", () => { + const before = settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_x: { enabled: false } }, + }); + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: before, + providerCounts: { twitter: 2 }, + }); + + expect(applyChanges(before, changes)?.social).toEqual({ + oauth_x: { enabled: true }, + } as unknown as UserSettingsJSON["social"]); + }); + + test("leaves the settings it was given untouched", () => { + const before = settings({ attributes: { email_address: { enabled: true, required: true } } }); + const changes = changesFor({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, + }), + settings: before, + }); + + applyChanges(before, changes); + + expect(before.attributes.email_address).toMatchObject({ required: true }); + }); + + test("passes null through — unreadable settings flag nothing to change", () => { + expect(applyChanges(null, [])).toBeNull(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts new file mode 100644 index 000000000..8136f6ca1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts @@ -0,0 +1,191 @@ +/** + * Turning a flagged Migration Readiness row into the instance-config change + * that would stop it being flagged. + * + * The report already knows which settings will cost users; without this the + * only way to act on it is to leave the CLI, find the setting in the dashboard, + * and come back. Each change is a single leaf in the Platform API's config + * document, so they compose into one `PATCH` however many the operator picks. + * + * These are offers, not corrections. A flagged setting is not a wrong setting — + * an instance that genuinely requires an email address is configured exactly as + * its owner intended, and the right answer may well be to fix the export + * instead. Nothing here is preselected and nothing is applied unasked. + * + * Only the two verdicts `buildReadinessReport` produces are mapped: "required + * in Clerk" (relax the requirement) and "not enabled in Clerk" (turn it on). A + * row this file has no path for is simply not offered — the report still names + * it and still points at the dashboard. + */ + +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { toClerkStrategy } from "./clerk-config.ts"; +import type { ReadinessItem, ReadinessSection } from "./readiness.ts"; + +/** One leaf of the config document, and what to set it to. */ +export type SettingWrite = { path: string[]; value: boolean | string[] }; + +/** One offered change: what it says, and the config leaves it writes. */ +export type SettingChange = { + /** Stable identity for the multiselect, and for tests. */ + id: string; + label: string; + section: ReadinessSection; + /** `enable` turns something on; `relax` drops a requirement. */ + kind: "enable" | "relax"; + writes: SettingWrite[]; +}; + +type ChangeWrites = { enable: SettingWrite[]; relax: SettingWrite[] }; + +/** + * Where each attribute lives in the config document. `used_for_sign_up` is the + * enable field that matters here: `POST /v1/users` validates an import against + * the instance's sign-up requirements, not its sign-in strategies. + * + * Email and phone are **verifiable** attributes, so enabling one takes two + * writes rather than one. Clerk rejects a verifiable attribute that is on with + * no way to verify it — `422 phone_number: verifiable attributes need to have + * at least one verification` — and switching the attribute off empties + * `verification_strategies`, so whatever turns it back on has to put a strategy + * back. Username, password and the name fields are not verifiable and take one + * write each. + */ +const ATTRIBUTE_WRITES: Record = { + email_address: { + enable: [ + { path: ["auth_email", "used_for_sign_up"], value: true }, + { path: ["auth_email", "verification_strategies"], value: ["email_code"] }, + ], + relax: [{ path: ["auth_email", "required_for_sign_up"], value: false }], + }, + phone_number: { + enable: [ + { path: ["auth_phone", "used_for_sign_up"], value: true }, + { path: ["auth_phone", "verification_strategies"], value: ["phone_code"] }, + ], + relax: [{ path: ["auth_phone", "required_for_sign_up"], value: false }], + }, + username: { + enable: [{ path: ["auth_username", "used_for_sign_up"], value: true }], + relax: [{ path: ["auth_username", "required_for_sign_up"], value: false }], + }, + password: { + enable: [{ path: ["auth_password", "enabled"], value: true }], + relax: [{ path: ["auth_password", "required"], value: false }], + }, + first_name: { + enable: [{ path: ["user_model", "first_name", "enabled"], value: true }], + relax: [{ path: ["user_model", "first_name", "required"], value: false }], + }, + last_name: { + enable: [{ path: ["user_model", "last_name", "enabled"], value: true }], + relax: [{ path: ["user_model", "last_name", "required"], value: false }], + }, +}; + +/** `github` → `connection_oauth_github`, via Clerk's own strategy name. */ +function socialPath(provider: string): string[] { + return [`connection_oauth_${toClerkStrategy(provider).replace(/^oauth_/, "")}`, "enabled"]; +} + +function changeFor(item: ReadinessItem): SettingChange | undefined { + // Required-but-not-universal is the only verdict that relaxes rather than + // enables; every other flagged row is something switched off in Clerk. + const relax = item.clerkRequired === true; + + if (item.section === "social") { + // A provider has no "required" in Clerk, so there is nothing to relax. + if (relax) return undefined; + return { + id: item.key, + label: `Enable ${item.label} sign-in`, + section: item.section, + kind: "enable", + writes: [{ path: socialPath(item.key), value: true }], + }; + } + + const writes = ATTRIBUTE_WRITES[item.key]; + if (!writes) return undefined; + + return { + id: item.key, + label: relax ? `Make ${item.label} optional at sign-up` : `Enable ${item.label}`, + section: item.section, + kind: relax ? "relax" : "enable", + writes: relax ? writes.relax : writes.enable, + }; +} + +/** The changes offerable for a report's flagged rows, in report order. */ +export function buildSettingChanges(flagged: ReadinessItem[]): SettingChange[] { + return flagged.map(changeFor).filter((change): change is SettingChange => change !== undefined); +} + +/** + * Collapses the chosen changes into one config payload. + * + * Changes share parents — `first_name` and `last_name` both write `user_model` + * — so leaves are written into a shared tree rather than merged after the fact. + */ +export function buildChangePayload(changes: SettingChange[]): Record { + const payload: Record = {}; + + for (const write of changes.flatMap((change) => change.writes)) { + let node = payload; + for (const key of write.path.slice(0, -1)) { + node = (node[key] ??= {}) as Record; + } + node[write.path[write.path.length - 1] as string] = write.value; + } + + return payload; +} + +/** + * The instance's settings as they stand once `changes` have been written. + * + * Deliberately not a re-read. Clerk's Frontend API is eventually consistent, so + * a `/v1/environment` fetch issued straight after the config write routinely + * still reports the pre-write settings — which would redraw the report with + * every row it just cleared still flagged. The Platform API answering the write + * is the authoritative statement of what took, exactly as `clerk config patch` + * treats it. + * + * @param settings - `null` passes through: when the settings could not be read + * nothing is ever flagged, so there is nothing to have changed. + */ +export function applyChanges( + settings: UserSettingsJSON | null, + changes: SettingChange[], +): UserSettingsJSON | null { + if (!settings) return null; + + const next = structuredClone(settings); + + for (const change of changes) { + if (change.section === "social") { + const social = next.social as Record; + const strategy = toClerkStrategy(change.id); + social[strategy] = { ...social[strategy], enabled: true }; + continue; + } + + const attributes = next.attributes as Record; + attributes[change.id] = + change.kind === "enable" + ? { + ...attributes[change.id], + enabled: true, + required: attributes[change.id]?.required ?? false, + } + : { + ...attributes[change.id], + enabled: attributes[change.id]?.enabled ?? true, + required: false, + }; + } + + return next; +} diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts new file mode 100644 index 000000000..842e9c4b5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts @@ -0,0 +1,489 @@ +import { describe, expect, test } from "bun:test"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { analyzeFields, type FieldAnalysis } from "./analysis.ts"; +import { buildReadinessReport, formatReadinessReport, type ReadinessItem } from "./readiness.ts"; + +/** Instance settings carrying only the attributes and providers a test names. */ +function settings(config: { + attributes?: Record; + social?: Record; +}): UserSettingsJSON { + return { + attributes: Object.fromEntries( + Object.entries(config.attributes ?? {}).map(([name, value]) => [ + name, + { enabled: value.enabled, required: value.required ?? false }, + ]), + ), + social: config.social ?? {}, + } as unknown as UserSettingsJSON; +} + +/** Field analysis with everything absent unless the test says otherwise. */ +function analysis(overrides: Partial & { totalUsers: number }): FieldAnalysis { + return { + identifiers: { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: overrides.totalUsers, + ...overrides.identifiers, + }, + fieldCounts: overrides.fieldCounts ?? {}, + totalUsers: overrides.totalUsers, + }; +} + +const item = (report: { items: ReadinessItem[] }, label: string) => + report.items.find((entry) => entry.label === label); + +describe("which rows appear", () => { + test("reports only the fields the file actually carries", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 3, identifiers: { verifiedEmails: 3 } as never }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }); + expect(report.items.map((entry) => entry.label)).toEqual(["Email"]); + }); + + test("counts verified and unverified identifiers together", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, unverifiedEmails: 2, hasAnyIdentifier: 5 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }); + expect(item(report, "Email")?.userCount).toBe(5); + }); + + test("groups rows into identifiers, auth and user model", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, username: 2, hasAnyIdentifier: 2 } as never, + fieldCounts: { password: 2, firstName: 2, lastName: 1 }, + }), + settings: settings({}), + }); + expect(report.items.map((entry) => [entry.label, entry.section])).toEqual([ + ["Email", "identifiers"], + ["Username", "identifiers"], + ["Password", "auth"], + ["First name", "model"], + ["Last name", "model"], + ]); + }); +}); + +describe("required in Clerk but missing from the file", () => { + // The expensive case: those users fail one at a time, mid-import, after + // earlier users have already been created. + test("flags an attribute Clerk requires that not every user has", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 10, username: 10 } as never, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }), + }); + + const email = item(report, "Email"); + expect(email?.blocking).toBe(true); + expect(email?.detail).toContain("required in Clerk"); + // A required identifier is the one verdict Clerk refuses the user over. + expect(email?.consequence).toBe("rejects"); + expect(report.blocking).toHaveLength(1); + }); + + test("does not flag a required attribute every user has", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + expect(report.blocking).toHaveLength(0); + }); + + test("does not flag an enabled-but-optional attribute that some users lack", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 10, hasAnyIdentifier: 10 } as never, + fieldCounts: { firstName: 2 }, + }), + settings: settings({ + attributes: { email_address: { enabled: true }, first_name: { enabled: true } }, + }), + }); + expect(report.blocking).toHaveLength(0); + }); + + // The import sends `skip_password_requirement`, so a required password costs + // the user their password rather than their whole account. + test("a required password drops rather than rejects", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + fieldCounts: { password: 1 }, + }), + settings: settings({ + attributes: { + email_address: { enabled: true }, + password: { enabled: true, required: true }, + }, + }), + }); + expect(item(report, "Password")?.consequence).toBe("drops"); + }); +}); + +describe("present in the file but disabled in Clerk", () => { + test("flags an attribute the instance has switched off", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, username: 3, hasAnyIdentifier: 3 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true }, username: { enabled: false } }, + }), + }); + + const username = item(report, "Username"); + expect(username?.blocking).toBe(true); + expect(username?.detail).toBe("not enabled in Clerk"); + }); + + test("flags a social provider users signed up with that Clerk lacks", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }), + providerCounts: { google: 3, discord: 1 }, + }); + + expect(item(report, "Google")?.blocking).toBe(false); + expect(item(report, "Discord")?.blocking).toBe(true); + expect(report.blocking.map((entry) => entry.label)).toEqual(["Discord"]); + }); + + test("maps a provider whose Clerk strategy name differs", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 1, identifiers: { verifiedEmails: 1 } as never }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_microsoft: { enabled: true } }, + }), + providerCounts: { azure: 1 }, + }); + expect(item(report, "Microsoft (Azure)")?.blocking).toBe(false); + }); + + test("ignores a provider no user actually signed up with", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 1, identifiers: { verifiedEmails: 1 } as never }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + providerCounts: { discord: 0 }, + }); + expect(item(report, "Discord")).toBeUndefined(); + }); +}); + +describe("when the instance settings cannot be read", () => { + const unreadable = () => + buildReadinessReport({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 3 } as never, + }), + settings: null, + }); + + test("marks the report as degraded rather than failing", () => { + expect(unreadable().settingsUnavailable).toBe(true); + }); + + // `null` means "not read", which must not be confused with `false` + // ("read, and it is off") — the latter blocks, the former cannot. + test("claims nothing about Clerk, so nothing blocks", () => { + const report = unreadable(); + expect(item(report, "Email")?.clerkEnabled).toBeNull(); + expect(report.blocking).toHaveLength(0); + }); + + test("still reports what the file contains", () => { + expect(unreadable().items.map((entry) => entry.label)).toEqual(["Email"]); + }); + + test("renders a note explaining the checks are coverage only", () => { + const output = formatReadinessReport(unreadable()).join("\n"); + expect(output).toContain("Could not read this instance's settings"); + expect(output).toContain("dashboard.clerk.com"); + }); +}); + +describe("file-level totals", () => { + test("counts users with no identifier at all", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 7 } as never, + }), + settings: settings({}), + }); + expect(report.withoutIdentifier).toBe(3); + }); + + test("carries the validation failure count through", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 2 }), + settings: settings({}), + validationFailed: 5, + }); + expect(report.validationFailed).toBe(5); + }); +}); + +/** + * The counts an operator actually decides on. Built from the users themselves, + * because per-field coverage cannot answer them: the users missing an email and + * the users missing a password overlap by an amount only a per-user pass knows. + */ +describe("what the settings mean for these users", () => { + const REQUIRE_EMAIL_AND_PASSWORD = settings({ + attributes: { + email_address: { enabled: true, required: true }, + password: { enabled: true, required: true }, + }, + }); + + /** Two with everything, two with no email, one with an email but no password. */ + const USERS = [ + { userId: "a", email: "a@x.dev", password: "hash" }, + { userId: "b", email: "b@x.dev", password: "hash" }, + { userId: "c", username: "c" }, + { userId: "d", username: "d" }, + { userId: "e", email: "e@x.dev" }, + ] as never; + + const outcomes = () => + buildReadinessReport({ + analysis: analyzeFields(USERS), + users: USERS, + settings: REQUIRE_EMAIL_AND_PASSWORD, + }).outcomes; + + test("the three totals account for every user exactly once", () => { + const result = outcomes(); + expect(result).toMatchObject({ rejected: 2, incomplete: 1, complete: 2 }); + expect((result?.rejected ?? 0) + (result?.incomplete ?? 0) + (result?.complete ?? 0)).toBe(5); + }); + + // The file has three users without a password, but two of them are already + // rejected for the email — counting them twice would overstate the damage. + test("a rejected user is not also counted as incomplete", () => { + expect(outcomes()?.incompleteReasons).toEqual([ + { + label: "Password", + count: 1, + detail: expect.stringContaining("1 has no password, which this instance requires"), + }, + ]); + }); + + test("names why the rejected users are rejected", () => { + expect(outcomes()?.rejectedReasons).toEqual([ + { label: "Email", count: 2, detail: "2 have no email, which this instance requires" }, + ]); + }); + + /** + * The rejected users lose nothing today — they are not being created. But the + * moment the operator relaxes the requirement rejecting them (one of the + * changes on offer) every masked setting lands at once. Surfacing it here is + * what saves an apply → re-check → discover → apply → re-check loop. + */ + describe("what is masked behind a rejection", () => { + // b and c have no email, so both are rejected; b also carries a phone the + // instance is not set up to store. Exactly the shape the supabase sample + // hits: every phone belongs to a user who has no email. + const MASKED_USERS = [ + { userId: "a", email: "a@x.dev" }, + { userId: "b", username: "b", phone: "+15551234567" }, + { userId: "c", username: "c" }, + ] as never; + + const report = (attributes: Record) => + buildReadinessReport({ + analysis: analyzeFields(MASKED_USERS), + users: MASKED_USERS, + settings: settings({ attributes }), + }); + + const REQUIRE_EMAIL_PHONE_OFF = { + email_address: { enabled: true, required: true }, + phone_number: { enabled: false }, + username: { enabled: true }, + }; + + test("counts a setting that only bites once the rejected users get in", () => { + const outcomes = report(REQUIRE_EMAIL_PHONE_OFF).outcomes; + + expect(outcomes).toMatchObject({ rejected: 2, incomplete: 0, complete: 1 }); + expect(outcomes?.maskedReasons).toEqual([ + { + label: "Phone", + count: 1, + detail: "1 has a phone, which this instance is not set up to store", + }, + ]); + }); + + test("keeps it out of the incomplete count, which is about users being imported", () => { + expect(report(REQUIRE_EMAIL_PHONE_OFF).outcomes?.incompleteReasons).toEqual([]); + }); + + test("renders it under the rejected group", () => { + const output = formatReadinessReport(report(REQUIRE_EMAIL_PHONE_OFF)).join("\n"); + + expect(output).toContain("If you import them, this applies to them too:"); + expect(output).toContain("1 has a phone, which this instance is not set up to store"); + }); + + // Enabling phone is the other change on offer, and it empties the block — + // which is the check that the two offers really do interact this way. + test("is empty once the masked setting is no longer a problem", () => { + const outcomes = report({ + email_address: { enabled: true, required: true }, + phone_number: { enabled: true }, + username: { enabled: true }, + }).outcomes; + + expect(outcomes).toMatchObject({ rejected: 2 }); + expect(outcomes?.maskedReasons).toEqual([]); + }); + }); + + test("a disabled attribute costs the users who carry it, not the ones who don't", () => { + const users = [ + { userId: "a", email: "a@x.dev", username: "a" }, + { userId: "b", email: "b@x.dev" }, + ] as never; + + const result = buildReadinessReport({ + analysis: analyzeFields(users), + users, + settings: settings({ + attributes: { email_address: { enabled: true }, username: { enabled: false } }, + }), + }).outcomes; + + expect(result).toMatchObject({ rejected: 0, incomplete: 1, complete: 1 }); + expect(result?.incompleteReasons[0]?.detail).toContain("not set up to store"); + }); + + test("is omitted when the caller passes no users", () => { + const report = buildReadinessReport({ + analysis: analyzeFields(USERS), + settings: REQUIRE_EMAIL_AND_PASSWORD, + }); + expect(report.outcomes).toBeUndefined(); + }); + + test("renders each outcome with the reasons behind it", () => { + const output = formatReadinessReport( + buildReadinessReport({ + analysis: analyzeFields(USERS), + users: USERS, + settings: REQUIRE_EMAIL_AND_PASSWORD, + }), + ).join("\n"); + + expect(output).toContain("2 users will not be imported"); + expect(output).toContain("1 user will be imported, but not everything they carry"); + expect(output).toContain("2 users will be imported in full"); + expect(output).toContain("they will have to reset it to sign in"); + }); +}); + +describe("rendering", () => { + const blocked = () => + buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 8, username: 10 } as never, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }), + validationFailed: 2, + }); + + test("leads with the counts an operator needs before confirming", () => { + const output = formatReadinessReport(blocked()).join("\n"); + expect(output).toContain("10 users in this file"); + expect(output).toContain("2 failed validation"); + expect(output).toContain("2 without any identifier"); + }); + + test("names the blocking rows and points at the dashboard", () => { + const output = formatReadinessReport(blocked()).join("\n"); + expect(output).toContain("1 setting needs attention"); + expect(output).toContain("required in Clerk, and not every user has one"); + expect(output).toContain("dashboard.clerk.com"); + }); + + test("confirms a clean report when nothing blocks", () => { + const output = formatReadinessReport( + buildReadinessReport({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }), + ).join("\n"); + expect(output).toContain("Every field in this file is configured in Clerk"); + }); + + test("does not claim everything is configured when settings were unreadable", () => { + const output = formatReadinessReport( + buildReadinessReport({ analysis: analysis({ totalUsers: 1 }), settings: null }), + ).join("\n"); + expect(output).not.toContain("Every field in this file is configured"); + }); + + test("renders section headings only for sections that have rows", () => { + const output = formatReadinessReport( + buildReadinessReport({ + analysis: analysis({ + totalUsers: 1, + identifiers: { verifiedEmails: 1, hasAnyIdentifier: 1 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }), + ).join("\n"); + expect(output).toContain("Identifiers"); + expect(output).not.toContain("Social connections"); + expect(output).not.toContain("User model"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts new file mode 100644 index 000000000..e476da4a1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -0,0 +1,478 @@ +/** + * The Migration Readiness report: what the import file contains, cross- + * referenced against what the destination instance actually accepts. + * + * Ported from the standalone migration-tool's `displayCrossReference`. Split + * into a pure {@link buildReadinessReport} and a separate renderer so the + * cross-reference decisions are testable without parsing coloured output. + * + * The point of the report is to surface, *before* anything is written to + * Clerk, the two failure modes a migration only discovers halfway through: + * a field Clerk requires that some users lack, and a social provider users + * signed up with that Clerk has not enabled. + */ + +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { bold, dim, green, red, yellow } from "../../../lib/color.ts"; +// Pure attribute lookups, shared with the `users` create wizard. +import { isEnabled, isRequired, type AttributeName } from "../../users/interactive/attributes.ts"; +import type { User } from "../types.ts"; +import { hasValue, type FieldAnalysis } from "./analysis.ts"; +import { providerLabel, toClerkStrategy } from "./clerk-config.ts"; + +export const DASHBOARD_URL = "https://dashboard.clerk.com/~/user-authentication"; + +export type ReadinessSection = "identifiers" | "auth" | "social" | "model"; + +/** + * One row of the report. + * + * @property clerkEnabled - `null` when the instance settings could not be read, + * which is different from `false` ("read, and it is off"). + * @property blocking - This row will cost users unless the operator acts. + */ +export type ReadinessItem = { + label: string; + /** + * What the row is about, machine-side: an {@link AttributeName} for every + * section but `social`, and the source platform's provider key for that one. + * `label` is for humans; this is what `modify-settings.ts` looks up. + */ + key: string; + section: ReadinessSection; + /** Users in the file that carry this field or provider. */ + userCount: number; + clerkEnabled: boolean | null; + clerkRequired: boolean | null; + blocking: boolean; + /** + * What this row costs the users it affects. + * + * - `rejects` — Clerk refuses the user outright. Only a required identifier + * does this: `POST /v1/users` enforces the instance's sign-up identifier + * requirements, and a user carrying none of them has nothing to be created + * under. + * - `drops` — the user is created, but this piece of them is not. A required + * password is in this group rather than `rejects` because the import sends + * `skip_password_requirement` (see `import-users.ts`), so the user lands + * without one and has to reset it before they can sign in that way. + */ + consequence?: "rejects" | "drops"; + /** Why it blocks — omitted when it does not. */ + detail?: string; +}; + +/** One reason users are affected, and how many of them it affects. */ +export type OutcomeReason = { label: string; count: number; detail: string }; + +/** + * What the settings mean for the users in the file, counted per user rather + * than per field. + * + * Per-field coverage cannot answer "how many users will not be imported" — + * the users missing an email and the users missing a username overlap by an + * unknown amount. Each user is classified once, into the worst outcome that + * applies to them, so the three totals add up to the file. + */ +export type ImportOutcomes = { + rejected: number; + rejectedReasons: OutcomeReason[]; + /** + * What *else* affects the rejected users — surfaced now rather than after + * they become importable. + * + * A user who is not being created cannot lose a field, so these settings cost + * nothing today and would otherwise go unmentioned. But the moment the + * operator relaxes the requirement rejecting them, every one of these lands. + * Reporting it only afterwards turns one decision into a apply → re-check → + * discover → apply → re-check loop, which is exactly what the report exists + * to prevent. + */ + maskedReasons: OutcomeReason[]; + incomplete: number; + incompleteReasons: OutcomeReason[]; + /** Imported with everything the file carries for them. */ + complete: number; +}; + +export type ReadinessReport = { + totalUsers: number; + /** Users with no identifier at all; they cannot be imported under any settings. */ + withoutIdentifier: number; + validationFailed: number; + items: ReadinessItem[]; + /** Every item flagged `blocking`, in report order. */ + blocking: ReadinessItem[]; + /** True when the instance settings could not be read. */ + settingsUnavailable: boolean; + /** Omitted when the caller passed no users to classify. */ + outcomes?: ImportOutcomes; +}; + +type BuildInput = { + analysis: FieldAnalysis; + /** `null` when no publishable key was available, or FAPI could not be read. */ + settings: UserSettingsJSON | null; + validationFailed?: number; + /** Source-platform provider key → user count. Supabase exports only. */ + providerCounts?: Record; + /** + * The users themselves, for the per-user outcome counts. Optional so callers + * that only need the coverage rows (and tests working from a synthetic + * {@link FieldAnalysis}) do not have to supply them. + */ + users?: User[]; +}; + +/** + * Identifiers Clerk creates a user *under*. A required one that a user does not + * carry leaves nothing to create them with, so the API refuses them — which is + * why these are the only attributes whose consequence is `rejects`. + */ +const IDENTIFIER_ATTRIBUTES = new Set(["email_address", "phone_number", "username"]); + +/** An identifier or user-model row, with its blocking verdict. */ +function buildAttributeItem( + label: string, + section: ReadinessSection, + attribute: AttributeName, + userCount: number, + settings: UserSettingsJSON | null, + totalUsers: number, +): ReadinessItem { + const enabled = settings ? isEnabled(settings, attribute) : null; + const required = settings ? isRequired(settings, attribute) : null; + const missing = totalUsers - userCount; + + // Required but not universal is the expensive case: those users fail one by + // one, mid-import, after earlier users have already been created. + if (required === true && missing > 0) { + return { + label, + key: attribute, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: true, + consequence: IDENTIFIER_ATTRIBUTES.has(attribute) ? "rejects" : "drops", + // How many users this costs is the outcome block's job. Restating it here + // reads as a contradiction, because that block counts each user once and + // this row counts the field — a user missing both an email and a password + // appears in both rows but only in the first outcome. + detail: "required in Clerk, and not every user has one", + }; + } + + // Present in the file but switched off in Clerk: the data is silently dropped. + if (enabled === false && userCount > 0) { + return { + label, + key: attribute, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: true, + consequence: "drops", + detail: "not enabled in Clerk", + }; + } + + return { + label, + key: attribute, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: false, + }; +} + +/** + * Cross-references the file against the instance. + * + * A field absent from the file contributes no row — the report describes what + * is actually being imported, not every setting Clerk supports. + */ +export function buildReadinessReport(input: BuildInput): ReadinessReport { + const { analysis, settings, validationFailed = 0, providerCounts = {} } = input; + const total = analysis.totalUsers; + const items: ReadinessItem[] = []; + + const emailCount = analysis.identifiers.verifiedEmails + analysis.identifiers.unverifiedEmails; + const phoneCount = analysis.identifiers.verifiedPhones + analysis.identifiers.unverifiedPhones; + + const attributeRows: [string, ReadinessSection, AttributeName, number][] = [ + ["Email", "identifiers", "email_address", emailCount], + ["Phone", "identifiers", "phone_number", phoneCount], + ["Username", "identifiers", "username", analysis.identifiers.username], + ["Password", "auth", "password", analysis.fieldCounts.password ?? 0], + ["First name", "model", "first_name", analysis.fieldCounts.firstName ?? 0], + ["Last name", "model", "last_name", analysis.fieldCounts.lastName ?? 0], + ]; + + for (const [label, section, attribute, count] of attributeRows) { + if (count > 0) { + items.push(buildAttributeItem(label, section, attribute, count, settings, total)); + } + } + + for (const [provider, count] of Object.entries(providerCounts)) { + if (count === 0) continue; + const enabled = settings + ? (settings.social?.[toClerkStrategy(provider) as keyof typeof settings.social]?.enabled ?? + false) + : null; + items.push({ + label: providerLabel(provider), + key: provider, + section: "social", + userCount: count, + clerkEnabled: enabled, + clerkRequired: null, + blocking: enabled === false, + ...(enabled === false + ? { consequence: "drops" as const, detail: "not enabled in Clerk" } + : {}), + }); + } + + const blocking = items.filter((item) => item.blocking); + + return { + totalUsers: total, + withoutIdentifier: total - analysis.identifiers.hasAnyIdentifier, + validationFailed, + items, + blocking, + settingsUnavailable: settings === null, + ...(input.users ? { outcomes: countOutcomes(input.users, blocking) } : {}), + }; +} + +/** Whether a user carries the field an attribute row is about. */ +const CARRIES: Record) => boolean> = { + email_address: (u) => + hasValue(u.email) || hasValue(u.emailAddresses) || hasValue(u.unverifiedEmailAddresses), + phone_number: (u) => + hasValue(u.phone) || hasValue(u.phoneNumbers) || hasValue(u.unverifiedPhoneNumbers), + username: (u) => hasValue(u.username), + password: (u) => hasValue(u.password), + first_name: (u) => hasValue(u.firstName), + last_name: (u) => hasValue(u.lastName), +}; + +/** Which users a flagged row actually affects: the ones missing it, or carrying it. */ +function affects(item: ReadinessItem, user: Record): boolean { + const carries = CARRIES[item.key]; + if (!carries) return false; + // A required row costs the users without it; a disabled row costs the ones with it. + return item.clerkRequired === true ? !carries(user) : carries(user); +} + +/** + * One reason line: how many users, what they are missing or carrying, and what + * the instance does about it. Count first, because that is what is being + * decided on. + */ +function describe(item: ReadinessItem, count: number): string { + const noun = item.label.toLowerCase(); + const have = count === 1 ? "has" : "have"; + + if (item.clerkRequired === true) { + const consequence = item.key === "password" ? " — they will have to reset it to sign in" : ""; + return `${count} ${have} no ${noun}, which this instance requires${consequence}`; + } + return `${count} ${have} a ${noun}, which this instance is not set up to store`; +} + +function toReasons(counts: Map): OutcomeReason[] { + return [...counts].map(([label, { item, count }]) => ({ + label, + count, + detail: describe(item, count), + })); +} + +/** + * Classifies every user into exactly one outcome, worst first. + * + * Social rows are left out: which providers a user signed up with lives in the + * raw export rather than the transformed `User`, so they cannot be counted per + * user here. Their coverage row still names them. + */ +function countOutcomes(users: User[], blocking: ReadinessItem[]): ImportOutcomes { + const rejecting = blocking.filter((item) => item.consequence === "rejects"); + const dropping = blocking.filter( + (item) => item.consequence === "drops" && item.section !== "social", + ); + + type Tally = Map; + const rejectedBy: Tally = new Map(); + const droppedBy: Tally = new Map(); + const maskedBy: Tally = new Map(); + let rejected = 0; + let incomplete = 0; + let complete = 0; + + const tally = (into: Tally, item: ReadinessItem) => { + const entry = into.get(item.label) ?? { item, count: 0 }; + entry.count++; + into.set(item.label, entry); + }; + + for (const entry of users) { + const user = entry as unknown as Record; + const gaps = dropping.filter((item) => affects(item, user)); + + const refusals = rejecting.filter((item) => affects(item, user)); + if (refusals.length > 0) { + rejected++; + for (const item of refusals) tally(rejectedBy, item); + // Their gaps are still tallied, into a separate bucket. Dropping them + // here is what makes the settings interact invisibly: relaxing the + // requirement that rejects these users lets them in, and only then does + // whatever else affects them show up — a second round trip to learn + // something that was knowable now. + for (const item of gaps) tally(maskedBy, item); + continue; + } + + if (gaps.length === 0) { + complete++; + continue; + } + + incomplete++; + for (const item of gaps) tally(droppedBy, item); + } + + return { + rejected, + rejectedReasons: toReasons(rejectedBy), + maskedReasons: toReasons(maskedBy), + incomplete, + incompleteReasons: toReasons(droppedBy), + complete, + }; +} + +const SECTION_ORDER: ReadinessSection[] = ["identifiers", "auth", "social", "model"]; +const SECTION_LABELS: Record = { + identifiers: "Identifiers", + auth: "Authentication", + social: "Social connections", + model: "User model", +}; + +function renderItem(item: ReadinessItem, total: number): string { + const coverage = item.userCount === total ? "all users" : `${item.userCount}/${total} users`; + + if (item.blocking) { + return ` ${yellow("⚠")} ${item.label} — ${yellow(item.detail ?? "needs attention")} — ${dim(coverage)}`; + } + if (item.clerkEnabled === true) { + return ` ${green("✓")} ${item.label} — ${dim(`enabled in Clerk — ${coverage}`)}`; + } + // Settings unavailable: state coverage without claiming anything about Clerk. + return ` ${yellow("!")} ${item.label} — ${dim(`${coverage} — check it is enabled in Clerk`)}`; +} + +const users = (count: number) => `${count} user${count === 1 ? "" : "s"}`; + +/** + * The three outcomes, each with the reasons behind it. + * + * This is the part of the report that answers "so what": which users the + * instance will refuse, which will arrive with something missing, and why. + * Per-field coverage lives further down and is a different question. + */ +function renderOutcomes(outcomes: ImportOutcomes): string[] { + const lines: string[] = []; + + const group = ( + symbol: string, + colour: (text: string) => string, + headline: string, + reasons: OutcomeReason[], + ) => { + lines.push(` ${colour(symbol)} ${colour(headline)}`); + for (const reason of reasons) lines.push(` ${dim(reason.detail)}`); + }; + + if (outcomes.rejected > 0) { + group("✗", red, `${users(outcomes.rejected)} will not be imported`, outcomes.rejectedReasons); + + // Named here rather than left for a second run of the report: these are the + // settings that start costing something the moment the rejection above is + // lifted, and lifting it is one of the changes on offer. + if (outcomes.maskedReasons.length > 0) { + lines.push(` ${dim("If you import them, this applies to them too:")}`); + for (const reason of outcomes.maskedReasons) lines.push(` ${dim(reason.detail)}`); + } + } + if (outcomes.incomplete > 0) { + group( + "⚠", + yellow, + `${users(outcomes.incomplete)} will be imported, but not everything they carry`, + outcomes.incompleteReasons, + ); + } + if (outcomes.complete > 0) { + lines.push(` ${green("✓")} ${green(`${users(outcomes.complete)} will be imported in full`)}`); + } + + return lines; +} + +/** Renders the report for a human, as lines. */ +export function formatReadinessReport(report: ReadinessReport): string[] { + const lines: string[] = [bold("Migration readiness")]; + + lines.push(` ${users(report.totalUsers)} in this file`); + if (report.validationFailed > 0) { + lines.push(` ${yellow(`${report.validationFailed} failed validation and will be skipped`)}`); + } + if (report.withoutIdentifier > 0) { + lines.push( + ` ${red(`${report.withoutIdentifier} without any identifier — cannot be imported`)}`, + ); + } + + if (report.outcomes) { + const outcomeLines = renderOutcomes(report.outcomes); + if (outcomeLines.length > 0) lines.push("", ...outcomeLines); + } + + if (report.settingsUnavailable) { + lines.push( + "", + ` ${yellow("!")} ${dim("Could not read this instance's settings, so the checks below are coverage only.")}`, + ` ${dim(` Verify your settings at ${DASHBOARD_URL}`)}`, + ); + } + + for (const section of SECTION_ORDER) { + const sectionItems = report.items.filter((item) => item.section === section); + if (sectionItems.length === 0) continue; + + lines.push("", bold(SECTION_LABELS[section])); + for (const item of sectionItems) lines.push(renderItem(item, report.totalUsers)); + } + + lines.push(""); + if (report.blocking.length > 0) { + const count = report.blocking.length; + lines.push( + yellow(`⚠ ${count} setting${count === 1 ? "" : "s"} need${count === 1 ? "s" : ""} attention`), + dim(` ${DASHBOARD_URL}`), + ); + } else if (!report.settingsUnavailable) { + lines.push(green("✓ Every field in this file is configured in Clerk")); + } + + return lines; +} diff --git a/packages/cli-core/src/commands/migrate/lib/retry.test.ts b/packages/cli-core/src/commands/migrate/lib/retry.test.ts new file mode 100644 index 000000000..ed7f10091 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/retry.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, test } from "bun:test"; +import { BapiError, CliError } from "../../../lib/errors.ts"; +import { RateLimitExceededError, readRetryAfter, retryOn429 } from "./retry.ts"; + +const rateLimited = (headers: Record = {}) => + new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow" }] }), + new Headers(headers), + ); + +const failed = (status: number) => + new BapiError( + status, + JSON.stringify({ errors: [{ code: "e", message: "nope" }] }), + new Headers(), + ); + +describe("readRetryAfter", () => { + test.each([ + ["12", 12], + ["0", undefined], + ["-1", undefined], + ["soon", undefined], + ])("Retry-After: %s -> %p", (header, expected) => { + expect(readRetryAfter(rateLimited({ "retry-after": header }))).toBe( + expected as number | undefined, + ); + }); + + test("falls back to the error body's retryAfter meta", () => { + const error = new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow", meta: { retryAfter: 7 } }] }), + new Headers(), + ); + expect(readRetryAfter(error)).toBe(7); + }); + + test("prefers the header over the body", () => { + const error = new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow", meta: { retryAfter: 7 } }] }), + new Headers({ "retry-after": "3" }), + ); + expect(readRetryAfter(error)).toBe(3); + }); + + test("returns undefined when neither carries a value", () => { + expect(readRetryAfter(rateLimited())).toBeUndefined(); + }); +}); + +describe("retryOn429", () => { + test("returns the value when the call succeeds first time", async () => { + expect(await retryOn429(async () => "ok")).toBe("ok"); + }); + + test("retries after a 429 and returns the eventual value", async () => { + let attempts = 0; + const result = await retryOn429( + async () => { + attempts++; + if (attempts === 1) throw rateLimited({ "retry-after": "1" }); + return "ok"; + }, + { defaultDelayMs: 5 }, + ); + + expect(result).toBe("ok"); + expect(attempts).toBe(2); + }); + + test("waits the interval the server asked for", async () => { + let attempts = 0; + const started = performance.now(); + + await retryOn429(async () => { + attempts++; + if (attempts === 1) throw rateLimited({ "retry-after": "1" }); + return "ok"; + }); + + expect(performance.now() - started).toBeGreaterThanOrEqual(900); + }); + + test("falls back to the default delay when no Retry-After is given", async () => { + let attempts = 0; + await retryOn429( + async () => { + attempts++; + if (attempts === 1) throw rateLimited(); + return "ok"; + }, + { defaultDelayMs: 5 }, + ); + expect(attempts).toBe(2); + }); + + test("reports each backoff to the caller so it can log against its own run", async () => { + const seen: { attempt: number; delaySeconds: number }[] = []; + let attempts = 0; + + await retryOn429( + async () => { + attempts++; + if (attempts <= 2) throw rateLimited(); + return "ok"; + }, + { + defaultDelayMs: 5, + onRetry: ({ attempt, delaySeconds }) => seen.push({ attempt, delaySeconds }), + }, + ); + + expect(seen.map((entry) => entry.attempt)).toEqual([1, 2]); + expect(seen[0]?.delaySeconds).toBe(0.005); + }); + + test("gives up after the ceiling, distinctly from an ordinary failure", async () => { + let attempts = 0; + + await expect( + retryOn429( + async () => { + attempts++; + throw rateLimited(); + }, + { maxRetries: 2, defaultDelayMs: 5 }, + ), + ).rejects.toThrow(RateLimitExceededError); + + // One initial attempt plus maxRetries retries. + expect(attempts).toBe(3); + }); + + // Only rate limiting is transient; retrying a 422 would just repeat it. + test.each([[400], [401], [404], [422], [500]])("lets a %i through untouched", async (status) => { + let attempts = 0; + + await expect( + retryOn429(async () => { + attempts++; + throw failed(status); + }), + ).rejects.toThrow(BapiError); + + expect(attempts).toBe(1); + }); + + test("lets a non-API error through untouched", async () => { + await expect(retryOn429(async () => Promise.reject(new CliError("boom")))).rejects.toThrow( + CliError, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/retry.ts b/packages/cli-core/src/commands/migrate/lib/retry.ts new file mode 100644 index 000000000..a01de49a2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/retry.ts @@ -0,0 +1,66 @@ +/** + * Rate-limit backoff, shared by `migrate import` and `migrate delete`. + * + * Both walk the whole user set through BAPI and hit the same limits, so they + * back off identically rather than approximately: extracting this is what + * makes "deletion retries the same as import" true by construction. + */ + +import { BapiError } from "../../../lib/errors.ts"; +import { MAX_RETRIES, RETRY_DELAY_MS, getRetryDelay } from "./instance.ts"; + +/** Seconds to wait per a 429's `Retry-After` header or error meta, if given. */ +export function readRetryAfter(error: BapiError): number | undefined { + const header = error.headers?.get("retry-after"); + if (header) { + const parsed = Number(header); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + const meta = error.meta?.retryAfter; + return typeof meta === "number" && meta > 0 ? meta : undefined; +} + +/** Raised once a 429 has been retried {@link MAX_RETRIES} times. */ +export class RateLimitExceededError extends Error { + constructor(public readonly attempts: number) { + super(`Rate limit exceeded after ${attempts} retries`); + this.name = "RateLimitExceededError"; + } +} + +export type RetryOptions = { + /** Called before each backoff, so the caller can log it against its own run. */ + onRetry?: (info: { attempt: number; delaySeconds: number; message: string }) => void; + maxRetries?: number; + /** Backoff when the response carries no `Retry-After`. */ + defaultDelayMs?: number; +}; + +/** + * Runs `fn`, backing off and retrying whenever BAPI answers 429. + * + * Anything other than a 429 propagates untouched — only rate limiting is + * transient. Exhausting the retries raises {@link RateLimitExceededError} so + * the caller can record it distinctly from an ordinary API failure. + */ +export async function retryOn429(fn: () => Promise, options: RetryOptions = {}): Promise { + const maxRetries = options.maxRetries ?? MAX_RETRIES; + const defaultDelayMs = options.defaultDelayMs ?? RETRY_DELAY_MS; + + for (let attempt = 0; ; attempt++) { + try { + return await fn(); + } catch (error) { + if (!(error instanceof BapiError) || error.status !== 429) throw error; + if (attempt >= maxRetries) throw new RateLimitExceededError(maxRetries); + + const { delayMs, delaySeconds } = getRetryDelay(readRetryAfter(error), defaultDelayMs); + options.onRetry?.({ + attempt: attempt + 1, + delaySeconds, + message: `Rate limit hit (429), retrying in ${delaySeconds}s (attempt ${attempt + 1}/${maxRetries})`, + }); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts b/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts new file mode 100644 index 000000000..68c44e50b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts @@ -0,0 +1,69 @@ +import { expect, test } from "bun:test"; +import { createApiScheduler } from "./scheduler.ts"; + +/** Resolves after `ms`, so a task can be held open while others queue behind it. */ +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test("never runs more tasks at once than the concurrency limit", async () => { + const schedule = createApiScheduler(3, 10_000); + let active = 0; + let peak = 0; + + await Promise.all( + Array.from({ length: 20 }, () => + schedule(async () => { + active++; + peak = Math.max(peak, active); + await wait(5); + active--; + }), + ), + ); + + expect(peak).toBe(3); + expect(active).toBe(0); +}); + +test("frees a slot when a task throws, instead of deadlocking the queue", async () => { + const schedule = createApiScheduler(1, 10_000); + + await expect(schedule(() => Promise.reject(new Error("boom")))).rejects.toThrow("boom"); + + // If release() had been skipped on the failure path, this would hang. + expect(await schedule(async () => "ok")).toBe("ok"); +}); + +test("paces calls to the rate limit", async () => { + // 100 req/s -> 10ms between starts; 5 calls span at least 4 intervals. + const schedule = createApiScheduler(5, 100); + const started = performance.now(); + + await Promise.all(Array.from({ length: 5 }, () => schedule(async () => {}))); + + expect(performance.now() - started).toBeGreaterThanOrEqual(35); +}); + +test("returns each task's own resolved value", async () => { + const schedule = createApiScheduler(2, 10_000); + const results = await Promise.all([1, 2, 3].map((n) => schedule(async () => n * 2))); + expect(results).toEqual([2, 4, 6]); +}); + +test("treats zero or negative limits as one", async () => { + const schedule = createApiScheduler(0, 10_000); + let active = 0; + let peak = 0; + + await Promise.all( + Array.from({ length: 4 }, () => + schedule(async () => { + active++; + peak = Math.max(peak, active); + await wait(2); + active--; + }), + ), + ); + + expect(peak).toBe(1); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/scheduler.ts b/packages/cli-core/src/commands/migrate/lib/scheduler.ts new file mode 100644 index 000000000..1a7e65992 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/scheduler.ts @@ -0,0 +1,51 @@ +/** + * Concurrency gate plus rate pacing for BAPI calls. + * + * Replaces the standalone migration-tool's `p-limit` dependency: a bounded + * queue is a few lines, and the compiled binary carries one fewer package. + * + * Both limits apply to individual API calls rather than whole users, so a user + * with ten extra email addresses cannot burst past the instance's rate limit. + */ + +/** Runs `fn` once a slot is free and the pacing interval has elapsed. */ +export type ApiScheduler = (fn: () => Promise) => Promise; + +export function createApiScheduler(concurrencyLimit: number, rateLimit: number): ApiScheduler { + const maxConcurrent = Math.max(1, Math.floor(concurrencyLimit)); + const intervalMs = Math.ceil(1000 / Math.max(1, rateLimit)); + const waiting: (() => void)[] = []; + let active = 0; + let nextRequestAt = 0; + + async function acquire(): Promise { + if (active < maxConcurrent) { + active++; + return Promise.resolve(); + } + return new Promise((resolve) => waiting.push(resolve)); + } + + function release(): void { + const next = waiting.shift(); + // Hand the slot straight to the next waiter; `active` is unchanged because + // the slot never actually frees up. + if (next) next(); + else active--; + } + + return async (fn) => { + await acquire(); + try { + const now = Date.now(); + const waitMs = Math.max(0, nextRequestAt - now); + nextRequestAt = Math.max(now, nextRequestAt) + intervalMs; + if (waitMs > 0) { + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + return await fn(); + } finally { + release(); + } + }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/settings.test.ts b/packages/cli-core/src/commands/migrate/lib/settings.test.ts new file mode 100644 index 000000000..7ad3cf9ed --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/settings.test.ts @@ -0,0 +1,74 @@ +import { afterAll, afterEach, beforeAll, beforeEach, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir, getMigrationEntry, getProjectKey } from "../../../lib/config.ts"; +import { loadSettings, saveSettings } from "./settings.ts"; + +let workDir: string; +let configDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + // Realpath'd because the project key is derived from `process.cwd()`, which + // resolves the /var → /private/var symlink macOS puts in front of tmpdir. + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-config-")); + _setConfigDir(configDir); +}); + +afterEach(() => { + _setConfigDir(undefined); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +test("returns empty settings when nothing was saved", async () => { + expect(await loadSettings()).toEqual({}); +}); + +test("round-trips the transformer key and file path", async () => { + await saveSettings({ transformer: "clerk", file: "users.json" }); + expect(await loadSettings()).toEqual({ transformer: "clerk", file: "users.json" }); +}); + +test("writes to the CLI config file, not the working directory", async () => { + await saveSettings({ transformer: "clerk" }); + + expect(fs.existsSync(path.join(workDir, ".settings"))).toBe(false); + const config = JSON.parse(fs.readFileSync(path.join(configDir, "config.json"), "utf-8")); + expect(config.migrations).toEqual({ [await getProjectKey(workDir)]: { transformer: "clerk" } }); +}); + +test("keys the record by project, so another directory does not see it", async () => { + await saveSettings({ transformer: "clerk", file: "users.json" }); + + const elsewhere = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-other-"))); + try { + expect(await getMigrationEntry(await getProjectKey(elsewhere))).toBeUndefined(); + } finally { + fs.rmSync(elsewhere, { recursive: true, force: true }); + } +}); + +test("treats a corrupt config file as empty rather than failing the run", async () => { + fs.writeFileSync(path.join(configDir, "config.json"), "{not json"); + expect(await loadSettings()).toEqual({}); +}); + +test("leaves the run standing when the config cannot be written", async () => { + fs.rmSync(configDir, { recursive: true, force: true }); + fs.writeFileSync(configDir, "not a directory"); + + await saveSettings({ transformer: "clerk" }); + expect(await loadSettings()).toEqual({}); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/settings.ts b/packages/cli-core/src/commands/migrate/lib/settings.ts new file mode 100644 index 000000000..958bf1cd2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/settings.ts @@ -0,0 +1,39 @@ +/** + * What this project last migrated, and with which transformer. + * + * Kept in the CLI's own config file under `migrations`, keyed by project — the + * same shape `clerk webhooks listen` files its relay token under. An earlier + * version wrote a `.settings` file into the user's cwd instead, which the CLI + * cannot gitignore on the user's behalf and which put migration state inside + * the repository being migrated. + * + * Both halves fail silently: an unreadable or unwritable config only costs the + * user a remembered default, so it must not take the run down with it. + */ + +import { + getMigrationEntry, + getProjectKey, + setMigrationEntry, + type MigrationEntry, +} from "../../../lib/config.ts"; +import { log } from "../../../lib/log.ts"; + +/** Reads saved settings, or `{}` when absent or unreadable. */ +export async function loadSettings(): Promise { + try { + return (await getMigrationEntry(await getProjectKey(process.cwd()))) ?? {}; + } catch (error) { + log.debug(`config: could not read migration settings — ${error}`); + return {}; + } +} + +/** Persists settings for the next run in this project. */ +export async function saveSettings(settings: MigrationEntry): Promise { + try { + await setMigrationEntry(await getProjectKey(process.cwd()), settings); + } catch (error) { + log.debug(`config: could not save migration settings — ${error}`); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts b/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts new file mode 100644 index 000000000..00cde92c5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, test } from "bun:test"; +import { toClerkStrategy } from "./clerk-config.ts"; +import { + countProviders, + findDisabledProviders, + findUsersWithOnlyDisabledProviders, + getUserProviders, +} from "./supabase-providers.ts"; + +/** A Supabase row carrying the given providers, in the JSON export's shape. */ +const user = (id: string, providers: string[] | string | undefined) => ({ + id, + raw_app_meta_data: + providers === undefined ? undefined : JSON.stringify({ provider: "email", providers }), +}); + +describe("getUserProviders", () => { + test("reads providers from a JSON-string column, as a CSV export writes it", () => { + expect(getUserProviders(user("u1", ["email", "discord"]))).toEqual(["email", "discord"]); + }); + + test("reads providers from an object column, as a JSON export writes it", () => { + expect(getUserProviders({ id: "u1", raw_app_meta_data: { providers: ["google"] } })).toEqual([ + "google", + ]); + }); + + test("splits a delimited providers string", () => { + expect( + getUserProviders({ id: "u1", raw_app_meta_data: { providers: "email, discord" } }), + ).toEqual(["email", "discord"]); + }); + + test.each([ + ["missing column", { id: "u1" }], + ["unparseable column", { id: "u1", raw_app_meta_data: "{not json" }], + ["array column", { id: "u1", raw_app_meta_data: "[]" }], + ["no providers key", { id: "u1", raw_app_meta_data: '{"provider":"email"}' }], + ])("returns nothing for a %s", (_label, row) => { + expect(getUserProviders(row)).toEqual([]); + }); +}); + +describe("toClerkStrategy", () => { + test.each([ + ["google", "oauth_google"], + ["discord", "oauth_discord"], + ["github", "oauth_github"], + ["azure", "oauth_microsoft"], + ["twitter", "oauth_x"], + ["slack_oidc", "oauth_slack"], + ])("%s -> %s", (provider, strategy) => { + expect(toClerkStrategy(provider)).toBe(strategy); + }); +}); + +describe("countProviders", () => { + test("counts each provider across the export", () => { + expect( + countProviders([ + user("u1", ["email"]), + user("u2", ["email", "discord"]), + user("u3", ["discord"]), + ]), + ).toEqual({ email: 2, discord: 2 }); + }); +}); + +describe("findDisabledProviders", () => { + test("names the social providers Clerk does not have enabled", () => { + const rows = [user("u1", ["email", "google"]), user("u2", ["discord"])]; + expect(findDisabledProviders(rows, ["oauth_google"], toClerkStrategy)).toEqual(["discord"]); + }); + + test("never treats email or phone as disabled", () => { + const rows = [user("u1", ["email"]), user("u2", ["phone"]), user("u3", ["anonymous_users"])]; + expect(findDisabledProviders(rows, [], toClerkStrategy)).toEqual([]); + }); + + test("returns nothing when every provider is enabled", () => { + const rows = [user("u1", ["google"]), user("u2", ["github"])]; + expect(findDisabledProviders(rows, ["oauth_google", "oauth_github"], toClerkStrategy)).toEqual( + [], + ); + }); +}); + +describe("findUsersWithOnlyDisabledProviders", () => { + test("excludes a user whose sole provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", ["discord"])], ["discord"]); + expect([...result.excludedIds]).toEqual(["u1"]); + expect(result.byProvider).toEqual({ discord: 1 }); + }); + + test("keeps a user who can still sign in with email", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["email", "discord"])], + ["discord"], + ); + expect(result.excludedIds.size).toBe(0); + }); + + test("keeps a user who has another enabled social provider", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["google", "discord"])], + ["discord"], + ); + expect(result.excludedIds.size).toBe(0); + }); + + test("excludes a user whose every provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["discord", "twitch"])], + ["discord", "twitch"], + ); + expect([...result.excludedIds]).toEqual(["u1"]); + expect(result.byProvider).toEqual({ discord: 1, twitch: 1 }); + }); + + test("keeps a user with no provider data at all", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", undefined)], ["discord"]); + expect(result.excludedIds.size).toBe(0); + }); + + test("excludes nobody when no provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", ["discord"])], []); + expect(result.excludedIds.size).toBe(0); + }); + + test("reports a per-provider breakdown across many users", () => { + const result = findUsersWithOnlyDisabledProviders( + [ + user("u1", ["discord"]), + user("u2", ["discord"]), + user("u3", ["twitch"]), + user("u4", ["email", "discord"]), + ], + ["discord", "twitch"], + ); + expect([...result.excludedIds]).toEqual(["u1", "u2", "u3"]); + expect(result.byProvider).toEqual({ discord: 2, twitch: 1 }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts b/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts new file mode 100644 index 000000000..d123dd5e0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts @@ -0,0 +1,145 @@ +/** + * Cross-references the social providers in a Supabase export against what the + * destination Clerk instance has enabled. + * + * Ported from the standalone migration-tool's `src/lib/supabase.ts`, minus its + * hand-rolled CSV parser — the export is read through the same + * `readRawUsers` path every other Supabase read uses. + */ + +import { readRawUsers } from "./transform.ts"; + +/** + * Supabase lists these alongside social providers in `providers`, but they are + * built into Clerk and can never be "not enabled". + */ +export const NON_SOCIAL_PROVIDERS = new Set(["email", "phone", "anonymous_users"]); + +function parseMaybeJson(value: unknown): unknown { + if (typeof value !== "string") return value; + try { + return JSON.parse(value); + } catch { + return value; + } +} + +/** + * Reads a user's auth providers from `raw_app_meta_data`. + * + * The column arrives as a JSON string from a CSV export and as an object from + * a JSON one, and its `providers` value is itself sometimes a string. + */ +export function getUserProviders(user: Record): string[] { + const appMeta = parseMaybeJson(user.raw_app_meta_data); + if (!appMeta || typeof appMeta !== "object" || Array.isArray(appMeta)) return []; + + const providers = parseMaybeJson((appMeta as Record).providers); + if (Array.isArray(providers)) { + return providers.map((provider) => String(provider).trim()).filter(Boolean); + } + if (typeof providers === "string") { + return providers + .split(/[,|]/) + .map((provider) => provider.trim()) + .filter(Boolean); + } + return []; +} + +export type ProviderExclusions = { + /** Source IDs of users to skip. */ + excludedIds: Set; + /** How many excluded users each disabled provider accounts for. */ + byProvider: Record; +}; + +/** + * Finds the users whose *only* way in is a provider Clerk does not have + * enabled. + * + * A user keeps their place if any one of their providers still works — + * including email and phone. Excluding on "has at least one disabled provider" + * instead would drop users who could sign in perfectly well another way. + * + * @param disabled - Supabase provider keys not enabled in Clerk. + */ +export function findUsersWithOnlyDisabledProviders( + users: Record[], + disabled: string[], +): ProviderExclusions { + const empty: ProviderExclusions = { excludedIds: new Set(), byProvider: {} }; + if (disabled.length === 0) return empty; + + const disabledSet = new Set(disabled); + const excludedIds = new Set(); + const byProvider: Record = {}; + + for (const user of users) { + const providers = getUserProviders(user); + // No provider data means no basis to exclude — err towards importing. + if (providers.length === 0) continue; + + const hasUsableProvider = providers.some( + (provider) => NON_SOCIAL_PROVIDERS.has(provider) || !disabledSet.has(provider), + ); + if (hasUsableProvider) continue; + + excludedIds.add(String(user.id)); + for (const provider of providers.filter((p) => disabledSet.has(p))) { + byProvider[provider] = (byProvider[provider] ?? 0) + 1; + } + } + + return { excludedIds, byProvider }; +} + +/** Counts users per provider across the export, for reporting. */ +export function countProviders(users: Record[]): Record { + const counts: Record = {}; + for (const user of users) { + for (const provider of getUserProviders(user)) { + counts[provider] = (counts[provider] ?? 0) + 1; + } + } + return counts; +} + +/** + * The same counts, minus Supabase's pseudo-providers. + * + * Supabase lists `email` and `phone` in `providers` next to real connections, + * but Clerk has no `oauth_email` to enable — so anything cross-referencing + * against the instance's social settings must drop them, or every + * password-based user reads as "not enabled in Clerk". + */ +export function countSocialProviders(users: Record[]): Record { + return Object.fromEntries( + Object.entries(countProviders(users)).filter( + ([provider]) => !NON_SOCIAL_PROVIDERS.has(provider), + ), + ); +} + +/** + * Every social provider present in the export that Clerk does not have + * enabled. + * + * @param enabledStrategies - Clerk strategy names (`oauth_google`, …). + * @param toStrategy - Maps a Supabase provider key to its Clerk strategy. + */ +export function findDisabledProviders( + users: Record[], + enabledStrategies: string[], + toStrategy: (provider: string) => string, +): string[] { + const enabled = new Set(enabledStrategies); + return Object.keys(countSocialProviders(users)).filter( + (provider) => !enabled.has(toStrategy(provider)), + ); +} + +/** Reads a Supabase export and returns its raw rows for provider analysis. */ +export async function readSupabaseRows(file: string): Promise[]> { + return readRawUsers(file, "supabase"); +} diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts new file mode 100644 index 000000000..efb9f1e52 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -0,0 +1,218 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import clerkTransformer from "../transformers/clerk.ts"; +import { + consolidateClerkIdentifiers, + flattenObjectSelectively, + getFileType, + loadUsersFromFile, + normalizeUserData, + transformKeys, + transformUsers, + validatePreparedUsers, +} from "./transform.ts"; + +const DATE_TIME = "2026-01-01T00-00-00"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-transform-")); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("getFileType", () => { + test.each([ + ["users.json", "application/json"], + ["users.CSV", "text/csv"], + ["users.txt", undefined], + ["users", undefined], + ])("%s -> %p", (file, expected) => { + expect(getFileType(file)).toBe(expected as never); + }); +}); + +describe("flattenObjectSelectively", () => { + test("flattens only paths the transformer references", () => { + const result = flattenObjectSelectively( + { _id: { $oid: "123" }, meta: { keep: "nested" }, email: "a@example.com" }, + { "_id.$oid": "userId", email: "email" }, + ); + expect(result).toEqual({ + "_id.$oid": "123", + meta: { keep: "nested" }, + email: "a@example.com", + }); + }); + + test("leaves arrays intact", () => { + expect(flattenObjectSelectively({ tags: [{ a: 1 }] }, { "tags.a": "x" })).toEqual({ + tags: [{ a: 1 }], + }); + }); +}); + +describe("transformKeys", () => { + test("renames mapped fields and passes unmapped ones through", () => { + expect( + transformKeys( + { id: "u1", primary_email_address: "a@example.com", extra: "kept" }, + clerkTransformer, + ), + ).toEqual({ userId: "u1", email: "a@example.com", extra: "kept" }); + }); + + test.each([ + ["empty string", ""], + ["stringified empty object", '"{}"'], + ["null", null], + ])("drops fields whose value is %s", (_label, value) => { + expect(transformKeys({ id: "u1", first_name: value }, clerkTransformer)).toEqual({ + userId: "u1", + }); + }); +}); + +describe("normalizeUserData", () => { + test.each([ + ["comma-delimited emails", { email: "a@x.dev,b@x.dev" }, { email: ["a@x.dev", "b@x.dev"] }], + ["pipe-delimited emails", { email: "a@x.dev|b@x.dev" }, { email: ["a@x.dev", "b@x.dev"] }], + ["JSON array string", { email: '["a@x.dev"]' }, { email: ["a@x.dev"] }], + ["string boolean", { banned: "true" }, { banned: true }], + ["numeric boolean", { banned: 1 }, { banned: true }], + ["numeric string limit", { createOrganizationsLimit: "5" }, { createOrganizationsLimit: 5 }], + ["JSON metadata", { publicMetadata: '{"plan":"pro"}' }, { publicMetadata: { plan: "pro" } }], + ["date string", { createdAt: "2024-01-01" }, { createdAt: "2024-01-01T00:00:00.000Z" }], + ])("normalizes %s", (_label, input, expected) => { + expect(normalizeUserData(input)).toMatchObject(expected); + }); + + test("deletes fields that normalize to nothing", () => { + const result = normalizeUserData({ email: " ", publicMetadata: "", createdAt: "" }); + expect("email" in result).toBe(false); + expect("publicMetadata" in result).toBe(false); + expect("createdAt" in result).toBe(false); + }); + + test("leaves an unparseable date as-is for the schema to reject", () => { + expect(normalizeUserData({ createdAt: "yesterday" }).createdAt).toBe("yesterday"); + }); +}); + +describe("consolidateClerkIdentifiers", () => { + test("merges primary and verified emails, deduping", () => { + const user: Record = { + email: "a@x.dev", + emailAddresses: ["a@x.dev", "b@x.dev"], + unverifiedEmailAddresses: ["b@x.dev", "c@x.dev"], + }; + consolidateClerkIdentifiers(user); + expect(user.email).toEqual(["a@x.dev", "b@x.dev"]); + expect(user.emailAddresses).toBeUndefined(); + // b@x.dev is already verified, so it must not reappear as unverified. + expect(user.unverifiedEmailAddresses).toEqual(["c@x.dev"]); + }); + + test("drops the unverified list when every entry is already verified", () => { + const user: Record = { + phone: "+15555550100", + unverifiedPhoneNumbers: ["+15555550100"], + }; + consolidateClerkIdentifiers(user); + expect(user.phone).toEqual(["+15555550100"]); + expect("unverifiedPhoneNumbers" in user).toBe(false); + }); +}); + +describe("validatePreparedUsers", () => { + test("keeps valid users and counts the rest", () => { + const result = validatePreparedUsers( + [{ userId: "u1", email: "a@x.dev" }, { userId: "u2" }, { userId: "u3", username: "carol" }], + DATE_TIME, + ); + expect(result.users.map((user) => user.userId)).toEqual(["u1", "u3"]); + expect(result.validationFailed).toBe(1); + }); + + test("aborts the whole run on an unknown password hasher", () => { + expect(() => + validatePreparedUsers( + [{ userId: "u1", email: "a@x.dev", password: "d", passwordHasher: "rot13" }], + DATE_TIME, + ), + ).toThrow(CliError); + }); +}); + +describe("transformUsers", () => { + test("maps, consolidates and validates a Clerk export", () => { + const { transformedData, validationFailed } = transformUsers( + [ + { + id: "u1", + primary_email_address: "a@x.dev", + verified_email_addresses: ["a@x.dev", "b@x.dev"], + first_name: "Alice", + }, + ], + "clerk", + DATE_TIME, + ); + expect(validationFailed).toBe(0); + expect(transformedData[0]).toMatchObject({ + userId: "u1", + email: ["a@x.dev", "b@x.dev"], + firstName: "Alice", + }); + }); + + test("skips validation when asked, so analysis passes see every row", () => { + const { transformedData, validationFailed } = transformUsers( + [{ id: "u1" }], + "clerk", + DATE_TIME, + { + validate: false, + }, + ); + expect(transformedData).toHaveLength(1); + expect(validationFailed).toBe(0); + }); +}); + +describe("loadUsersFromFile", () => { + test("reads a JSON export", async () => { + fs.writeFileSync( + path.join(workDir, "users.json"), + JSON.stringify([{ id: "u1", primary_email_address: "a@x.dev" }]), + ); + const { users } = await loadUsersFromFile("users.json", "clerk", DATE_TIME); + expect(users).toHaveLength(1); + expect(users[0]?.userId).toBe("u1"); + }); + + test("reads a CSV export, including quoted commas", async () => { + fs.writeFileSync( + path.join(workDir, "users.csv"), + 'id,primary_email_address,verified_email_addresses\nu2,a@x.dev,"a@x.dev,b@x.dev"\n', + ); + const { users } = await loadUsersFromFile("users.csv", "clerk", DATE_TIME); + expect(users[0]?.userId).toBe("u2"); + expect(users[0]?.email).toEqual(["a@x.dev", "b@x.dev"]); + }); + + test("rejects a JSON file that is not an array of users", async () => { + fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); + await expect(loadUsersFromFile("wrapped.json", "clerk", DATE_TIME)).rejects.toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts new file mode 100644 index 000000000..9f1b4fe08 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -0,0 +1,469 @@ +/** + * The load → transform → validate pipeline. + * + * Ported from the standalone migration-tool's `src/migrate/functions.ts` and + * the transform helpers in its `src/lib/index.ts`. Two dependencies were + * dropped along the way: `mime-types` (an extension check covers the two + * formats we accept) and the repo-specific `/samples/` path special-case. + */ + +import fs from "node:fs"; +import path from "node:path"; +import csvParser from "csv-parser"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import { getTransformer } from "../transformers/registry.ts"; +import { + PASSWORD_HASHERS, + type TransformContext, + type TransformerRegistryEntry, + type User, +} from "../types.ts"; +import { userSchema } from "../validator.ts"; +import { validationLogger } from "./logger.ts"; + +export type FileType = "application/json" | "text/csv"; + +export type TransformOptions = { + /** Set `false` to keep invalid rows, for analysis passes that count fields. */ + validate?: boolean; + /** Per-run values `postTransform` may need. */ + context?: TransformContext; +}; + +/** Resolves an import path against the current working directory. */ +export function resolveImportFilePath(file: string): string { + return path.resolve(process.cwd(), file.trim()); +} + +export function fileExists(file: string): boolean { + return fs.existsSync(resolveImportFilePath(file)); +} + +/** + * Classifies an import file by extension. + * + * @returns The MIME type, or `undefined` for anything that is not JSON or CSV. + */ +export function getFileType(file: string): FileType | undefined { + const ext = path.extname(resolveImportFilePath(file)).toLowerCase(); + if (ext === ".json") return "application/json"; + if (ext === ".csv") return "text/csv"; + return undefined; +} + +// --- Field mapping --------------------------------------------------------- + +/** + * Flattens only the nested paths a transformer actually references. + * + * Lets a transformer map `"_id.$oid"` onto `userId` without flattening + * (and thereby mangling) metadata objects it does not mention. + */ +export function flattenObjectSelectively( + obj: Record, + transformer: Record, + prefix = "", +): Record { + const result: Record = {}; + + for (const [key, value] of Object.entries(obj)) { + const currentPath = prefix ? `${prefix}.${key}` : key; + const hasNestedMapping = Object.keys(transformer).some((mapped) => + mapped.startsWith(`${currentPath}.`), + ); + + if (hasNestedMapping && value && typeof value === "object" && !Array.isArray(value)) { + Object.assign( + result, + flattenObjectSelectively(value as Record, transformer, currentPath), + ); + } else { + result[currentPath] = value; + } + } + + return result; +} + +/** Renames source fields onto Clerk's import schema, dropping empty values. */ +export function transformKeys( + data: Record, + transformerConfig: { transformer: Record }, +): Record { + const transformed: Record = {}; + const { transformer } = transformerConfig; + const flat = flattenObjectSelectively(data, transformer); + + for (const [key, value] of Object.entries(flat)) { + if (value !== "" && value !== '"{}"' && value !== null) { + transformed[transformer[key] ?? key] = value; + } + } + + return transformed; +} + +// --- Value normalization --------------------------------------------------- + +function parseJsonValue(value: string): unknown { + const trimmed = value.trim(); + if (!trimmed) return value; + if (!["[", "{", '"'].includes(trimmed[0] ?? "")) return value; + try { + return JSON.parse(trimmed); + } catch { + return value; + } +} + +function parseDelimitedStrings(field: unknown): string[] { + if (Array.isArray(field)) return field as string[]; + if (typeof field === "string" && field) { + const parsed = parseJsonValue(field); + if (Array.isArray(parsed)) { + return parsed.map((value) => String(value).trim()).filter(Boolean); + } + return field + .split(/[,|]/) + .map((value) => value.trim()) + .filter(Boolean); + } + return []; +} + +function normalizeStringArrayField(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => String(item).trim()).filter(Boolean); + } + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const parsed = parseJsonValue(trimmed); + if (Array.isArray(parsed)) { + return parsed.map((item) => String(item).trim()).filter(Boolean); + } + if (typeof parsed === "string") { + const parsedString = parsed.trim(); + if (parsedString.includes(",") || parsedString.includes("|")) { + return parsedString + .split(/[,|]/) + .map((item) => item.trim()) + .filter(Boolean); + } + return parsedString; + } + return parsed; +} + +function normalizeBooleanField(value: unknown): unknown { + if (typeof value === "boolean") return value; + if (typeof value === "number") { + if (value === 1) return true; + if (value === 0) return false; + return value; + } + if (typeof value !== "string") return value; + + const normalized = value.trim().toLowerCase(); + if (["true", "1", "yes", "y"].includes(normalized)) return true; + if (["false", "0", "no", "n"].includes(normalized)) return false; + return value; +} + +function normalizeNumberField(value: unknown): unknown { + if (typeof value === "number") return value; + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? parsed : value; +} + +function normalizeMetadataField(value: unknown): unknown { + if (value === undefined || value === null || value === "") return undefined; + if (typeof value !== "string") return value; + + const parsed = parseJsonValue(value); + return typeof parsed === "string" ? value : parsed; +} + +function normalizeDateField(value: unknown): unknown { + if (value instanceof Date) return value.toISOString(); + if (typeof value === "number") { + const date = new Date(value); + return Number.isNaN(date.getTime()) ? value : date.toISOString(); + } + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const date = new Date(trimmed); + return Number.isNaN(date.getTime()) ? value : date.toISOString(); +} + +const ARRAY_FIELDS = [ + "email", + "emailAddresses", + "unverifiedEmailAddresses", + "phone", + "phoneNumbers", + "unverifiedPhoneNumbers", + "backupCodes", +] as const; + +const BOOLEAN_FIELDS = [ + "backupCodesEnabled", + "banned", + "bypassClientTrust", + "createOrganizationEnabled", + "deleteSelfEnabled", + "skipLegalChecks", + "skipPasswordChecks", +] as const; + +const METADATA_FIELDS = ["unsafeMetadata", "publicMetadata", "privateMetadata"] as const; + +const DATE_FIELDS = ["createdAt", "legalAcceptedAt"] as const; + +/** + * Coerces CSV's all-strings-everything into the shapes the schema expects. + * + * A field that normalizes to `undefined` is deleted rather than set, so an + * empty CSV column does not look like an explicitly-null value to Clerk. + */ +export function normalizeUserData(user: Record): Record { + const normalized = { ...user }; + + const setOrDelete = (field: string, value: unknown) => { + if (value === undefined) delete normalized[field]; + else normalized[field] = value; + }; + + for (const field of ARRAY_FIELDS) { + setOrDelete(field, normalizeStringArrayField(normalized[field])); + } + for (const field of BOOLEAN_FIELDS) { + normalized[field] = normalizeBooleanField(normalized[field]); + } + for (const field of METADATA_FIELDS) { + setOrDelete(field, normalizeMetadataField(normalized[field])); + } + for (const field of DATE_FIELDS) { + setOrDelete(field, normalizeDateField(normalized[field])); + } + setOrDelete( + "createOrganizationsLimit", + normalizeNumberField(normalized.createOrganizationsLimit), + ); + + return normalized; +} + +/** + * Merges a Clerk export's three email fields (and three phone fields) into the + * verified/unverified pair the schema models, deduping across all of them. + */ +export function consolidateClerkIdentifiers(user: Record): void { + const merge = (primaryKey: string, verifiedKey: string, unverifiedKey: string) => { + const primary = user[primaryKey] as string | undefined; + const verified = parseDelimitedStrings(user[verifiedKey]); + const unverified = parseDelimitedStrings(user[unverifiedKey]); + + const all: string[] = []; + if (primary) all.push(primary); + for (const value of verified) { + if (!all.includes(value)) all.push(value); + } + if (all.length > 0) user[primaryKey] = all; + delete user[verifiedKey]; + + const extraUnverified = unverified.filter((value) => !all.includes(value)); + if (extraUnverified.length > 0) user[unverifiedKey] = extraUnverified; + else delete user[unverifiedKey]; + }; + + merge("email", "emailAddresses", "unverifiedEmailAddresses"); + merge("phone", "phoneNumbers", "unverifiedPhoneNumbers"); +} + +// --- Validation ------------------------------------------------------------ + +/** + * Validates prepared users, logging each failure and dropping it from the run. + * + * An unrecognized `passwordHasher` is the one failure that aborts instead: + * importing those users would store credentials nobody can ever sign in with, + * and the fix is a one-word edit to the transformer. + */ +export function validatePreparedUsers( + users: Record[], + dateTime: string, +): { users: User[]; validationFailed: number } { + const validated: User[] = []; + let validationFailed = 0; + + for (let i = 0; i < users.length; i++) { + const user = users[i] as Record; + const result = userSchema.safeParse(user); + + if (result.success) { + validated.push(result.data); + continue; + } + + validationFailed++; + const firstIssue = result.error.issues[0]; + if (!firstIssue) continue; + + if (firstIssue.path.includes("passwordHasher") && user.passwordHasher) { + const invalidHasher = + typeof user.passwordHasher === "string" + ? user.passwordHasher + : JSON.stringify(user.passwordHasher); + throw new CliError( + `Invalid password hasher "${invalidHasher}" on user ${String(user.userId)} (row ${i + 1}).\n` + + `Expected one of: ${PASSWORD_HASHERS.join(", ")}`, + { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: "https://clerk.com/docs/guides/development/migrating/overview", + }, + ); + } + + validationLogger( + { + error: firstIssue.message, + path: firstIssue.path as (string | number)[], + userId: (user.userId as string) || `row-${i}`, + row: i, + }, + dateTime, + ); + } + + return { users: validated, validationFailed }; +} + +function addDefaultFields( + users: Record[], + transformer: TransformerRegistryEntry, +): Record[] { + if (!transformer.defaults) return users; + return users.map((user) => ({ ...user, ...transformer.defaults })); +} + +/** + * Maps, normalizes and (unless disabled) validates a batch of raw users. + * + * @param options.validate - Set `false` to get the mapped shape without + * dropping invalid rows, for analysis passes that count fields. + * @param options.context - Per-run values `postTransform` may need, e.g. + * Firebase's hash parameters. + */ +export function transformUsers( + users: Record[], + key: string, + dateTime: string, + options: TransformOptions = {}, +): { transformedData: User[]; validationFailed: number } { + const transformer = getTransformer(key); + const context = options.context ?? {}; + const transformed: Record[] = []; + + for (const user of users) { + const mapped = transformKeys(user, transformer); + + if (key === "clerk") { + consolidateClerkIdentifiers(mapped); + } + transformer.postTransform?.(mapped, context); + + transformed.push(normalizeUserData(mapped)); + } + + if (options.validate === false) { + return { transformedData: transformed as User[], validationFailed: 0 }; + } + + const result = validatePreparedUsers(transformed, dateTime); + return { transformedData: result.users, validationFailed: result.validationFailed }; +} + +// --- File loading ---------------------------------------------------------- + +async function readCsv(filePath: string): Promise[]> { + return new Promise((resolve, reject) => { + const users: Record[] = []; + fs.createReadStream(filePath) + .pipe(csvParser({ skipComments: true })) + .on("data", (row: Record) => users.push(row)) + .on("error", reject) + .on("end", () => resolve(users)); + }); +} + +async function readUsersFromFile( + file: string, + transformer: TransformerRegistryEntry, +): Promise[]> { + let filePath = resolveImportFilePath(file); + const type = getFileType(file); + let preExtracted: Record[] | undefined; + + if (transformer.preTransform) { + const result = await transformer.preTransform(filePath, type ?? ""); + filePath = result.filePath; + preExtracted = result.data; + } + + if (type === "text/csv") return readCsv(filePath); + if (preExtracted) return preExtracted; + + const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + if (!Array.isArray(parsed)) { + throw new CliError(`Expected ${file} to contain a JSON array of users, got ${typeof parsed}.`, { + code: ERROR_CODE.INVALID_JSON, + }); + } + return parsed as Record[]; +} + +/** + * Reads the export exactly as the transformer sees it, before any field + * mapping. + * + * Used by the Supabase provider cross-reference, which reads + * `raw_app_meta_data` — a column no transformer maps, so it is gone by the time + * users are transformed. + */ +export async function readRawUsers(file: string, key: string): Promise[]> { + return readUsersFromFile(file, getTransformer(key)); +} + +/** + * Reads a JSON or CSV export and returns the users ready to import. + * + * @param options - Passed through to {@link transformUsers}. + */ +export async function loadUsersFromFile( + file: string, + key: string, + dateTime: string, + options: TransformOptions = {}, +): Promise<{ users: User[]; validationFailed: number }> { + const transformer = getTransformer(key); + const raw = await readUsersFromFile(file, transformer); + const withDefaults = addDefaultFields(raw, transformer); + const { transformedData, validationFailed } = transformUsers( + withDefaults, + key, + dateTime, + options, + ); + return { users: transformedData, validationFailed }; +} diff --git a/packages/cli-core/src/commands/migrate/logs/clean.ts b/packages/cli-core/src/commands/migrate/logs/clean.ts new file mode 100644 index 000000000..4d17d684d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/clean.ts @@ -0,0 +1,73 @@ +/** + * `clerk migrate logs clean` — delete the local log files. + * + * Ported from the standalone migration-tool's `src/clean-logs/index.ts`. + * + * Destructive, and it sits one word away from `clerk migrate delete`, which + * destroys something entirely different (users in a Clerk instance). So the + * confirmation is not optional: interactive runs prompt, and non-interactive + * ones must say `-y` rather than being allowed to assume. + */ + +import fs from "node:fs"; +import { throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { confirm } from "../../../lib/prompts.ts"; +import { withGutter } from "../../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { listLogFiles } from "../lib/log-files.ts"; +import { getLogDir, resolveLogDir } from "../lib/logger.ts"; + +export type LogsCleanOptions = { + yes?: boolean; +}; + +export async function clean(options: LogsCleanOptions = {}): Promise { + await withGutter("Cleaning migration logs", async () => { + await resolveLogDir(); + const files = listLogFiles(); + + if (files.length === 0) { + log.info(`No migration logs to clean in ${getLogDir()}.`); + return; + } + + const label = `${files.length} log file${files.length === 1 ? "" : "s"}`; + + if (!options.yes) { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate logs clean\` deletes ${label} from ${getLogDir()} and cannot prompt here. Pass -y to confirm.`, + undefined, + undefined, + [ + { + command: "clerk migrate logs clean -y", + description: "Delete every migration log without prompting", + }, + ], + ); + } + + const proceed = await confirm({ message: `Delete ${label}?`, default: false }); + if (!proceed) throwUserAbort(); + } + + let deleted = 0; + const failures: string[] = []; + + for (const file of files) { + try { + fs.unlinkSync(file.path); + deleted++; + } catch (error) { + failures.push(`${file.name}: ${(error as Error).message}`); + } + } + + for (const failure of failures) log.warn(`Could not delete ${failure}`); + + log.success(`Deleted ${deleted} log file${deleted === 1 ? "" : "s"}.`); + if (failures.length > 0) process.exitCode = 1; + }); +} diff --git a/packages/cli-core/src/commands/migrate/logs/convert.ts b/packages/cli-core/src/commands/migrate/logs/convert.ts new file mode 100644 index 000000000..4536e70a5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/convert.ts @@ -0,0 +1,129 @@ +/** + * `clerk migrate logs convert` — NDJSON to a JSON array. + * + * Ported from the standalone migration-tool's `src/convert-logs/index.ts`, + * with two changes: files can be named as positionals or `--all` instead of + * only through a picker, and a malformed line is reported with its line number + * rather than aborting the whole file. + */ + +import fs from "node:fs"; +import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { multiselect } from "../../../lib/prompts.ts"; +import { withGutter } from "../../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { findLogFile, listLogFiles, readNdjson, type LogFile } from "../lib/log-files.ts"; +import { getLogDir, resolveLogDir } from "../lib/logger.ts"; + +export type LogsConvertOptions = { + all?: boolean; + files?: string[]; +}; + +/** The `.json` sibling a log converts into. */ +export function outputPathFor(file: LogFile): string { + return file.path.replace(/\.log$/, ".json"); +} + +/** + * Resolves which files to convert: explicit positionals, `--all`, or a + * multiselect when a human gave neither. + */ +async function resolveTargets(options: LogsConvertOptions): Promise { + const available = listLogFiles(); + + if (available.length === 0) { + log.info(`No migration logs to convert in ${getLogDir()}.`); + return []; + } + + if (options.files && options.files.length > 0) { + return options.files.map((name) => { + const found = findLogFile(name); + if (!found) { + throw new CliError(`No log file named ${name} in ${getLogDir()}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + }); + } + return found; + }); + } + + if (options.all) return available; + + if (isAgent() || !isHuman()) { + throwUsageError( + "`clerk migrate logs convert` needs a file to convert and cannot prompt here. Name one or more log files, or pass --all.", + undefined, + undefined, + [ + { command: "clerk migrate logs convert --all", description: "Convert every log file" }, + { + command: `clerk migrate logs convert ${available[0]?.name ?? "migration-....log"}`, + description: "Convert one log file", + }, + ], + ); + } + + const chosen = await multiselect({ + message: "Which log files should be converted to JSON?", + options: available.map((file) => ({ + value: file.name, + label: file.name, + hint: `${file.entryCount} entries`, + })), + }); + if (chosen.length === 0) throwUserAbort(); + + return available.filter((file) => chosen.includes(file.name)); +} + +export async function convert(options: LogsConvertOptions = {}): Promise { + // The multiselect lives inside the gutter so cancelling it closes with + // `└ Paused` rather than leaving a half-drawn frame. + await withGutter("Converting migration logs", async () => { + await resolveLogDir(); + const targets = await resolveTargets(options); + if (targets.length === 0) return; + + let converted = 0; + let malformed = 0; + + for (const file of targets) { + const output = outputPathFor(file); + + try { + const { entries, errors } = readNdjson(file.path); + + // Reported per line, so a truncated final line from an interrupted run + // is visible rather than silently missing from the output. + for (const error of errors) { + malformed++; + log.warn( + `${file.name}:${error.line} is not valid JSON and was skipped — ${error.message}`, + ); + } + + fs.writeFileSync(output, JSON.stringify(entries, null, 2)); + converted++; + const count = `${entries.length} ${entries.length === 1 ? "entry" : "entries"}`; + log.info(`${file.name} → ${output.split("/").pop()} ${dim(`(${count})`)}`); + } catch (error) { + log.warn(`Could not convert ${file.name}: ${(error as Error).message}`); + process.exitCode = 1; + } + } + + if (converted > 0) { + log.success( + `Converted ${converted} log file${converted === 1 ? "" : "s"}. Originals left in place.`, + ); + } + if (malformed > 0) { + log.warn(`${malformed} malformed line${malformed === 1 ? "" : "s"} skipped.`); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/logs/index.ts b/packages/cli-core/src/commands/migrate/logs/index.ts new file mode 100644 index 000000000..abb352d34 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/index.ts @@ -0,0 +1,76 @@ +import { createArgument } from "@commander-js/extra-typings"; +import type { Command } from "@commander-js/extra-typings"; +import { clean } from "./clean.ts"; +import { convert } from "./convert.ts"; +import { list } from "./list.ts"; + +const logs = { clean, convert, list }; + +/** + * Registers `logs list|clean|convert` under the `migrate` group. + * + * Noun-verb, matching every other group in the CLI (`config pull`, `users + * list`) rather than the standalone tool's `clean-logs`/`convert-logs`, which + * were npm script names. Grouping also disambiguates the two deletes in this + * tree: `migrate logs clean` removes local files, `migrate delete` removes + * users from a Clerk instance. + */ +export function registerMigrateLogs(migrateCommand: Command<[], Record>): void { + const logsCommand = migrateCommand + .command("logs") + .description("Inspect, convert and clean up local migration logs") + .setExamples([ + { command: "clerk migrate logs", description: "List the local migration logs" }, + { command: "clerk migrate logs clean -y", description: "Delete every migration log" }, + { + command: "clerk migrate logs convert --all", + description: "Convert every log to a JSON array", + }, + ]); + + // Listing is read-only, so it is safe as the default for a bare + // `clerk migrate logs`. + logsCommand + .command("list", { isDefault: true }) + .description("List the migration log files") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate logs list", description: "Show type, timestamp, size and entries" }, + { command: "clerk migrate logs list --json", description: "Machine-readable listing" }, + ]) + .action(async (_opts, cmd) => + logs.list(cmd.optsWithGlobals() as Parameters[0]), + ); + + logsCommand + .command("clean") + .description("Delete the migration log files") + .option("-y, --yes", "Skip the confirmation prompt") + .setExamples([ + { command: "clerk migrate logs clean", description: "Delete after confirming" }, + { command: "clerk migrate logs clean -y", description: "Delete without prompting" }, + ]) + .action(async (_opts, cmd) => + logs.clean(cmd.optsWithGlobals() as Parameters[0]), + ); + + logsCommand + .command("convert") + .description("Convert NDJSON logs to JSON arrays for analysis") + .addArgument(createArgument("[file...]", "Log files to convert. Omit to pick interactively.")) + .option("--all", "Convert every log file") + .setExamples([ + { command: "clerk migrate logs convert --all", description: "Convert every log file" }, + { + command: "clerk migrate logs convert migration-2026-01-01T12-00-00.log", + description: "Convert one log file", + }, + { command: "clerk migrate logs convert", description: "Pick files interactively" }, + ]) + .action(async (files, _opts, cmd) => + logs.convert({ + ...(cmd.optsWithGlobals() as Parameters[0]), + files, + }), + ); +} diff --git a/packages/cli-core/src/commands/migrate/logs/list.ts b/packages/cli-core/src/commands/migrate/logs/list.ts new file mode 100644 index 000000000..81806166e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/list.ts @@ -0,0 +1,128 @@ +/** + * `clerk migrate logs list` — what is in `./logs/`. + * + * New in the CLI: the standalone tool enumerated the directory only to build + * its own pickers. Exposing it gives a human a "what did I just do" view and + * an agent a read-only way to inspect a migration without parsing NDJSON. + */ + +import { bold, cyan, dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { withGutter } from "../../../lib/spinner.ts"; +import { formatSize, listLogFiles, type LogFile, type LogKind } from "../lib/log-files.ts"; +import { displayLogDir, resolveLogDir } from "../lib/logger.ts"; + +/** Every kind a log file can be, and what one entry in it records. */ +const KIND_LEGEND: Record, string> = { + export: "One entry per user pulled from the source platform.", + import: "One entry per user created in Clerk, with any error.", + delete: "One entry per user removed from Clerk, with any error.", +}; + +const legendWidth = Math.max(...Object.keys(KIND_LEGEND).map((kind) => kind.length)) + 2; + +export type LogsListOptions = { + json?: boolean; +}; + +function toJson(files: LogFile[]) { + return files.map((file) => ({ + name: file.name, + kind: file.kind, + timestamp: file.timestamp, + size_bytes: file.sizeBytes, + entry_count: file.entryCount, + path: file.path, + })); +} + +/** + * The filename stamp as a date a human reads at a glance. + * + * The stamp is UTC (`getDateTimeStamp` is an ISO string with its colons swapped + * for filename-legal dashes), so it is parsed as UTC and rendered in the + * viewer's own zone — "which run was that" is a question about local time. + * Returns the raw stamp for anything unparseable rather than printing + * "Invalid Date". + */ +export function formatTimestamp(stamp: string): string { + if (!stamp) return ""; + const date = new Date(`${stamp.replace(/T(\d{2})-(\d{2})-(\d{2})$/, "T$1:$2:$3")}Z`); + if (Number.isNaN(date.getTime())) return stamp; + return date.toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short" }); +} + +export async function list(options: LogsListOptions = {}): Promise { + // Resolve, never ask: listing is read-only, and "where should logs go?" is + // not a question to answer before showing someone the ones they have. + await resolveLogDir(); + const files = listLogFiles(); + + if (options.json) { + log.data(JSON.stringify(toJson(files), null, 2)); + return; + } + + await withGutter("Listing migration logs", async () => { + if (files.length === 0) { + log.info(`No migration logs in ${displayLogDir()}.`); + return; + } + + const rows = files.map((file) => ({ + name: file.name, + kind: file.kind, + when: formatTimestamp(file.timestamp), + size: formatSize(file.sizeBytes), + entries: String(file.entryCount), + })); + + const width = (header: string, pick: (row: (typeof rows)[number]) => string) => + Math.max(header.length, ...rows.map((row) => pick(row).length)) + 2; + + /** + * Pads to the visible width, then colours. Colouring first would count the + * ANSI escape bytes towards the width and pull every later column left. + */ + const column = (text: string, size: number, paint: (value: string) => string) => + paint(text) + " ".repeat(Math.max(0, size - text.length)); + + const nameWidth = width("FILE", (row) => row.name); + const kindWidth = width("TYPE", (row) => row.kind); + const whenWidth = width("DATE", (row) => row.when); + const sizeWidth = width("SIZE", (row) => row.size); + + log.info("Each log represents a user export, user import, or a user delete run."); + log.info("Each log consists of a single NDJSON entry per user."); + log.blank(); + + log.info( + dim("FILE".padEnd(nameWidth)) + + dim("TYPE".padEnd(kindWidth)) + + dim("DATE".padEnd(whenWidth)) + + dim("SIZE".padEnd(sizeWidth)) + + dim("ENTRIES"), + ); + + for (const row of rows) { + log.info( + column(row.name, nameWidth, cyan) + + row.kind.padEnd(kindWidth) + + column(row.when || "—", whenWidth, row.when ? (value) => value : dim) + + column(row.size, sizeWidth, dim) + + row.entries, + ); + } + + log.blank(); + log.info(`${files.length} log file${files.length === 1 ? "" : "s"} in ${displayLogDir()}`); + log.blank(); + + // A listing only shows the kinds that happen to be present, so the legend + // is fixed: it also answers "what else could be here". + log.info(bold("Log types:")); + for (const [kind, description] of Object.entries(KIND_LEGEND)) { + log.info(` ${cyan(bold(kind))}${" ".repeat(legendWidth - kind.length)}${description}`); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts new file mode 100644 index 000000000..72da1cffc --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/logs-interactive.test.ts @@ -0,0 +1,201 @@ +/** + * The prompting half of `logs clean` and `logs convert`. + * + * Kept separate because `mock.module` registrations are process-lifetime, and + * `bun test --parallel` puts several files in each worker — so a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. Human mode itself needs no mock: `setMode` is the + * supported override. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +type ConfirmPrompt = { message: string; default?: boolean }; +type MultiselectPrompt = { + message: string; + options: { value: string; label: string; hint?: string }[]; +}; + +const mockConfirm = mock(async (_config: ConfirmPrompt) => true); +const mockMultiselect = mock(async (_config: MultiselectPrompt) => [] as string[]); + +mock.module("../../../lib/prompts.ts", () => ({ + confirm: (config: ConfirmPrompt) => mockConfirm(config), + multiselect: (config: MultiselectPrompt) => mockMultiselect(config), + text: async () => "", + password: async () => "", + editor: async () => "{}", +})); + +const { clean } = await import("./clean.ts"); +const { convert } = await import("./convert.ts"); +const { UserAbortError } = await import("../../../lib/errors.ts"); +const { getLogDir } = await import("../lib/logger.ts"); + +let originalMode: Mode; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; + +const IMPORT = "import-2026-01-01T12-00-00.log"; +const DELETE = "delete-2026-02-01T12-00-00.log"; + +beforeAll(() => { + originalMode = getMode(); + setMode("human"); + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logs-int-"))); + process.chdir(workDir); +}); + +afterAll(() => { + setMode(originalMode); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + mockConfirm.mockReset(); + mockMultiselect.mockReset(); + mockConfirm.mockResolvedValue(true); + mockMultiselect.mockResolvedValue([]); + fs.rmSync(getLogDir(), { recursive: true, force: true }); + process.exitCode = 0; +}); + +function writeLog(name: string, entries: unknown[]): void { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync( + path.join(getLogDir(), name), + entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n", + ); +} + +describe("logs clean", () => { + test("prompts before deleting anything", async () => { + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); + + await clean(); + + expect(mockConfirm).toHaveBeenCalledTimes(1); + expect(mockConfirm.mock.calls[0]?.[0]?.message).toContain("2 log files"); + expect(fs.readdirSync(getLogDir())).toEqual([]); + }); + + // Deleting on a stray enter would be the wrong default for a destructive + // command sitting next to `clerk migrate delete`. + test("defaults the prompt to no", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await clean(); + + expect(mockConfirm.mock.calls[0]?.[0]?.default).toBe(false); + }); + + test("declining leaves every file in place", async () => { + writeLog(IMPORT, [{ a: 1 }]); + mockConfirm.mockResolvedValue(false); + + await expect(clean()).rejects.toThrow(UserAbortError); + + expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); + }); + + test("-y skips the prompt entirely", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await clean({ yes: true }); + + expect(mockConfirm).not.toHaveBeenCalled(); + expect(fs.readdirSync(getLogDir())).toEqual([]); + }); + + test("does not prompt when there is nothing to delete", async () => { + await clean(); + expect(mockConfirm).not.toHaveBeenCalled(); + }); +}); + +describe("logs convert", () => { + test("offers a multiselect when given neither files nor --all", async () => { + writeLog(IMPORT, [{ a: 1 }, { b: 2 }]); + writeLog(DELETE, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([IMPORT]); + + await convert(); + + const options = mockMultiselect.mock.calls[0]?.[0]?.options; + expect(options?.map((option) => option.value)).toEqual([DELETE, IMPORT]); + expect(options?.[1]?.hint).toBe("2 entries"); + }); + + test("converts only what was selected", async () => { + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([IMPORT]); + + await convert(); + + expect(fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json"))).toEqual([ + "import-2026-01-01T12-00-00.json", + ]); + }); + + test("selecting nothing aborts without writing", async () => { + writeLog(IMPORT, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([]); + + await expect(convert()).rejects.toThrow(UserAbortError); + + expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); + }); + + test("does not prompt when --all was passed", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await convert({ all: true }); + + expect(mockMultiselect).not.toHaveBeenCalled(); + expect(captured.err).toContain("Converted 1 log file"); + }); + + test("does not prompt when files were named", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await convert({ files: [IMPORT] }); + + expect(mockMultiselect).not.toHaveBeenCalled(); + }); +}); + +// withGutter turns a UserAbortError into `└ Paused`; a real failure would close +// with `└ Failed`. Declining a prompt is not a failure, so the two must not swap. +describe("cancelling inside the gutter", () => { + test("declining the logs clean confirm closes with Paused, not Failed", async () => { + writeLog(IMPORT, [{ a: 1 }]); + mockConfirm.mockResolvedValue(false); + + await expect(clean()).rejects.toThrow(UserAbortError); + + expect(captured.err).toContain("Paused"); + expect(captured.err).not.toContain("Failed"); + }); + + test("selecting nothing in the logs convert multiselect closes with Paused", async () => { + writeLog(IMPORT, [{ a: 1 }]); + mockMultiselect.mockResolvedValue([]); + + await expect(convert()).rejects.toThrow(UserAbortError); + + expect(captured.err).toContain("Paused"); + expect(captured.err).not.toContain("Failed"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/logs/logs.test.ts b/packages/cli-core/src/commands/migrate/logs/logs.test.ts new file mode 100644 index 000000000..2e80ce5d5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/logs/logs.test.ts @@ -0,0 +1,288 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { clean } from "./clean.ts"; +import { convert } from "./convert.ts"; +import { formatTimestamp, list } from "./list.ts"; + +const captured = useCaptureLog(); + +const ANSI_ESCAPE_PATTERN = new RegExp(String.raw`\u001b\[[0-9;]*m`, "g"); +const stripAnsi = (value: string) => value.replace(ANSI_ESCAPE_PATTERN, ""); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-logs-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); + process.exitCode = 0; +}); + +function writeLog(name: string, entries: unknown[]): void { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync( + path.join(getLogDir(), name), + entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n", + ); +} + +const IMPORT = "import-2026-01-01T12-00-00.log"; +const DELETE = "delete-2026-02-01T12-00-00.log"; + +describe("logs list", () => { + test("says so plainly when there is no logs directory", async () => { + await list(); + expect(captured.err).toContain("No migration logs in"); + }); + + test("says so plainly when the directory is empty", async () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + await list(); + expect(captured.err).toContain("No migration logs in"); + }); + + test("reports file, type, date, size and entry count", async () => { + writeLog(IMPORT, [{ userId: "u1" }, { userId: "u2" }, { userId: "u3" }]); + + await list(); + + expect(captured.err).toContain("FILE"); + expect(captured.err).toContain("TYPE"); + expect(captured.err).toContain("DATE"); + expect(captured.err).toContain("SIZE"); + expect(captured.err).toContain("ENTRIES"); + expect(captured.err).toContain(IMPORT); + expect(captured.err).toContain("import"); + expect(captured.err).toContain(formatTimestamp("2026-01-01T12-00-00")); + expect(captured.err).toMatch(/\bB\b/); + expect(captured.err).toContain("3"); + }); + + // The filename stamp is for sorting and for `logs convert`; the column a + // human scans should read like a date. + test("shows the date rendered, not the raw filename stamp", async () => { + writeLog(IMPORT, [{ userId: "u1" }]); + + await list(); + + const dateColumn = stripAnsi(captured.err) + .split("\n") + .find((line) => line.includes(IMPORT)); + expect(dateColumn?.replace(IMPORT, "")).not.toContain("2026-01-01T12-00-00"); + }); + + test("reports the log directory relative to the current directory", async () => { + writeLog(IMPORT, [{ userId: "u1" }]); + + await list(); + + expect(stripAnsi(captured.err)).toContain(`1 log file in .${path.sep}logs`); + expect(captured.err).not.toContain(getLogDir()); + }); + + test("lists every log kind", async () => { + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); + + await list(); + + expect(captured.err).toContain("import"); + expect(captured.err).toContain("delete"); + expect(captured.err).toContain("2 log files"); + }); + + test("--json emits a machine-readable listing on stdout", async () => { + writeLog(IMPORT, [{ userId: "u1" }]); + + await list({ json: true }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed).toHaveLength(1); + expect(parsed[0]).toMatchObject({ + name: IMPORT, + kind: "import", + timestamp: "2026-01-01T12-00-00", + entry_count: 1, + }); + }); + + test("--json emits an empty array rather than prose when there are no logs", async () => { + await list({ json: true }); + expect(JSON.parse(captured.out)).toEqual([]); + }); +}); + +describe("logs clean", () => { + test("says so plainly when there is nothing to clean", async () => { + await clean({ yes: true }); + expect(captured.err).toContain("No migration logs to clean"); + }); + + // Tests run non-TTY, which is the same signal an agent gives. + test("refuses without -y when it cannot prompt, and explains", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await expect(clean()).rejects.toThrow(/cannot prompt here.*Pass -y/s); + expect(fs.existsSync(path.join(getLogDir(), IMPORT))).toBe(true); + }); + + test("names how many files are at stake when it refuses", async () => { + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); + + await expect(clean()).rejects.toThrow(/2 log files/); + }); + + test("-y deletes the log files and reports the count", async () => { + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ a: 1 }]); + + await clean({ yes: true }); + + expect(fs.readdirSync(getLogDir())).toEqual([]); + expect(captured.err).toContain("Deleted 2 log files"); + }); + + test("leaves converted JSON output alone", async () => { + writeLog(IMPORT, [{ a: 1 }]); + fs.writeFileSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"), "[]"); + + await clean({ yes: true }); + + expect(fs.readdirSync(getLogDir())).toEqual(["import-2026-01-01T12-00-00.json"]); + }); +}); + +describe("logs convert", () => { + test("says so plainly when there is nothing to convert", async () => { + await convert({ all: true }); + expect(captured.err).toContain("No migration logs to convert"); + }); + + test("writes a JSON array alongside the original, leaving it intact", async () => { + writeLog(IMPORT, [{ userId: "u1" }, { userId: "u2" }]); + + await convert({ files: [IMPORT] }); + + const output = path.join(getLogDir(), "import-2026-01-01T12-00-00.json"); + expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([ + { userId: "u1" }, + { userId: "u2" }, + ]); + expect(fs.existsSync(path.join(getLogDir(), IMPORT))).toBe(true); + expect(captured.err).toContain("Originals left in place"); + }); + + test("--all converts every log file", async () => { + writeLog(IMPORT, [{ a: 1 }]); + writeLog(DELETE, [{ b: 2 }]); + + await convert({ all: true }); + + const written = fs.readdirSync(getLogDir()).filter((name) => name.endsWith(".json")); + expect(written.sort()).toEqual([ + "delete-2026-02-01T12-00-00.json", + "import-2026-01-01T12-00-00.json", + ]); + }); + + test("accepts a path and resolves it against ./logs/", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await convert({ files: [`./logs/${IMPORT}`] }); + + expect(fs.existsSync(path.join(getLogDir(), "import-2026-01-01T12-00-00.json"))).toBe(true); + }); + + test("fails clearly on a file that is not there", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await expect(convert({ files: ["migration-nope.log"] })).rejects.toThrow(CliError); + }); + + // Silently dropping the line would leave a JSON array that looks complete. + test("reports a malformed line by number and converts the rest", async () => { + fs.mkdirSync(getLogDir(), { recursive: true }); + fs.writeFileSync(path.join(getLogDir(), IMPORT), '{"a":1}\n{"b":\n{"c":3}\n'); + + await convert({ files: [IMPORT] }); + + expect(captured.err).toContain(`${IMPORT}:2`); + expect(captured.err).toContain("1 malformed line skipped"); + + const output = path.join(getLogDir(), "import-2026-01-01T12-00-00.json"); + expect(JSON.parse(fs.readFileSync(output, "utf-8"))).toEqual([{ a: 1 }, { c: 3 }]); + }); + + test("refuses without a target when it cannot prompt, naming the alternatives", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await expect(convert()).rejects.toThrow(/cannot prompt here/); + expect(fs.readdirSync(getLogDir())).toEqual([IMPORT]); + }); + + test("reports the entry count per converted file", async () => { + writeLog(IMPORT, [{ a: 1 }, { b: 2 }, { c: 3 }]); + + await convert({ all: true }); + + expect(captured.err).toContain("3 entries"); + }); +}); + +describe("human-mode frame", () => { + let originalMode: Mode; + + beforeAll(() => { + originalMode = getMode(); + setMode("human"); + }); + + afterAll(() => { + setMode(originalMode); + }); + + test("logs list wraps its output in an intro/outro gutter", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await list(); + + expect(captured.err).toContain("\u250c"); + expect(captured.err).toContain("Listing migration logs"); + expect(captured.err).toContain("\u2514"); + expect(captured.err).toContain("Done"); + }); + + test("--json stays outside the gutter, on stdout only", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await list({ json: true }); + + expect(JSON.parse(captured.out)).toHaveLength(1); + expect(captured.err).not.toContain("\u250c"); + }); + + test("a failure inside logs convert closes with Failed and still throws", async () => { + writeLog(IMPORT, [{ a: 1 }]); + + await expect(convert({ files: ["nope.log"] })).rejects.toThrow(CliError); + + expect(captured.err).toContain("Failed"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts new file mode 100644 index 000000000..150bf3713 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -0,0 +1,146 @@ +/** + * Keeps README.md and the command tree honest about each other. + * + * This README documents six export platforms, six transformers and three log + * subcommands across ~600 lines. Checking it by eye at review time does not + * scale, and a doc that names a flag the binary rejects is worse than no doc: + * the reader trusts it and gets a usage error. + * + * Both directions are checked — every example must resolve, and every flag must + * be written down — so neither renaming a flag nor adding one passes silently. + */ + +import { describe, expect, test } from "bun:test"; +import type { Command } from "commander"; +import { createProgram } from "../../cli-program.ts"; + +const README = await Bun.file(new URL("./README.md", import.meta.url)).text(); + +/** Fenced blocks only, so prose that merely mentions a flag is not parsed. */ +function fencedBlocks(markdown: string): string[] { + return [...markdown.matchAll(/^```[a-z]*\n([\s\S]*?)^```/gm)].map((match) => match[1] ?? ""); +} + +/** + * Every `clerk migrate …` invocation the README puts in front of a reader, from + * fenced blocks and inline backticks alike — both get copied. + */ +function documentedCommands(markdown: string): string[] { + const found = new Set(); + + for (const block of fencedBlocks(markdown)) { + // Line continuations first: the Firebase example spans three lines. + for (const line of block.replace(/\\\n\s*/g, " ").split("\n")) { + const start = line.indexOf("clerk migrate"); + // A command never contains a backtick, a `#`, or a run of two spaces; + // the sample error output that quotes `clerk migrate` mid-sentence does, + // and so does a pasted listing whose descriptions sit in a padded column. + if (start !== -1) + found.add( + line + .slice(start) + .split(/[`#]|\s{2,}/)[0]! + .trim(), + ); + } + } + + for (const match of markdown.matchAll(/`(clerk migrate[^`]*)`/g)) { + found.add(match[1]!.trim()); + } + + return [...found]; +} + +/** Walks as deep as the tree allows; the first flag or positional stops it. */ +function resolve(tokens: string[]): { command: Command; rest: string[] } { + let command = createProgram() as Command; + let index = 0; + for (; index < tokens.length; index++) { + const child = command.commands.find( + (candidate) => + candidate.name() === tokens[index] || candidate.aliases().includes(tokens[index]!), + ); + if (!child) break; + command = child; + } + return { command, rest: tokens.slice(index) }; +} + +/** + * The flags a command accepts, including those of a default subcommand. + * + * `migrate logs` and `migrate transformers` register their `list` `isDefault`, + * so Commander hands it everything after the group name. The documented + * spelling is `clerk migrate logs --json`, and this has to see the same flags + * Commander does or every such example reads as unsupported. + */ +function flagsOf(command: Command): string[] { + const own = command.options.flatMap( + (option) => [option.short, option.long].filter(Boolean) as string[], + ); + + const defaultChild = command.commands.find( + // Commander records the default subcommand on the parent, not the child. + (child) => child.name() === (command as any)._defaultCommandName, + ); + + return defaultChild ? [...own, ...flagsOf(defaultChild)] : own; +} + +/** Every command under `migrate`, so no subcommand escapes the flag sweep. */ +function migrateTree(): { path: string; command: Command }[] { + const collected: { path: string; command: Command }[] = []; + const visit = (command: Command, path: string) => { + collected.push({ path, command }); + for (const child of command.commands) { + if (child.name() !== "help") visit(child, `${path} ${child.name()}`); + } + }; + visit(resolve(["migrate"]).command, "migrate"); + return collected; +} + +const EXAMPLES = documentedCommands(README); + +/** One case per (example, flag) pair, so a failure names the exact flag. */ +const FLAG_USES: [string, string][] = EXAMPLES.flatMap((example) => + example + .split(/\s+/) + .filter((token) => token.startsWith("-")) + .map((token) => [example, token.split("=")[0]!] as [string, string]), +); + +describe("migrate README", () => { + // Guards the extractor: a regex that silently matched nothing would make + // every check below pass vacuously. + test("finds the documented examples", () => { + expect(EXAMPLES.length).toBeGreaterThan(20); + expect(FLAG_USES.length).toBeGreaterThan(20); + }); + + test.each(EXAMPLES)("`%s` resolves to a real command", (example) => { + const tokens = example.split(/\s+/).slice(1); + const { command, rest } = resolve(tokens); + const firstFlag = rest.findIndex((token) => token.startsWith("-")); + const positionals = firstFlag === -1 ? rest : rest.slice(0, firstFlag); + // Leftover words before any flag are positionals — only some commands take + // them, and a subcommand that does not exist lands here too. + if (positionals.length > 0) expect(command.registeredArguments.length).toBeGreaterThan(0); + // A parent means at least `migrate` resolved. Checking the name instead + // would be wrong: `migrate export clerk` is itself named `clerk`. + expect(command.parent).not.toBeNull(); + }); + + test.each(FLAG_USES)("`%s` uses %s, which the command accepts", (example, flag) => { + const { command } = resolve(example.split(/\s+/).slice(1)); + expect(flagsOf(command)).toContain(flag); + }); + + test.each(migrateTree())("$path documents every flag it accepts", ({ command }) => { + const undocumented = flagsOf(command).filter( + (flag) => flag.startsWith("--") && flag !== "--help" && !README.includes(flag), + ); + expect(undocumented).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts new file mode 100644 index 000000000..1c7728c1d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -0,0 +1,602 @@ +/** + * The human-mode half of `migrate import`: the wizard fills in missing flags, the + * readiness report renders, and declining the confirmation writes nothing. + * + * Kept in its own file because `mock.module` registrations are process-lifetime, + * and `bun test --parallel` puts several files in each worker — so a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. Human mode itself needs no mock: `setMode` is the + * supported override. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { getMode, setMode, type Mode } from "../../mode.ts"; +import { + keylessTargetStubs, + listageStubs, + useCaptureLog, + useMigrateLogDir, +} from "../../test/lib/stubs.ts"; +import type { InstanceTarget } from "../../lib/keyless-target.ts"; + +const mockSelect = mock(async () => "clerk" as unknown); +const mockText = mock(async () => "export.json" as unknown); +type MultiselectConfig = { options: { value: string; label: string; hint?: string }[] }; +const mockMultiselect = mock(async (_config: MultiselectConfig) => [] as unknown[]); +let confirmAnswer = true; +/** Every confirmation the run put up, in order — the wording is the assertion. */ +let confirmMessages: string[] = []; +let originalMode: Mode; + +const ACCOUNT_TARGET: InstanceTarget = { + kind: "account", + ctx: { + appId: "app_1", + appLabel: "Migration Test", + instanceId: "ins_1", + instanceLabel: "development", + }, + label: "Migration Test (development)", +}; +let instanceTarget: InstanceTarget | Error = ACCOUNT_TARGET; + +mock.module("../../lib/listage.ts", () => ({ + ...listageStubs, + select: (...args: unknown[]) => mockSelect(...(args as [])), +})); + +mock.module("../../lib/keyless-target.ts", () => ({ + ...keylessTargetStubs, + resolveInstanceTarget: async () => { + if (instanceTarget instanceof Error) throw instanceTarget; + return instanceTarget; + }, +})); + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../lib/prompts.ts", () => ({ + confirm: async ({ message }: { message: string }) => { + confirmMessages.push(message); + return confirmAnswer; + }, + multiselect: (...args: unknown[]) => mockMultiselect(...(args as [MultiselectConfig])), + text: (...args: unknown[]) => mockText(...(args as [])), + password: async () => "", + editor: async () => "{}", +})); + +const { run } = await import("./run.ts"); +const { deleteMigration } = await import("./delete.ts"); +const { UserAbortError } = await import("../../lib/errors.ts"); +const { loadSettings, saveSettings } = await import("./lib/settings.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); + +const captured = useCaptureLog(); +useMigrateLogDir(); + +let workDir: string; +let configDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { method: string; url: string; body: unknown }[]; + +const EXPORT = [ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", primary_email_address: "b@x.dev" }, +]; + +const baseOptions = { transformer: "clerk", file: "export.json", secretKey: "sk_test_x" }; + +let originalPlatformKey: string | undefined; + +beforeAll(() => { + originalMode = getMode(); + setMode("human"); + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + // Pinned rather than inherited: the settings-fix write goes through the + // Platform API, and CI has neither a `.env.local` nor a login session. + originalPlatformKey = process.env.CLERK_PLATFORM_API_KEY; + process.env.CLERK_PLATFORM_API_KEY = "ak_test"; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-config-")); + _setConfigDir(configDir); + process.chdir(workDir); +}); + +afterAll(() => { + setMode(originalMode); + globalThis.fetch = originalFetch; + if (originalPlatformKey === undefined) delete process.env.CLERK_PLATFORM_API_KEY; + else process.env.CLERK_PLATFORM_API_KEY = originalPlatformKey; + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + confirmAnswer = true; + confirmMessages = []; + instanceTarget = ACCOUNT_TARGET; + mockSelect.mockReset(); + mockText.mockReset(); + mockMultiselect.mockReset(); + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("export.json"); + mockMultiselect.mockResolvedValue([]); + fs.rmSync(path.join(workDir, "logs"), { recursive: true, force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); + stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); +}); + +afterEach(() => { + process.exitCode = 0; +}); + +type StubSettings = { attributes?: object; social?: object } | null; + +let currentSettings: StubSettings = null; +/** What the instance reports once a config PATCH lands, when a test sets one. */ +let settingsAfterFix: StubSettings = null; + +/** Stubs BAPI plus the FAPI environment lookup the readiness report needs. */ +function stubInstanceSettings(settings: StubSettings) { + currentSettings = settings; + settingsAfterFix = null; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + if (url.endsWith("/v1/domains")) { + if (!currentSettings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.includes("/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.includes("/v1/environment")) return Response.json({ user_settings: currentSettings }); + if (url.endsWith("/instances/ins_1/config")) { + if (settingsAfterFix) currentSettings = settingsAfterFix; + return Response.json({ config_version: "v1_patched" }); + } + return Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; +} + +const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); + +describe("the wizard fills in missing flags", () => { + test("bare `clerk migrate import` prompts for the transformer and file, then imports", async () => { + await run({ secretKey: "sk_test_x" }); + + expect(mockSelect).toHaveBeenCalledTimes(1); + expect(mockText).toHaveBeenCalledTimes(1); + expect(created()).toHaveLength(2); + }); + + test("asks only for what the flags did not supply", async () => { + await run({ ...baseOptions, transformer: "clerk" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(mockText).not.toHaveBeenCalled(); + }); + + test("prompts for the file when only the transformer was passed", async () => { + await run({ transformer: "clerk", secretKey: "sk_test_x" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(mockText).toHaveBeenCalledTimes(1); + }); + + test("records the wizard's answers for the next run", async () => { + await run({ secretKey: "sk_test_x" }); + + expect(await loadSettings()).toMatchObject({ transformer: "clerk", file: "export.json" }); + }); +}); + +describe("the readiness report", () => { + test("renders before the confirmation", async () => { + await run(baseOptions); + expect(captured.err).toContain("Migration readiness"); + expect(captured.err).toContain("2 users in this file"); + }); + + // The whole point of the report: seeing what will go wrong, then backing out + // before a single user exists in the destination instance. + test("declining afterwards writes nothing to Clerk", async () => { + confirmAnswer = false; + + await expect(run(baseOptions)).rejects.toThrow(UserAbortError); + + expect(captured.err).toContain("Migration readiness"); + expect(created()).toHaveLength(0); + }); + + test("accepting proceeds with the import", async () => { + confirmAnswer = true; + + await run(baseOptions); + + expect(created()).toHaveLength(2); + }); + + test("flags a field Clerk requires that not every user has", async () => { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", username: "bob" }, + ]), + ); + + await run(baseOptions); + + // The outcome block is the point: one user has no email, and an instance + // that requires one will refuse them. + expect(captured.err).toContain("1 user will not be imported"); + expect(captured.err).toContain("1 has no email, which this instance requires"); + expect(captured.err).toContain("1 setting needs attention"); + }); + + test("degrades to a note when the instance settings cannot be read", async () => { + stubInstanceSettings(null); + + await run(baseOptions); + + expect(captured.err).toContain("Could not read this instance's settings"); + expect(created()).toHaveLength(2); + }); + + test("is skipped for a -y run, which pays for no extra round-trips", async () => { + await run({ ...baseOptions, yes: true }); + + expect(requests.some((r) => r.url.endsWith("/v1/domains"))).toBe(false); + expect(captured.err).not.toContain("Migration readiness"); + expect(created()).toHaveLength(2); + }); +}); + +describe("fixing the instance's settings from the report", () => { + /** An export whose second user has no email, against a required-email instance. */ + function blockedOnRequiredEmail() { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", username: "bob" }, + ]), + ); + } + + /** Two flagged rows: email required with a user lacking it, username switched off. */ + function blockedOnTwoSettings() { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: false }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev", username: "alice" }, + { id: "u2", username: "bob" }, + ]), + ); + } + + const patched = () => requests.filter((r) => r.url.endsWith("/instances/ins_1/config")); + const offered = (round = 0) => mockMultiselect.mock.calls[round]?.[0]?.options ?? []; + + // Plain labels only: the config leaf each one writes is internal detail an + // operator cannot act on and does not need to read. + test("offers one change per blocking row, named in plain terms", async () => { + blockedOnRequiredEmail(); + + await run(baseOptions); + + expect(offered()).toEqual([ + { value: "email_address", label: "Make Email optional at sign-up" }, + ]); + }); + + test("does not ask when nothing is blocking", async () => { + await run(baseOptions); + + expect(mockMultiselect).not.toHaveBeenCalled(); + expect(created()).toHaveLength(2); + }); + + // Relaxing an instance's sign-up requirements is a real decision, so nothing + // is preselected and an empty answer must leave the instance untouched. + test("selecting nothing changes nothing and continues to the import", async () => { + blockedOnRequiredEmail(); + mockMultiselect.mockResolvedValue([]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(created()).toHaveLength(2); + }); + + test("selecting a change patches the instance and re-renders the report", async () => { + blockedOnRequiredEmail(); + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(1); + expect(patched()[0]).toMatchObject({ + method: "PATCH", + body: { auth_email: { required_for_sign_up: false } }, + }); + expect(captured.err).toContain("Updated 1 setting"); + // The redraw clears the row that was just fixed, so the confirmation that + // follows is against the settings the write established. + expect(captured.err).toContain("Every field in this file is configured in Clerk"); + expect(created()).toHaveLength(2); + }); + + /** + * The redraw must not re-read the Frontend API. It is eventually consistent, + * so a fetch this soon after the write returns the pre-write settings and + * redraws the report with every row the operator just cleared still flagged. + */ + test("redraws from the write rather than re-reading stale settings", async () => { + blockedOnRequiredEmail(); + // Anything read back now would still say "required" — as it did in practice. + settingsAfterFix = { + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }; + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(requests.filter((r) => r.url.includes("/v1/environment"))).toHaveLength(1); + expect(captured.err).toContain("Every field in this file is configured in Clerk"); + // Flagged in the first report, and only there — the redraw is clean even + // though a re-read at this moment would still have reported it. + expect(captured.err.split("setting needs attention")).toHaveLength(2); + }); + + // A keyless application is only reachable through the Backend API, which has + // no route for any of these settings — saying so beats a confusing rejection. + test("stands down for a keyless application and still imports", async () => { + blockedOnRequiredEmail(); + instanceTarget = { + kind: "keyless", + keyless: { secretKey: "sk_test_x", source: ".env" }, + label: "keyless", + }; + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(captured.err).toContain("clerk auth login"); + expect(created()).toHaveLength(2); + }); + + /** + * Each redraw is another decision point, not a receipt. Applying one change + * routinely leaves others still worth making, and an operator should not have + * to re-run the whole command to reach them. + */ + describe("offering again while anything is still flagged", () => { + test("re-offers what is left, without the change already applied", async () => { + blockedOnTwoSettings(); + mockMultiselect.mockResolvedValueOnce(["email_address"]); + mockMultiselect.mockResolvedValueOnce(["username"]); + + await run(baseOptions); + + expect(offered(0).map((option) => option.value)).toEqual(["email_address", "username"]); + expect(offered(1).map((option) => option.value)).toEqual(["username"]); + expect(patched()).toHaveLength(2); + expect(patched()[1]).toMatchObject({ + body: { auth_username: { used_for_sign_up: true } }, + }); + }); + + test("stops once nothing is flagged, rather than asking again", async () => { + blockedOnTwoSettings(); + mockMultiselect.mockResolvedValueOnce(["email_address"]); + mockMultiselect.mockResolvedValueOnce(["username"]); + + await run(baseOptions); + + expect(mockMultiselect).toHaveBeenCalledTimes(2); + expect(captured.err).toContain("Every field in this file is configured in Clerk"); + expect(created()).toHaveLength(2); + }); + + test("stops when the operator skips, leaving the rest flagged", async () => { + blockedOnTwoSettings(); + mockMultiselect.mockResolvedValueOnce(["email_address"]); + mockMultiselect.mockResolvedValueOnce([]); + + await run(baseOptions); + + expect(mockMultiselect).toHaveBeenCalledTimes(2); + expect(patched()).toHaveLength(1); + expect(created()).toHaveLength(2); + }); + + // A selection naming nothing on offer is the same as no selection, and must + // not become an empty PATCH. + test("sends nothing when the selection matches no offered change", async () => { + blockedOnRequiredEmail(); + mockMultiselect.mockResolvedValue(["not_a_real_change"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(created()).toHaveLength(2); + }); + }); + + test("warns instead of failing the run when the instance cannot be resolved", async () => { + blockedOnRequiredEmail(); + instanceTarget = new Error("not linked"); + mockMultiselect.mockResolvedValue(["email_address"]); + + await run(baseOptions); + + expect(patched()).toHaveLength(0); + expect(captured.err).toContain("nothing was changed"); + expect(created()).toHaveLength(2); + }); +}); + +describe("guards that still apply interactively", () => { + /** Makes `GET /v1/users/count` report an instance with one seat left. */ + function stubNearlyFullInstance(): void { + const inner = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (input.toString().includes("/v1/users/count")) { + return Response.json({ object: "total_count", total_count: 99 }); + } + return inner(input, init); + }) as typeof fetch; + } + + test("the dev-instance user limit, which the operator can agree to import past", async () => { + stubNearlyFullInstance(); + confirmAnswer = true; + + await run(baseOptions); + + expect(captured.err).toContain("100-user limit"); + expect(created()).toHaveLength(2); + }); + + // Asking "Import 2 users?" after warning that one of them cannot fit is the + // report and the quota disagreeing in the same run. + test("the final prompt restates the quota split rather than the file size", async () => { + stubNearlyFullInstance(); + + await run(baseOptions); + + expect(confirmMessages).toContain("Import 1 user and expect 1 to fail?"); + }); + + test("the final prompt names the whole file when the quota is not in play", async () => { + await run(baseOptions); + + expect(confirmMessages).toContain("Import 2 users?"); + }); + + test("declining the user-limit prompt writes nothing to Clerk", async () => { + stubNearlyFullInstance(); + confirmAnswer = false; + + await expect(run(baseOptions)).rejects.toThrow(UserAbortError); + + // Aborted before the readiness report, so nothing was read from FAPI either. + expect(captured.err).not.toContain("Migration readiness"); + expect(created()).toHaveLength(0); + }); + + test("an unrecognized password hasher aborts before any request", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + await expect(run(baseOptions)).rejects.toThrow(/Invalid password hasher/); + expect(created()).toHaveLength(0); + }); +}); + +describe("migrate delete confirmation", () => { + /** Answers the external-id lookup, then the deletes. */ + function stubDeleteTargets(present: Record) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ method: init?.method ?? "GET", url, body: null }); + + if (url.includes("/v1/users?")) { + const asked = new URL(url).searchParams.getAll("external_id"); + return Response.json( + asked + .filter((externalId) => externalId in present) + .map((externalId) => ({ id: present[externalId], external_id: externalId })), + ); + } + return Response.json({ deleted: true }); + }) as unknown as typeof fetch; + } + + const deleted = () => requests.filter((r) => r.method === "DELETE"); + + beforeEach(async () => { + await saveSettings({ transformer: "clerk", file: "export.json" }); + stubDeleteTargets({ legacy_a: "user_1", legacy_b: "user_2" }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "legacy_a", primary_email_address: "a@x.dev" }, + { id: "legacy_b", primary_email_address: "b@x.dev" }, + ]), + ); + }); + + test("reports the count and confirms before deleting", async () => { + confirmAnswer = true; + + await deleteMigration({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("About to delete 2 users"); + expect(deleted()).toHaveLength(2); + }); + + // The undo for a bad undo does not exist, so declining must cost nothing. + test("declining deletes nobody", async () => { + confirmAnswer = false; + + await expect(deleteMigration({ secretKey: "sk_test_x" })).rejects.toThrow(UserAbortError); + + expect(deleted()).toHaveLength(0); + }); + + test("-y skips the prompt", async () => { + confirmAnswer = false; + + await deleteMigration({ yes: true, secretKey: "sk_test_x" }); + + expect(deleted()).toHaveLength(2); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts new file mode 100644 index 000000000..f7e5364c2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -0,0 +1,750 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../lib/config.ts"; +import { CliError } from "../../lib/errors.ts"; +import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +// Every test below names its own `--secret-key`, which short-circuits the +// signed-in check — except the one that asserts what happens without it. +mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); +import { getLogDir } from "./lib/logger.ts"; +import { __resetCustomTransformersForTesting } from "./transformers/registry.ts"; +import { loadSettings } from "./lib/settings.ts"; +import { applyResumeAfter, explainErrors, run, validateRunOptions } from "./run.ts"; +import type { User } from "./types.ts"; + +let workDir: string; +let configDir: string; +let originalCwd: string; + +const users = (...ids: string[]): User[] => ids.map((userId) => ({ userId }) as User); + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-config-")); + _setConfigDir(configDir); + process.chdir(workDir); + fs.writeFileSync(path.join(workDir, "users.json"), "[]"); + fs.writeFileSync(path.join(workDir, "users.txt"), ""); +}); + +afterAll(() => { + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +describe("validateRunOptions", () => { + test("accepts a transformer and an existing JSON file", () => { + expect(validateRunOptions({ transformer: "clerk", file: "users.json" })).toEqual({ + transformer: "clerk", + file: "users.json", + }); + }); + + test.each([ + ["no transformer", { file: "users.json" }, /--transformer/], + ["an unknown transformer", { transformer: "okta", file: "users.json" }, /Unknown transformer/], + ["no file", { transformer: "clerk" }, /--file/], + ["a missing file", { transformer: "clerk", file: "nope.json" }, /File not found/], + [ + "an unsupported extension", + { transformer: "clerk", file: "users.txt" }, + /Unsupported file type/, + ], + ])("rejects %s", (_label, options, message) => { + expect(() => validateRunOptions(options)).toThrow(message); + }); + + test("names the valid transformers when one is missing", () => { + expect(() => validateRunOptions({ file: "users.json" })).toThrow(/clerk/); + }); +}); + +describe("applyResumeAfter", () => { + test("returns everything when no ID is given", () => { + expect(applyResumeAfter(users("a", "b"), undefined)).toHaveLength(2); + }); + + test("skips up to and including the named user", () => { + expect(applyResumeAfter(users("a", "b", "c"), "b").map((u) => u.userId)).toEqual(["c"]); + }); + + test("returns nothing when the named user is last", () => { + expect(applyResumeAfter(users("a", "b"), "b")).toEqual([]); + }); + + test("throws rather than silently re-importing everyone", () => { + expect(() => applyResumeAfter(users("a"), "zz")).toThrow(CliError); + }); +}); + +describe("run", () => { + const captured = useCaptureLog(); + let originalFetch: typeof globalThis.fetch; + let requests: { method: string; url: string; body: unknown }[]; + + const export2 = [ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d1", + password_hasher: "bcrypt", + }, + { id: "u2", primary_email_address: "b@x.dev" }, + ]; + + beforeAll(() => { + originalFetch = globalThis.fetch; + }); + + beforeEach(() => { + requests = []; + delete process.env.CLERK_MIGRATE_RATE_LIMIT; + fs.rmSync(getLogDir(), { recursive: true, force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ + method: init?.method ?? "GET", + url: input.toString(), + body: init?.body ? JSON.parse(init.body as string) : null, + }); + return new Response(JSON.stringify({ id: "user_created" }), { status: 200 }); + }) as typeof fetch; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + process.exitCode = 0; + }); + + const baseOptions = { + transformer: "clerk", + file: "export.json", + yes: true, + secretKey: "sk_test_x", + }; + + test("refuses before the wizard when nobody is signed in", async () => { + const previous = process.env.CLERK_SECRET_KEY; + delete process.env.CLERK_SECRET_KEY; + try { + await expect(run({ transformer: "clerk", file: "export.json", yes: true })).rejects.toThrow( + /Not logged in/, + ); + expect(requests).toHaveLength(0); + } finally { + if (previous !== undefined) process.env.CLERK_SECRET_KEY = previous; + } + }); + + test("imports every user in the file end to end", async () => { + await run(baseOptions); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created).toHaveLength(2); + expect(created[0]?.method).toBe("POST"); + expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "u1", + "u2", + ]); + expect(captured.err).toContain("Imported:"); + }); + + test("writes a timestamped NDJSON log for the run", async () => { + await run(baseOptions); + + const logs = fs.readdirSync(getLogDir()); + expect(logs).toHaveLength(1); + expect(logs[0]).toMatch(/^import-\d{4}-\d{2}-\d{2}T[\d-]+\.log$/); + + const entries = fs + .readFileSync(path.join(getLogDir(), logs[0] as string), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(entries.filter((e) => e.status === "success")).toHaveLength(2); + }); + + test("records the run's transformer and file for the next run", async () => { + await run(baseOptions); + expect(await loadSettings()).toEqual({ transformer: "clerk", file: "export.json" }); + }); + + test("--require-password imports only the users that have one", async () => { + await run({ ...baseOptions, requirePassword: true }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u1"]); + expect(captured.err).toContain("skipping 1 user without a password"); + }); + + test("--resume-after skips everyone up to and including that ID", async () => { + await run({ ...baseOptions, resumeAfter: "u1" }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created.map((r) => (r.body as { external_id: string }).external_id)).toEqual(["u2"]); + }); + + test("logs validation failures and imports the rest", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify([...export2, { id: "u3" }])); + + await run(baseOptions); + + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + expect(captured.err).toContain("1 user failed validation"); + }); + + /** Makes `GET /v1/users/count` report an instance that already holds users. */ + function stubUserCount(total: number): void { + const inner = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (input.toString().includes("/v1/users/count")) { + return Response.json({ object: "total_count", total_count: total }); + } + return inner(input, init); + }) as typeof fetch; + } + + // `baseOptions` passes -y, which has nobody to answer the prompt this warning + // otherwise raises — see run-interactive.test.ts for the prompt itself. + test("warns under -y when an import may exceed the development-instance user limit", async () => { + stubUserCount(99); + + await run(baseOptions); + + expect(captured.err).toContain("100-user limit"); + expect(captured.err).toContain("already holds 99"); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + }); + + test("stays quiet when the instance has room for the whole file", async () => { + stubUserCount(10); + + await run(baseOptions); + + expect(captured.err).not.toContain("100-user limit"); + }); + + test("aborts before any API call when the hasher is unrecognized", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + await expect(run(baseOptions)).rejects.toThrow(/Invalid password hasher/); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + }); + + test("exits non-zero when some users failed", async () => { + globalThis.fetch = (async () => + new Response(JSON.stringify({ errors: [{ code: "e", message: "taken" }] }), { + status: 422, + })) as unknown as typeof fetch; + + await run(baseOptions); + expect(process.exitCode).toBe(1); + }); + + // Tests run non-TTY, so `isHuman()` is false and the wizard path is never + // reached — the same guard an agent hits. + describe("without --transformer or --file", () => { + test.each([ + [{}, /--transformer and --file /], + [{ transformer: "clerk" }, /--file /], + [{ file: "export.json" }, /--transformer /], + ])("names the missing flags rather than prompting (%p)", async (partial, expected) => { + await expect(run({ ...partial, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + expected, + ); + expect(requests).toHaveLength(0); + }); + + test("explains that it cannot prompt", async () => { + await expect(run({ yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /cannot prompt in agent mode/, + ); + }); + }); + + describe("readiness report", () => { + /** Stubs BAPI plus the FAPI environment lookup the report depends on. */ + function stubInstanceSettings( + settings: { attributes?: object; social?: object } | null, + onUsers?: () => Response, + ) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + if (url.endsWith("/v1/domains")) { + if (!settings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.includes("/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.includes("/v1/environment")) return Response.json({ user_settings: settings }); + return onUsers ? onUsers() : Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; + } + + const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); + + // `-y` means nobody is watching, so the two extra round-trips buy nothing. + test("is skipped for a -y run", async () => { + stubInstanceSettings({ attributes: { email_address: { enabled: true } } }); + + await run(baseOptions); + + expect(requests.some((r) => r.url.endsWith("/v1/domains"))).toBe(false); + expect(captured.err).not.toContain("Migration readiness"); + expect(created()).toHaveLength(2); + }); + + test("renders before any user is created, and flags a required-but-missing field", async () => { + stubInstanceSettings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }); + // One user has no email, so a required email address will cost them. + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", username: "bob" }, + ]), + ); + + await run({ ...baseOptions, yes: false }); + + expect(captured.err).toContain("Migration readiness"); + expect(captured.err).toContain("1 user will not be imported"); + + // The report was printed before the first POST /v1/users. + const reportIndex = requests.findIndex((r) => r.url.includes("/v1/environment")); + const firstCreate = requests.findIndex((r) => r.url.endsWith("/v1/users")); + expect(reportIndex).toBeGreaterThanOrEqual(0); + expect(reportIndex).toBeLessThan(firstCreate); + }); + + test("degrades to a note when the instance settings cannot be read", async () => { + stubInstanceSettings(null); + + await run({ ...baseOptions, yes: false }); + + expect(captured.err).toContain("Could not read this instance's settings"); + expect(created()).toHaveLength(2); + }); + + test("cross-references supabase providers against the instance", async () => { + stubInstanceSettings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["discord"]}', + }, + ]), + ); + + await run({ ...baseOptions, transformer: "supabase", yes: false }); + + expect(captured.err).toContain("Social connections"); + expect(captured.err).toContain("Discord"); + expect(captured.err).toContain("not enabled in Clerk"); + }); + + // Supabase lists `email` and `phone` in `providers` alongside real social + // connections, and Clerk has no `oauth_email` to enable — so counting them + // as social flagged every password user as a blocking problem. + test("leaves supabase's email and phone pseudo-providers out of the social section", async () => { + stubInstanceSettings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email","discord"]}', + }, + ]), + ); + + await run({ ...baseOptions, transformer: "supabase", yes: false }); + + const social = captured.err.slice(captured.err.indexOf("Social connections")); + expect(social).toContain("Discord"); + expect(social).not.toContain("Email"); + expect(social).not.toContain("Phone"); + }); + }); + + describe("--transformer-file", () => { + const CUSTOM = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email", given: "firstName", pw: "password" }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { if (!user.firstName) delete user.firstName; }, + };`; + + let customFile: string; + let customCounter = 0; + + beforeEach(() => { + // A fresh filename each time: dynamic import() caches by URL, so reusing + // one would silently return a previous test's module. + customFile = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, customFile), CUSTOM); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { account_ref: "mp_1", contact_email: "a@x.dev", given: "Ada", pw: "$2b$10$hash" }, + { account_ref: "mp_2", contact_email: "b@x.dev", given: "", pw: "$2b$10$hash" }, + ]), + ); + }); + + afterEach(() => { + __resetCustomTransformersForTesting(); + }); + + const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); + + test("imports through a user-authored transformer", async () => { + await run({ + file: "export.json", + transformerFile: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_1", + "mp_2", + ]); + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("transformer from"); + }); + + test("applies the custom transformer's defaults and postTransform", async () => { + await run({ + file: "export.json", + transformerFile: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ first_name: "Ada", password_hasher: "bcrypt" }); + // postTransform dropped the empty given name rather than sending "". + expect("first_name" in (bodies[1] ?? {})).toBe(false); + }); + + // No sensible precedence between "the one you wrote" and "the one we ship". + test("conflicts with --transformer rather than picking one", async () => { + await expect( + run({ + transformer: "clerk", + file: "export.json", + transformerFile: customFile, + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/both name a transformer. Pass one or the other/); + expect(created()).toHaveLength(0); + }); + + test("fails before any request when the file is not there", async () => { + await expect( + run({ + file: "export.json", + transformerFile: "./nope.ts", + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/No transformer file at/); + expect(requests).toHaveLength(0); + }); + + test("fails before any request when the file is malformed", async () => { + const bad = `./bad-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, bad), + `export default { key: "x", label: "X", transformer: {} };`, + ); + + await expect( + run({ file: "export.json", transformerFile: bad, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/no source field maps to `userId`/); + expect(requests).toHaveLength(0); + }); + + test("still requires --file", async () => { + await expect( + run({ transformerFile: customFile, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/--file/); + }); + }); + + describe("per-platform imports", () => { + /** One realistic record per platform, in that platform's export shape. */ + const PLATFORMS: [string, unknown, string][] = [ + [ + "auth0", + [ + { + user_id: "auth0|1", + email: "a@x.dev", + email_verified: true, + given_name: "Ada", + family_name: "L", + }, + ], + "auth0|1", + ], + ["authjs", [{ id: "aj1", email: "a@x.dev", email_verified: "2024-01-01T00:00:00Z" }], "aj1"], + [ + "betterauth", + [{ user_id: "ba1", email: "a@x.dev", email_verified: true, password_hash: "$2a$10$h" }], + "ba1", + ], + [ + "supabase", + [ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-06-29 20:25:06+00", + encrypted_password: "$2b$10$h", + }, + ], + "sb1", + ], + ]; + + test.each(PLATFORMS)( + "%s transforms, validates and imports its export", + async (key, records, externalId) => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(records)); + + await run({ ...baseOptions, transformer: key }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created).toHaveLength(1); + expect((created[0]?.body as { external_id: string } | undefined)?.external_id).toBe( + externalId, + ); + }, + ); + + test("firebase imports its wrapped export and builds the scrypt digest", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify({ + users: [ + { + localId: "fb1", + email: "a@x.dev", + emailVerified: true, + passwordHash: "SGFzaA==", + salt: "U2FsdA==", + }, + ], + }), + ); + + await run({ + ...baseOptions, + transformer: "firebase", + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }); + + const body = requests.find((r) => r.url.endsWith("/v1/users"))?.body as Record< + string, + unknown + >; + expect(body).toMatchObject({ + external_id: "fb1", + password_digest: "SGFzaA==$U2FsdA==$SIGNER$Bw==$8$14", + password_hasher: "scrypt_firebase", + }); + }); + + test("a partial firebase flag set fails before anything is read", async () => { + await expect( + run({ ...baseOptions, transformer: "firebase", firebaseSignerKey: "SIGNER" }), + ).rejects.toThrow(/--firebase-salt-separator/); + expect(requests).toHaveLength(0); + }); + + test("an unknown transformer fails listing the valid keys", async () => { + await expect(run({ ...baseOptions, transformer: "okta" })).rejects.toThrow( + /Unknown transformer "okta".*clerk.*supabase/s, + ); + }); + }); + + describe("--skip-unsupported-providers", () => { + const supabaseExport = [ + { + id: "sb_email", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email"]}', + }, + { + id: "sb_discord", + email: "b@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["discord"]}', + }, + { + id: "sb_both", + email: "c@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email","discord"]}', + }, + ]; + + /** Stubs BAPI plus the FAPI environment lookup the check depends on. */ + function stubInstance(enabledSocial: Record | null) { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + if (url.endsWith("/v1/domains")) { + if (!enabledSocial) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.includes("/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.includes("/v1/environment")) { + return Response.json({ user_settings: { social: enabledSocial } }); + } + return Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; + } + + const created = () => + requests + .filter((r) => r.url.endsWith("/v1/users")) + .map((r) => (r.body as { external_id: string }).external_id); + + beforeEach(() => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(supabaseExport)); + }); + + test("skips only the user whose sole provider is disabled", async () => { + stubInstance({ oauth_google: { enabled: true }, oauth_discord: { enabled: false } }); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(created()).toEqual(["sb_email", "sb_both"]); + expect(captured.err).toContain("skipping 1 user "); + expect(captured.err).toContain("discord: 1"); + }); + + test("imports everyone when the provider is enabled", async () => { + stubInstance({ oauth_discord: { enabled: true } }); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(created()).toHaveLength(3); + }); + + // A failed lookup must not be read as "nothing is enabled" — that would + // silently drop every social user. + test("imports everyone when the instance config cannot be read", async () => { + stubInstance(null); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect(created()).toHaveLength(3); + expect(captured.err).toContain("Could not read the instance's enabled providers"); + }); + + test("is a no-op with a warning on a non-supabase transformer", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); + + await run({ ...baseOptions, skipUnsupportedProviders: true }); + + expect(created()).toHaveLength(2); + expect(captured.err).toContain("only applies to supabase"); + }); + + test("records the flag for the next run", async () => { + stubInstance({ oauth_discord: { enabled: true } }); + + await run({ ...baseOptions, transformer: "supabase", skipUnsupportedProviders: true }); + + expect((await loadSettings()).skipUnsupportedProviders).toBe(true); + }); + }); +}); + +describe("explainErrors", () => { + const COUNTRY = + "Phone numbers from this country (France) are currently not supported. For more information, please contact support."; + const QUOTA = + "You have reached your limit of 100 users. If you need more users, please use a Production instance."; + + test("names the development instance as the reason countries are blocked", () => { + const [note] = explainErrors([COUNTRY], "dev"); + expect(note).toContain("Development instances block SMS to most countries"); + expect(note).toContain("test-emails-and-phones"); + }); + + test("sends a production operator to the Dashboard instead of support", () => { + const [note] = explainErrors([COUNTRY], "prod"); + expect(note).toContain("customization/sms/settings"); + expect(note).not.toContain("Development instances"); + }); + + test("explains the user quota only where one applies", () => { + expect(explainErrors([QUOTA], "dev").join(" ")).toContain("development-instance quota"); + // Production has no such quota, and the API's message already names the + // plan upgrade in the one case it does. + expect(explainErrors([QUOTA], "prod")).toEqual([]); + }); + + test("says nothing about errors it does not recognize", () => { + expect(explainErrors(["Something else went wrong."], "dev")).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts new file mode 100644 index 000000000..905add1a6 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -0,0 +1,770 @@ +/** + * `clerk migrate import` — the user import itself. + * + * Ported from the standalone migration-tool's `src/migrate/cli.ts` + * (`runNonInteractive`), with auth moved onto the CLI's standard secret-key + * resolution chain and every failure raised as a `CliError` instead of + * `console.error` + `process.exit`. + * + * Registered as the `import` subcommand. The exported handler keeps the name + * `run` because `import` is a reserved word. Whatever the flags did not supply + * is filled in by `wizard.ts` for a human, or raised as a usage error naming + * the missing flags for an agent, which cannot answer a prompt. + */ + +import { describeBapiTarget, resolveBapiSecretKey } from "../../lib/bapi-command.ts"; +import { bold, dim, green, red, yellow } from "../../lib/color.ts"; +import { resolveProfile } from "../../lib/config.ts"; +import { hasAccountCredentials } from "../../lib/credential-store.ts"; +import { + AUTH_ERROR_REASON, + AuthError, + CliError, + ERROR_CODE, + throwUsageError, + throwUserAbort, +} from "../../lib/errors.ts"; +import { + resolveInstanceTarget, + resolveKeylessTarget, + type InstanceTarget, +} from "../../lib/keyless-target.ts"; +import { log } from "../../lib/log.ts"; +import { NEXT_STEPS } from "../../lib/next-steps.ts"; +import { confirm, multiselect } from "../../lib/prompts.ts"; +import { withGutter, withSpinner } from "../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../mode.ts"; +import { writeInstanceConfig } from "../config/io.ts"; +import { importUsers } from "./import-users.ts"; +import { analyzeFields } from "./lib/analysis.ts"; +import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebase-hash.ts"; +import { + enabledSocialProviders, + fetchInstanceSettings, + fetchUserCount, + toClerkStrategy, +} from "./lib/clerk-config.ts"; +import { + buildReadinessReport, + DASHBOARD_URL, + formatReadinessReport, + type ReadinessReport, +} from "./lib/readiness.ts"; +import { + applyChanges, + buildChangePayload, + buildSettingChanges, + type SettingChange, +} from "./lib/modify-settings.ts"; +import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; +import { startLogging, getLogFilePath } from "./lib/logger.ts"; +import { saveSettings } from "./lib/settings.ts"; +import { + countSocialProviders, + findDisabledProviders, + findUsersWithOnlyDisabledProviders, + readSupabaseRows, +} from "./lib/supabase-providers.ts"; +import { fileExists, getFileType, loadUsersFromFile } from "./lib/transform.ts"; +import { loadCustomTransformer } from "./transformers/load-custom.ts"; +import { registerCustomTransformer, transformerKeys } from "./transformers/registry.ts"; +import type { ImportSummary, User } from "./types.ts"; +import { runWizard, throwAgentFlagsRequired } from "./wizard.ts"; +import { login } from "../auth/login.ts"; +import { link } from "../link/index.ts"; + +export type MigrateRunOptions = { + transformer?: string; + file?: string; + resumeAfter?: string; + requirePassword?: boolean; + yes?: boolean; + secretKey?: string; + app?: string; + instance?: string; + /** Path to a user-authored transformer, for a platform with no built-in. */ + transformerFile?: string; + /** Supabase: drop users whose only social provider is disabled in Clerk. */ + skipUnsupportedProviders?: boolean; +} & FirebaseHashFlags; + +/** + * Validates the flags a run needs before anything is read or sent. + * + * @returns The transformer key and file path, both guaranteed present. + */ +export function validateRunOptions(options: MigrateRunOptions): { + transformer: string; + file: string; +} { + const valid = transformerKeys(); + + // A custom transformer has already been loaded and registered by the time + // this runs, so its key is resolvable even though it is not in `valid`. + if (options.transformerFile) { + if (!options.file) { + throwUsageError( + "Missing required option --file (path to a JSON or CSV export).", + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: + "clerk migrate import -y --transformer-file ./my-transformer.ts --file users.json", + description: "Import with a custom transformer", + }, + ], + ); + } + if (!fileExists(options.file)) { + throw new CliError(`File not found: ${options.file}`, { code: ERROR_CODE.FILE_NOT_FOUND }); + } + if (!getFileType(options.file)) { + throwUsageError(`Unsupported file type for ${options.file}. Provide a .json or .csv file.`); + } + return { transformer: options.transformer as string, file: options.file }; + } + + if (!options.transformer) { + throwUsageError( + `Missing required option --transformer. Valid values: ${valid.join(", ")}.`, + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: "clerk migrate import -y --transformer clerk --file users.json", + description: "Import a Clerk export", + }, + ], + ); + } + if (!valid.includes(options.transformer)) { + throwUsageError( + `Unknown transformer "${options.transformer}". Valid values: ${valid.join(", ")}.`, + ); + } + if (!options.file) { + throwUsageError( + "Missing required option --file (path to a JSON or CSV export).", + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: "clerk migrate import -y --transformer clerk --file users.json", + description: "Import a Clerk export", + }, + ], + ); + } + if (!fileExists(options.file)) { + throw new CliError(`File not found: ${options.file}`, { code: ERROR_CODE.FILE_NOT_FOUND }); + } + if (!getFileType(options.file)) { + throwUsageError(`Unsupported file type for ${options.file}. Provide a .json or .csv file.`); + } + + return { transformer: options.transformer, file: options.file }; +} + +/** + * Drops every user up to and including `resumeAfter`. + * + * @throws CliError when the ID is not in the file — silently importing the + * whole set would duplicate everything the previous run already created. + */ +export function applyResumeAfter(users: User[], resumeAfter: string | undefined): User[] { + if (!resumeAfter) return users; + + const index = users.findIndex((user) => user.userId === resumeAfter); + if (index === -1) { + throw new CliError(`Could not find user ID "${resumeAfter}" in the import file.`, { + code: ERROR_CODE.USAGE_ERROR, + }); + } + return users.slice(index + 1); +} + +/** Where a production instance's operator changes the SMS country blocklist. */ +const SMS_SETTINGS_URL = "https://dashboard.clerk.com/~/customization/sms/settings"; + +/** Clerk's fictional email addresses and phone numbers, for development. */ +const TEST_NUMBERS_URL = "https://clerk.com/docs/guides/development/testing/test-emails-and-phones"; + +/** + * What the API's error messages leave out: whether the operator can do + * something about them, and where. + * + * Both of these read as account-level restrictions and are not. Blocked + * countries are a per-instance SMS blocklist that development instances are + * created with far more of, and the user limit is a development-instance quota + * that production does not have at all — so "contact support", which both + * messages point at, is the wrong first move for most readers. + * + * @returns One note per recognized error family, empty when none apply. + */ +export function explainErrors(errors: Iterable, instanceType: InstanceType): string[] { + const all = [...errors]; + const notes: string[] = []; + + if (all.some((error) => error.includes("Phone numbers from this country"))) { + notes.push( + instanceType === "dev" + ? `Development instances block SMS to most countries by default — this is not a limit on your account. ` + + `Use Clerk's test phone numbers while developing (${TEST_NUMBERS_URL}), and contact support only if ` + + `you need real numbers in a specific country before going to production.` + : `Unblock the countries you need under SMS settings in the Dashboard (${SMS_SETTINGS_URL}). ` + + `Plans without SMS support cannot remove them; contact support if the setting is refused.`, + ); + } + + // Production has no user limit unless a plan imposes one, and the API's own + // message already names the fix ("upgrade to a paid plan") in that case. + if ( + instanceType === "dev" && + all.some((error) => /You have reached your limit of \d+ users/.test(error)) + ) { + notes.push( + `The user limit is a development-instance quota (${DEV_USER_LIMIT} by default). Import into a production ` + + `instance to bring everyone across, or contact support to raise this instance's limit.`, + ); + } + + return notes; +} + +function formatSummary( + summary: ImportSummary, + logFile: string, + instanceType: InstanceType, +): string { + const inFile = summary.totalProcessed + summary.validationFailed; + const lines = [ + `${bold("Total users in file:")} ${inFile}`, + `${green("Imported:")} ${summary.successful}`, + `${red("Failed:")} ${summary.failed}`, + ]; + + if (summary.validationFailed > 0) { + lines.push(`${yellow("Failed validation:")} ${summary.validationFailed}`); + } + if (summary.errorBreakdown.size > 0) { + lines.push("", bold("Error breakdown:")); + for (const [error, count] of summary.errorBreakdown) { + lines.push(` ${count} user${count === 1 ? "" : "s"}: ${error}`); + } + for (const note of explainErrors(summary.errorBreakdown.keys(), instanceType)) { + lines.push("", note); + } + } + lines.push("", dim(`Log: ${logFile}`)); + + return lines.join("\n"); +} + +/** + * Stops an import that looks likely to exhaust a development instance's user + * quota, and asks before letting it through anyway. + * + * A prompt rather than a hard refusal, because the number it checks against + * cannot be trusted to be this instance's: {@link DEV_USER_LIMIT} is only what + * an instance is *created* with, Clerk raises it per instance on request, and + * no public endpoint serves the real value. The existing user count is live; + * the limit it is measured against is not. Refusing outright would block + * imports the destination would happily accept, so the operator — who can ask + * Clerk what their limit is — gets the last word. + * + * `-y` and agent mode proceed on the warning alone, matching the import + * confirmation below: neither has anyone to answer the question. + * + * @returns How many of `incoming` the quota is expected to reject, or `0` when + * the whole file fits. The final import prompt reports the same split, so + * that "yes" is never a bigger number than the instance will accept. + * @throws UserAbortError when the operator declines. + */ +async function confirmDevUserLimit( + incoming: number, + secretKey: string, + yes: boolean, +): Promise { + const existing = await withSpinner("Checking the instance's user count...", async () => + fetchUserCount(secretKey), + ); + const headroom = Math.max(0, DEV_USER_LIMIT - (existing ?? 0)); + if (incoming <= headroom) return 0; + + const rejected = incoming - headroom; + const held = existing === null ? "" : `, and this one already holds ${existing}`; + log.warn( + `Development instances default to a ${DEV_USER_LIMIT}-user limit${held}. About ${rejected} of the ` + + `${incoming} user${incoming === 1 ? "" : "s"} in this file will be rejected with a quota error unless ` + + `Clerk has raised this instance's limit — the limit itself is not readable from the API.\n` + + `Import into a production instance to bring everyone across, or contact support to raise the limit.`, + ); + + if (yes || !isHuman() || isAgent()) return rejected; + + const proceed = await confirm({ + message: `Continue anyway, expecting about ${rejected} user${rejected === 1 ? "" : "s"} to be rejected?`, + default: false, + }); + if (!proceed) throwUserAbort(); + + return rejected; +} + +/** + * Drops users whose only way into Clerk is a social provider the destination + * instance has not enabled. + * + * Only meaningful for Supabase exports — it is the one platform whose export + * records per-user providers. If the instance's configuration cannot be read, + * nobody is dropped: a failed lookup must not be mistaken for "no providers + * are enabled". + */ +async function skipDisabledProviderUsers( + users: User[], + file: string, + transformer: string, + secretKey: string, +): Promise { + if (transformer !== "supabase") { + log.warn(`--skip-unsupported-providers only applies to supabase exports; ignoring.`); + return users; + } + + const settings = await withSpinner("Checking enabled providers...", async () => + fetchInstanceSettings(secretKey), + ); + const enabled = settings ? enabledSocialProviders(settings) : null; + if (!enabled) { + log.warn( + "Could not read the instance's enabled providers; importing every user. Re-run with --verbose for details.", + ); + return users; + } + + const rows = await readSupabaseRows(file); + const disabled = findDisabledProviders(rows, enabled, toClerkStrategy); + if (disabled.length === 0) { + log.info("Every provider in this export is enabled in Clerk; no users skipped."); + return users; + } + + const { excludedIds, byProvider } = findUsersWithOnlyDisabledProviders(rows, disabled); + if (excludedIds.size === 0) { + log.info( + `${disabled.join(", ")} not enabled in Clerk, but every user has another way to sign in; none skipped.`, + ); + return users; + } + + const breakdown = Object.entries(byProvider) + .map(([provider, count]) => `${provider}: ${count}`) + .join(", "); + log.warn( + `--skip-unsupported-providers: skipping ${excludedIds.size} user${excludedIds.size === 1 ? "" : "s"} whose only provider is not enabled in Clerk (${breakdown}).`, + ); + + return users.filter((user) => !excludedIds.has(user.userId)); +} + +type ReportInput = { + users: User[]; + file: string; + transformer: string; + secretKey: string; + validationFailed: number; +}; + +/** + * Everything the report needs except the instance's settings — the half that + * comes from the file, and so does not change when the instance does. + */ +async function readFileSide(input: ReportInput) { + // Only Supabase exports record per-user providers, so only they can be + // cross-referenced against the instance's social connections. + let providerCounts: Record | undefined; + if (input.transformer === "supabase") { + try { + providerCounts = countSocialProviders(await readSupabaseRows(input.file)); + } catch (error) { + log.debug(`migrate: could not read providers for the readiness report: ${String(error)}`); + } + } + + return { + analysis: analyzeFields(input.users), + validationFailed: input.validationFailed, + providerCounts, + }; +} + +function printReport(report: ReadinessReport): void { + log.blank(); + for (const line of formatReadinessReport(report)) log.info(line); + log.blank(); +} + +/** + * Offers to change the instance's settings, one selectable change per flagged + * row. + * + * Without this the report names something the operator has to leave the CLI to + * act on. Nothing is preselected and selecting nothing continues to the import + * prompt unchanged: a flagged setting is not a wrong setting, and relaxing an + * instance's sign-up requirements is a real decision rather than a default. + * + * @returns The changes that were written, so the caller can redraw the report. + */ +async function offerSettingChanges( + report: ReadinessReport, + options: MigrateRunOptions, +): Promise { + const changes = buildSettingChanges(report.blocking); + if (changes.length === 0) return []; + + // Navigation keys are in the prompt's own footer; what that footer cannot say + // is that selecting nothing is a valid answer rather than an unfinished one. + const chosen = await multiselect({ + message: "Update this instance's settings first? (enter to skip)", + options: changes.map((change) => ({ value: change.id, label: change.label })), + initialValues: [], + required: false, + }); + // Filtered before anything is resolved or sent: a selection that matches no + // offered change is the same as no selection, and must not become an empty + // PATCH. + const applied = changes.filter((change) => chosen.includes(change.id)); + if (applied.length === 0) return []; + + // Resolved here rather than up front: an operator who selects nothing should + // not pay for a Platform API round-trip, and a target that cannot be resolved + // (a bare `--secret-key` against an unlinked directory) should not fail the + // whole run before the report has even been offered. + let target: InstanceTarget; + try { + target = await resolveInstanceTarget({ app: options.app, instance: options.instance }); + } catch (error) { + log.warn( + "Could not resolve which instance to configure, so nothing was changed. " + + "Link a project with `clerk link`, or pass `--app `.", + ); + log.debug(`migrate: settings change target unresolved: ${String(error)}`); + return []; + } + + // The Backend API a keyless application is reachable through has no route for + // any of these settings — `config patch` rejects the same payload by name. + if (target.kind === "keyless") { + log.warn( + "These settings need an account to change. Run `clerk auth login` to claim this application, " + + `then re-run, or update them at ${DASHBOARD_URL}.`, + ); + return []; + } + + await withSpinner(`Updating settings on ${target.label}...`, async () => + writeInstanceConfig(target, buildChangePayload(applied), { + method: "PATCH", + failureContext: "Failed to update instance settings", + }), + ); + log.success(`Updated ${applied.length} setting${applied.length === 1 ? "" : "s"}.`); + + return applied; +} + +/** + * Prints the Migration Readiness report: what the file contains, cross- + * referenced against what the destination instance accepts. + * + * Rendered immediately before the confirmation prompt, so declining that + * prompt aborts with nothing written to Clerk. + * + * Skipped only for `-y`, which says "don't ask, don't lecture" and should not + * pay for two extra network round-trips. Agent mode still gets it: an agent + * driving a migration can act on "10 users will not be imported, because email + * is required" exactly as a human would — but not the prompt, which needs one. + */ +async function showReadinessReport( + input: ReportInput & { skipReport: boolean; options: MigrateRunOptions }, +): Promise { + if (input.skipReport) return; + + let settings = await withSpinner("Checking instance settings...", async () => + fetchInstanceSettings(input.secretKey), + ); + const fileSide = { ...(await readFileSide(input)), users: input.users }; + + let report = buildReadinessReport({ ...fileSide, settings }); + printReport(report); + + if (!isHuman() || isAgent()) return; + + // Every redraw is another decision point, not a receipt. Applying one change + // routinely leaves others still worth making — and can surface consequences + // that were masked behind the row just cleared — so the offer repeats for as + // long as the report has something to offer. + while (report.blocking.length > 0) { + const applied = await offerSettingChanges(report, input.options); + // Nothing selected, nothing offerable, or nowhere to write it: the operator + // has said their piece and the import prompt is next. + if (applied.length === 0) return; + + // Redrawn from the write, not from a re-read. Clerk's Frontend API is + // eventually consistent, so fetching settings again here routinely returns + // the pre-write ones and redraws every row the operator just cleared. + settings = applyChanges(settings, applied); + report = buildReadinessReport({ ...fileSide, settings }); + printReport(report); + } +} + +/** + * Fills in a missing `--transformer`/`--file` interactively, or explains what + * to pass. + * + * Agent mode is the CLI's existing non-interactive signal, so an agent that + * runs bare `clerk migrate import` gets a usage error naming the flags rather than a + * prompt it cannot answer. + */ +async function resolveMissingOptions(options: MigrateRunOptions): Promise { + const missing = { transformer: !options.transformer, file: !options.file }; + if (!missing.transformer && !missing.file) return options; + + if (isAgent() || !isHuman()) { + throwAgentFlagsRequired(missing); + } + + // Resolved before the prompt only when `--transformer firebase` was already + // passed; otherwise the wizard picks the platform first and looks them up + // itself, so a non-Firebase migration never reads them at all. + const firebaseHashConfig = await resolveFirebaseHashConfig(options, options.transformer); + const answers = await runWizard({ ...options, firebaseHashConfig }); + + return { + ...options, + transformer: answers.transformer, + file: answers.file, + ...(answers.firebaseHashConfig + ? { + firebaseSignerKey: answers.firebaseHashConfig.base64_signer_key, + firebaseSaltSeparator: answers.firebaseHashConfig.base64_salt_separator, + firebaseRounds: answers.firebaseHashConfig.rounds, + firebaseMemCost: answers.firebaseHashConfig.mem_cost, + } + : {}), + }; +} + +/** + * Loads and registers a `--transformer-file`, so the rest of the run treats it + * exactly like a built-in. + * + * @returns The options with `transformer` set to the loaded entry's key. + */ +async function applyCustomTransformer(options: MigrateRunOptions): Promise { + if (!options.transformerFile) return options; + + // Both name a transformer, and there is no sensible precedence between "the + // one you wrote" and "the one we ship" — say so rather than picking. + if (options.transformer) { + throwUsageError( + "--transformer and --transformer-file both name a transformer. Pass one or the other.", + undefined, + undefined, + [ + { + command: + "clerk migrate import -y --transformer-file ./my-transformer.ts --file users.json", + description: "Use a transformer you wrote", + }, + { + command: "clerk migrate import -y --transformer clerk --file users.json", + description: "Use a built-in transformer", + }, + ], + ); + } + + const custom = await loadCustomTransformer(options.transformerFile); + registerCustomTransformer(custom); + log.info(`Loaded the \`${custom.key}\` transformer from ${options.transformerFile}.`); + + return { ...options, transformer: custom.key }; +} + +/** + * Makes sure there is somewhere to import *into* before anything else happens. + * + * Without this the first complaint comes from deep inside the secret-key chain, + * which resolves the linked profile before it ever asks for a token — so a + * signed-out operator in an unlinked directory is told to `clerk link`, a + * command that will only turn around and ask them to sign in. Worse, both + * failures land after the wizard has already walked them through picking a + * platform and a file. + * + * A human gets the same sign-in-then-link flow `clerk link` already runs. An + * agent cannot answer a browser login or an application picker, so it gets the + * error naming whichever half is missing. + */ +async function ensureImportTarget(options: MigrateRunOptions): Promise { + // Each of these names the destination instance on its own, with no account + // and no linked directory involved — mirroring resolveBapiSecretKey. + if (options.secretKey || options.app || process.env.CLERK_SECRET_KEY) return; + // An unclaimed accountless application keeps its only secret key on disk. + if (await resolveKeylessTarget({ instance: options.instance })) return; + + const interactive = isHuman() && !isAgent(); + + if (!(await hasAccountCredentials())) { + if (!interactive) { + throw new AuthError({ + reason: AUTH_ERROR_REASON.NOT_LOGGED_IN, + message: + "Not logged in, so there is no Clerk instance to import into. Run `clerk auth login`, then `clerk link`.", + examples: [ + { command: "clerk auth login", description: "Sign in, then re-run the import" }, + { + command: + "clerk migrate import -y --secret-key sk_test_... --transformer clerk --file users.json", + description: "Import without signing in", + }, + ], + }); + } + log.info("Not logged in. Signing in first..."); + await login({ showNextSteps: false }); + } + + // Left to the secret-key chain when non-interactive: its `not_linked` error + // is the one every other command raises, and there is nothing to add to it. + if (interactive && !(await resolveProfile(process.cwd()))) { + log.info("This directory isn't linked to a Clerk application. Linking one first..."); + await link({ skipIfLinked: true }); + } +} + +export async function run(rawOptions: MigrateRunOptions): Promise { + await ensureImportTarget(rawOptions); + rawOptions = await applyCustomTransformer(rawOptions); + const options = await resolveMissingOptions(rawOptions); + + const { transformer, file } = validateRunOptions(options); + const firebaseHashConfig = await resolveFirebaseHashConfig(options, transformer); + + await withGutter("Migrating users to Clerk", async ({ setNextSteps }) => { + const target = await describeBapiTarget({ ...options, secretKey: options.secretKey }); + const secretKey = await resolveBapiSecretKey({ ...options, secretKey: options.secretKey }); + const limits = resolveLimits(secretKey); + const dateTime = await startLogging(); + const logFile = getLogFilePath("import", dateTime); + + const { users: loaded, validationFailed } = await withSpinner( + `Loading users from ${file}...`, + async () => + loadUsersFromFile(file, transformer, dateTime, { context: { firebaseHashConfig } }), + ); + + let users = applyResumeAfter(loaded, options.resumeAfter); + if (options.resumeAfter) { + log.info(`Resuming after ${options.resumeAfter} (${loaded.length - users.length} skipped).`); + } + + if (options.skipUnsupportedProviders) { + users = await skipDisabledProviderUsers(users, file, transformer, secretKey); + } + + if (options.requirePassword) { + const withPassword = users.filter((user) => Boolean(user.password)); + const dropped = users.length - withPassword.length; + if (dropped > 0) { + log.info( + `--require-password: skipping ${dropped} user${dropped === 1 ? "" : "s"} without a password.`, + ); + } + users = withPassword; + } + + if (validationFailed > 0) { + log.warn( + `${validationFailed} user${validationFailed === 1 ? "" : "s"} failed validation and will be skipped. See ${logFile}.`, + ); + } + + if (users.length === 0) { + log.warn("No users left to import."); + return; + } + + const quotaRejections = + limits.instanceType === "dev" + ? await confirmDevUserLimit(users.length, secretKey, Boolean(options.yes)) + : 0; + + // `target` already carries the instance's environment ("My App + // (development)"), so the detected type is only worth spelling out when + // there is no app context to name — an explicit `--secret-key`. + log.info( + `Importing ${users.length} user${users.length === 1 ? "" : "s"} via the ${transformer} transformer into ` + + `${target ?? `the resolved instance (${limits.instanceType})`}.`, + ); + + await showReadinessReport({ + users, + file, + transformer, + secretKey, + validationFailed, + skipReport: Boolean(options.yes), + options, + }); + + if (!options.yes && isHuman() && !isAgent()) { + // The readiness report counts the whole file, because settings decide + // what Clerk *accepts*. The quota decides how much of it gets in at all, + // so the last prompt — the one that starts writing — restates that split + // rather than asking about a number the instance will not take. + const importable = users.length - quotaRejections; + const proceed = await confirm({ + message: quotaRejections + ? `Import ${importable} user${importable === 1 ? "" : "s"} and expect ${quotaRejections} to fail?` + : `Import ${users.length} user${users.length === 1 ? "" : "s"}?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + // The Firebase hash parameters are deliberately not among these: the signer + // key is a secret, and remembering it would write it to disk in plaintext. + await saveSettings({ + transformer, + file, + ...(options.skipUnsupportedProviders ? { skipUnsupportedProviders: true } : {}), + }); + + const summary = await withSpinner(`Importing users: [0/${users.length}]...`, async (spinner) => + importUsers({ + users, + secretKey, + limits, + dateTime, + skipPasswordRequirement: !options.requirePassword, + validationFailed, + spinner, + }), + ); + + log.info(formatSummary(summary, logFile, limits.instanceType)); + + // Offered even when some users failed: a partial import is exactly when + // reading the log and knowing how to undo it matters most. When users did + // fail, the per-user record of *why* leads, since the breakdown above only + // counts each error and never names who hit it. + setNextSteps( + summary.failed > 0 ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(logFile) : NEXT_STEPS.MIGRATE_DONE, + ); + + if (summary.failed > 0) process.exitCode = 1; + }); +} diff --git a/packages/cli-core/src/commands/migrate/settings/clear.ts b/packages/cli-core/src/commands/migrate/settings/clear.ts new file mode 100644 index 000000000..a7d8a981d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/clear.ts @@ -0,0 +1,154 @@ +/** + * `clerk migrate settings clear [name]` — forget this project's migration + * settings, or just one of them. + * + * Clears both stores when given no name. The credentials half is the reason + * this command exists: after a migration finishes, a Firebase signer key + * sitting in the repo has no further use, and "delete the file yourself" is a + * step people skip. + * + * `migrate delete` reads the saved transformer and file to know what to undo, + * so clearing is confirmed unless `-y` — an operator who clears and then wants + * to undo has no record left to undo from. + */ + +import { throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { confirm } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { clearMigrateEnvValues, MIGRATE_ENV_FILE } from "../lib/env-file.ts"; +import { loadSettings, saveSettings } from "../lib/settings.ts"; +import type { MigrationEntry } from "../../../lib/config.ts"; +import { envNames, findSetting, SETTING_NAMES, SETTINGS } from "./registry.ts"; + +export type SettingsClearOptions = { + yes?: boolean; +}; + +/** + * Every variable the migration settings own, `log-dir`'s included. + * + * Keyed on declaring an `envVar` rather than on `store === "env"`: `log-dir` is + * remembered in the config but still answers to a variable, and a full clear + * that left that variable behind would not have cleared the setting. + */ +const ENV_VARS = SETTINGS.filter((s) => s.envVar).map((s) => s.envVar as string); + +/** + * Warns that clearing this is what `migrate delete` reads to find the users the + * last run created. + */ +function warnAboutUndo(saved: MigrationEntry): void { + if (!saved.file) return; + log.warn( + `\`clerk migrate delete\` uses the saved file (${saved.file}) to identify the users the last run created. ` + + "Clearing it leaves nothing to undo from.", + ); +} + +/** + * Clears one named setting, leaving the rest of the project's settings alone. + * + * Both stores are cleared, because a setting can sit in either and `log-dir` + * can sit in both. Clearing half of one is worse than clearing none: the + * command reports the setting gone while the next run still reads it. + * + * An `env` value goes under every spelling the setting answers to, not just the + * prefixed one — dropping `CLERK_FIREBASE_ROUNDS` while `ROUNDS` stayed in the + * same file would leave the old value winning. + */ +async function clearOne(name: string, options: SettingsClearOptions): Promise { + const setting = findSetting(name); + if (!setting) { + throwUsageError( + `Unknown setting "${name}". Valid names: ${SETTING_NAMES.join(", ")}.`, + undefined, + undefined, + [ + { + command: "clerk migrate settings", + description: "List the settings and their current values", + }, + ], + ); + } + + const saved = await loadSettings(); + + if (!options.yes && isHuman() && !isAgent()) { + // Only the file itself is what `migrate delete` cannot do without; the + // transformer it can be told again. + if (setting.configKey === "file") warnAboutUndo(saved); + const proceed = await confirm({ message: `Clear \`${name}\`?`, default: false }); + if (!proceed) throwUserAbort(); + } + + const cleared: string[] = []; + + if (setting.envVar && (await clearMigrateEnvValues(envNames(setting))).length > 0) { + cleared.push(MIGRATE_ENV_FILE); + } + + const key = setting.configKey as keyof MigrationEntry | undefined; + if (key && saved[key] !== undefined) { + const { [key]: _cleared, ...rest } = saved; + await saveSettings(rest); + cleared.push("this project's settings"); + } + + if (cleared.length === 0) { + log.info( + `\`${name}\` is not set here. A value coming from the app's own env files or the shell has ` + + "to be removed there — run `clerk migrate settings` to see which is supplying it.", + ); + return; + } + + log.success(`Cleared \`${name}\` from ${cleared.join(" and ")}.`); +} + +export async function clear(options: SettingsClearOptions = {}, name?: string): Promise { + if (name !== undefined) return clearOne(name, options); + + const saved = await loadSettings(); + const hadConfig = Object.keys(saved).length > 0; + + if (!options.yes) { + if (isAgent() || !isHuman()) { + throwUsageError( + "`clerk migrate settings clear` forgets this project's settings and every credential in " + + `${MIGRATE_ENV_FILE}, and cannot prompt here. Pass -y to confirm.`, + undefined, + undefined, + [ + { + command: "clerk migrate settings clear -y", + description: "Forget them all without prompting", + }, + ], + ); + } + + if (hadConfig) warnAboutUndo(saved); + const proceed = await confirm({ + message: "Clear this project's migration settings?", + default: false, + }); + if (!proceed) throwUserAbort(); + } + + if (hadConfig) await saveSettings({}); + const dropped = await clearMigrateEnvValues(ENV_VARS); + + if (!hadConfig && dropped.length === 0) { + log.info("No migration settings to clear for this project."); + return; + } + + if (hadConfig) log.success("Cleared the saved transformer and file."); + if (dropped.length > 0) { + log.success( + `Removed ${dropped.length} credential${dropped.length === 1 ? "" : "s"} from ${MIGRATE_ENV_FILE}.`, + ); + } +} diff --git a/packages/cli-core/src/commands/migrate/settings/index.ts b/packages/cli-core/src/commands/migrate/settings/index.ts new file mode 100644 index 000000000..833e0335c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/index.ts @@ -0,0 +1,120 @@ +import { createArgument, InvalidArgumentError } from "@commander-js/extra-typings"; +import type { Command } from "@commander-js/extra-typings"; +import { clear } from "./clear.ts"; +import { list } from "./list.ts"; +import { SETTING_NAMES, suggestSettingName } from "./registry.ts"; +import { set } from "./set.ts"; + +const settings = { clear, list, set }; + +/** + * The `` argument both `set` and `clear` take. + * + * `.choices()` is what drives tab-completion and the help output's choice list, + * but it is implemented as a `parseArg` that throws before the action runs — so + * the friendlier "Unknown setting" errors inside `set.ts` and `clear.ts` are + * unreachable from the CLI, and a one-character miss like `logs-dir` gets only + * the full list back. Wrapping that parser keeps the completion metadata and + * puts the near miss first, where a reader scanning eight names would not find + * it. + */ +function settingNameArgument` | `[${string}]`>( + spec: S, + description: string, +) { + const argument = createArgument(spec, description).choices(SETTING_NAMES); + const rejectUnlessAllowed = argument.parseArg; + + // Whether the value is allowed stays Commander's question — asking it here + // too would be a second copy of the rule, free to disagree with the first. + // This only adds to the answer when the answer is no. + argument.parseArg = (value: string, previous: T): T => { + try { + return rejectUnlessAllowed?.(value, previous) as T; + } catch (error) { + const suggestion = suggestSettingName(value); + if (!suggestion) throw error; + throw new InvalidArgumentError( + `Did you mean "${suggestion}"? Allowed choices are ${SETTING_NAMES.join(", ")}.`, + ); + } + }; + + return argument; +} + +/** + * Registers `settings list|set|clear` under the `migrate` group. + * + * Noun-verb like every other group in the tree, and listing is the default + * because it is the read-only one — a bare `clerk migrate settings` should show, + * never change. + */ +export function registerMigrateSettings( + migrateCommand: Command<[], Record>, +): void { + const settingsCommand = migrateCommand + .command("settings") + .description("Inspect and change this project's saved migration settings") + .setExamples([ + { + command: "clerk migrate settings", + description: "Show every setting and where it resolves from", + }, + { + command: "clerk migrate settings set firebase-signer-key abc123", + description: "Save a credential to the gitignored .env.clerk-migrate", + }, + { + command: "clerk migrate settings clear firebase-signer-key", + description: "Forget one setting", + }, + { command: "clerk migrate settings clear -y", description: "Forget this project's settings" }, + ]); + + settingsCommand + .command("list", { isDefault: true }) + .description("Show each setting, its value and which source supplied it") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate settings list", description: "Credentials shown redacted" }, + { command: "clerk migrate settings list --json", description: "Machine-readable listing" }, + ]) + .action(async (_opts, cmd) => + settings.list(cmd.optsWithGlobals() as Parameters[0]), + ); + + settingsCommand + .command("set") + .description("Set one setting for this project") + .addArgument(settingNameArgument("", "Setting to change")) + .addArgument(createArgument("", "New value")) + .setExamples([ + { + command: "clerk migrate settings set transformer firebase", + description: "Remember the source platform", + }, + { + command: "clerk migrate settings set firebase-signer-key abc123", + description: "Write a credential to .env.clerk-migrate", + }, + ]) + .action(async (name, value) => settings.set(name, value)); + + settingsCommand + .command("clear") + .description("Forget one saved setting, or every setting and saved credential") + .addArgument(settingNameArgument("[name]", "Setting to clear; omit to clear them all")) + .option("-y, --yes", "Skip the confirmation prompt") + .setExamples([ + { command: "clerk migrate settings clear", description: "Clear everything after confirming" }, + { + command: "clerk migrate settings clear file", + description: "Forget only the remembered export file", + }, + { command: "clerk migrate settings clear -y", description: "Clear without prompting" }, + ]) + .action(async (name, _opts, cmd) => + settings.clear(cmd.optsWithGlobals() as Parameters[0], name), + ); +} diff --git a/packages/cli-core/src/commands/migrate/settings/list.ts b/packages/cli-core/src/commands/migrate/settings/list.ts new file mode 100644 index 000000000..747a179f5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/list.ts @@ -0,0 +1,155 @@ +/** + * `clerk migrate settings` — what a run in this project would pick up, and + * where each value is coming from. + * + * The source column is the point. A migration reads from flags, the + * environment, two of the app's env files and the CLI's config; when a run uses + * a stale value, the question is never "what is it" but "which of those is + * winning". Credentials are redacted, so this is safe to paste into an issue. + * + * Laid out like the CLI's other listings — `migrate logs list` and `migrate + * transformers list`: a line or two of orientation, the table, then a count. + * It closes with next steps, the way `mcp list` and `whoami` do, because a + * listing is where someone lands before they know what to type. Those are for + * humans; the full command surface stays in `--help`. + */ + +import { cyan, dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { NEXT_STEPS, printNextSteps } from "../../../lib/next-steps.ts"; +import { findMigrateEnvValue } from "../lib/env-file.ts"; +import { loadSettings } from "../lib/settings.ts"; +import { displayValue, envNames, SETTINGS, type SettingDef } from "./registry.ts"; + +export type SettingsListOptions = { + json?: boolean; +}; + +interface ResolvedSetting { + setting: SettingDef; + value?: string; + source?: string; +} + +/** + * Names the variable as well as the file when an alias supplied the value. + * + * `.env.local` alone would be a half-answer for a setting that has four + * accepted spellings: the reader has to know *which* line in that file the run + * is reading before they can change it. An exported variable already carries + * its name in `source`. + */ +function describeSource(setting: SettingDef, located: { name: string; source: string }): string { + if (located.name === setting.envVar || located.source.startsWith(located.name)) { + return located.source; + } + return `${located.source} (${located.name})`; +} + +async function resolveAll(): Promise { + const saved = await loadSettings(); + + return Promise.all( + SETTINGS.map(async (setting): Promise => { + if (setting.store === "config") { + // `log-dir` is remembered in the config but yields to an environment + // value, so the environment has to be checked first here too — a + // listing that shows the remembered path while the run reads another + // is the one thing the source column exists to prevent. + if (setting.envVar) { + const located = await findMigrateEnvValue(envNames(setting)); + if (located) { + return { setting, value: located.value, source: describeSource(setting, located) }; + } + } + + const value = saved[setting.configKey as keyof typeof saved]; + return value === undefined + ? { setting } + : { setting, value: String(value), source: "clerk config" }; + } + + const located = await findMigrateEnvValue(envNames(setting)); + return located + ? { setting, value: located.value, source: describeSource(setting, located) } + : { setting }; + }), + ); +} + +function toJson(resolved: ResolvedSetting[]) { + return resolved.map(({ setting, value, source }) => ({ + name: setting.name, + store: setting.store, + description: setting.description, + // Redacted here too: `--json` is what gets piped into a log or a ticket. + value: value === undefined ? null : displayValue(setting, value), + set: value !== undefined, + secret: Boolean(setting.secret), + source: source ?? null, + })); +} + +/** + * Pads to a visible width, then colours. + * + * Colouring first and padding after would count the ANSI escape bytes towards + * the width and pull every later column left by however many they took. + */ +function column(text: string, width: number, paint: (value: string) => string): string { + return paint(text) + " ".repeat(Math.max(0, width - text.length)); +} + +export async function list(options: SettingsListOptions = {}): Promise { + const resolved = await resolveAll(); + + if (options.json) { + log.data(JSON.stringify(toJson(resolved), null, 2)); + return; + } + + // An unset value leaves the column empty rather than filling it with a + // placeholder: the source column already reads "not set" on the same row, and + // an empty cell is what makes the settings that do have a value stand out. + const cells = resolved.map(({ setting, value, source }) => ({ + setting, + name: setting.name, + value: value === undefined ? "" : displayValue(setting, value), + unset: value === undefined, + source: source ?? "not set", + })); + + const width = (header: string, pick: (cell: (typeof cells)[number]) => string) => + Math.max(header.length, ...cells.map((cell) => pick(cell).length)) + 2; + + const nameWidth = width("SETTING", (c) => c.name); + const valueWidth = width("VALUE", (c) => c.value); + const sourceWidth = width("SOURCE", (c) => c.source); + + log.info("A migration run in this directory picks these up unless a flag overrides them."); + log.info("Each setting is named after the `clerk migrate import` flag it stands in for."); + log.blank(); + + log.info( + column("SETTING", nameWidth, dim) + + column("VALUE", valueWidth, dim) + + column("SOURCE", sourceWidth, dim) + + dim("DESCRIPTION"), + ); + + for (const cell of cells) { + log.info( + column(cell.name, nameWidth, cyan) + + column(cell.value, valueWidth, (value) => value) + + column(cell.source, sourceWidth, dim) + + dim(cell.setting.description), + ); + } + + const set = cells.filter((cell) => !cell.unset).length; + log.blank(); + log.info(`${set} of ${cells.length} settings set. Credentials are shown redacted.`); + log.blank(); + + printNextSteps(NEXT_STEPS.MIGRATE_SETTINGS); +} diff --git a/packages/cli-core/src/commands/migrate/settings/registry.ts b/packages/cli-core/src/commands/migrate/settings/registry.ts new file mode 100644 index 000000000..f3eb1ffc7 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/registry.ts @@ -0,0 +1,211 @@ +/** + * What `clerk migrate settings` can show and change. + * + * Two stores, split by what the value is rather than by which command wrote it: + * + * - **config** — what this project last migrated. Not secret, not per-machine + * secret material, and useless to anyone but the CLI, so it lives in the + * CLI's own config file keyed by project. + * - **env** — credentials. They go to `.env.clerk-migrate`, which is + * gitignored on write and hand-editable, because a credential belongs + * somewhere the user can rotate it without the CLI's help. + * + * A setting is listed here exactly once; `list`, `set` and `clear` all read + * this table rather than each keeping their own idea of what exists. + */ + +import { REDACTED } from "../../../lib/constants.ts"; + +export type SettingStore = "config" | "env"; + +export interface SettingDef { + /** + * What the user types: `clerk migrate settings set `. + * + * Kebab-case, and identical to the `migrate import` flag it backs. A setting and + * its flag are the same knob reached two ways, so `firebase-signer-key` here + * and `--firebase-signer-key` there must not drift into two spellings the + * user has to learn separately. Sentence-case prose belongs in + * `description`, which is what the list renders alongside it. + */ + name: string; + store: SettingStore; + description: string; + /** + * The environment variable this setting is read from at run time. + * + * Required for an `env` setting, which lives nowhere else. A `config` + * setting may also declare one, meaning "remembered here, but an environment + * value wins" — `log-dir` is that shape, so an operator can pin a directory + * per shell without disturbing what the project remembers. + */ + envVar?: string; + /** + * Other variables accepted for the same setting, read only when + * {@link envVar} is absent. + * + * Firebase hands its four scrypt parameters over as `base64_signer_key`, + * `rounds` and friends, and every guide — including Clerk's own standalone + * migration script — tells the reader to paste them into `.env` under those + * names. Someone who did that has the values the CLI needs, spelled the way + * the source platform spells them, and a listing that reports "not set" is + * wrong about the project rather than strict about it. + * + * Prefixed names still win, and the listing names the variable it read, so a + * generic `ROUNDS` that means something else in the app is visible rather + * than silent. + */ + envAliases?: string[]; + /** For `config` settings, the key on the saved migration entry. */ + configKey?: "transformer" | "file" | "skipUnsupportedProviders" | "logDir"; + /** Redact when displaying — the value is a credential. */ + secret?: boolean; + /** Reject a value the run would only fail on later. */ + validate?: (value: string) => string | undefined; +} + +const positiveInteger = (value: string): string | undefined => { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? undefined : "Expected a positive whole number"; +}; + +const boolean = (value: string): string | undefined => + ["true", "false"].includes(value) ? undefined : "Expected true or false"; + +const path = (value: string): string | undefined => + value.trim().length > 0 ? undefined : "Expected a directory path"; + +export const SETTINGS: SettingDef[] = [ + { + name: "transformer", + store: "config", + configKey: "transformer", + description: "Source platform the export came from", + }, + { + name: "file", + store: "config", + configKey: "file", + description: "Export file to import users from", + }, + { + name: "skip-unsupported-providers", + store: "config", + configKey: "skipUnsupportedProviders", + description: "Skip users with no provider enabled in Clerk (Supabase)", + validate: boolean, + }, + { + name: "log-dir", + store: "config", + configKey: "logDir", + envVar: "CLERK_MIGRATE_LOG_DIR", + description: "Directory migration logs are written to", + validate: path, + }, + { + name: "firebase-signer-key", + store: "env", + envVar: "CLERK_FIREBASE_SIGNER_KEY", + envAliases: ["FIREBASE_BASE64_SIGNER_KEY", "BASE64_SIGNER_KEY"], + description: "Firebase base64 signer key", + secret: true, + }, + { + name: "firebase-salt-separator", + store: "env", + envVar: "CLERK_FIREBASE_SALT_SEPARATOR", + envAliases: ["FIREBASE_BASE64_SALT_SEPARATOR", "BASE64_SALT_SEPARATOR"], + description: "Firebase base64 salt separator", + }, + { + name: "firebase-rounds", + store: "env", + envVar: "CLERK_FIREBASE_ROUNDS", + envAliases: ["FIREBASE_ROUNDS", "ROUNDS"], + description: "Firebase scrypt rounds", + validate: positiveInteger, + }, + { + name: "firebase-mem-cost", + store: "env", + envVar: "CLERK_FIREBASE_MEM_COST", + envAliases: ["FIREBASE_MEM_COST", "MEM_COST"], + description: "Firebase scrypt memory cost", + validate: positiveInteger, + }, +]; + +export const SETTING_NAMES = SETTINGS.map((setting) => setting.name); + +export function findSetting(name: string): SettingDef | undefined { + return SETTINGS.find((setting) => setting.name === name); +} + +/** Levenshtein distance, iterative over a single row. */ +function distance(a: string, b: string): number { + const row = Array.from({ length: b.length + 1 }, (_, i) => i); + + for (let i = 1; i <= a.length; i++) { + let diagonal = row[0] as number; + row[0] = i; + for (let j = 1; j <= b.length; j++) { + const above = row[j] as number; + row[j] = Math.min( + above + 1, + (row[j - 1] as number) + 1, + diagonal + (a[i - 1] === b[j - 1] ? 0 : 1), + ); + diagonal = above; + } + } + + return row[b.length] as number; +} + +/** + * The setting a misspelling was probably reaching for. + * + * Every setting name is a compound of short words — `log-dir`, `firebase-mem-cost` + * — so the misses that matter are a pluralised segment or a transposed pair, + * not a different word entirely. One edit per three characters keeps + * `logs-dir` pointing at `log-dir` without letting an unrelated name match + * something and send the reader off after it. + * + * @returns The closest name within that budget, or `undefined` when nothing is + * close enough to be worth naming. + */ +export function suggestSettingName(name: string): string | undefined { + const budget = Math.max(1, Math.floor(name.length / 3)); + + let best: { name: string; distance: number } | undefined; + for (const candidate of SETTING_NAMES) { + const gap = distance(name, candidate); + if (gap <= budget && (!best || gap < best.distance)) best = { name: candidate, distance: gap }; + } + + return best?.name; +} + +/** + * Every variable an `env` setting answers to, highest priority first. + * + * One list, read by both the listing and the run, so `clerk migrate settings` + * can never show a value the import would ignore. + */ +export function envNames(setting: SettingDef): string[] { + return [setting.envVar as string, ...(setting.envAliases ?? [])]; +} + +/** + * The display value for a setting: withheld entirely when it is a credential. + * + * {@link REDACTED} is what `clerk users create --dry-run` already prints for a + * password, so a credential reads the same wherever the CLI declines to show + * one. Head-and-tail (`aVer…3456`) would say *which* key is set, but the source + * column answers that, and a partial value is one the reader has to recognise + * as partial. + */ +export function displayValue(setting: SettingDef, value: string): string { + return setting.secret ? REDACTED : value; +} diff --git a/packages/cli-core/src/commands/migrate/settings/set.ts b/packages/cli-core/src/commands/migrate/settings/set.ts new file mode 100644 index 000000000..1b0e1d997 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/set.ts @@ -0,0 +1,48 @@ +/** + * `clerk migrate settings set ` — change one setting. + * + * Which store it lands in is a property of the setting, not a flag: a + * credential always goes to `.env.clerk-migrate`, project state always goes to + * the CLI config. Letting the caller choose would mean a signer key could be + * put somewhere that is not gitignored. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { writeMigrateEnvValues } from "../lib/env-file.ts"; +import { loadSettings, saveSettings } from "../lib/settings.ts"; +import { displayValue, findSetting, SETTING_NAMES } from "./registry.ts"; + +export async function set(name: string, value: string): Promise { + const setting = findSetting(name); + if (!setting) { + throwUsageError( + `Unknown setting "${name}". Valid names: ${SETTING_NAMES.join(", ")}.`, + undefined, + undefined, + [ + { + command: "clerk migrate settings", + description: "List the settings and their current values", + }, + ], + ); + } + + const invalid = setting.validate?.(value); + if (invalid) throwUsageError(`Invalid value for ${name}: ${invalid}.`); + + if (setting.store === "env") { + const file = await writeMigrateEnvValues({ [setting.envVar as string]: value }); + log.success(`Set \`${name}\` in ${file} (gitignored).`); + return; + } + + const saved = await loadSettings(); + await saveSettings({ + ...saved, + [setting.configKey as string]: + setting.configKey === "skipUnsupportedProviders" ? value === "true" : value, + }); + log.success(`Set \`${name}\` to ${displayValue(setting, value)} for this project.`); +} diff --git a/packages/cli-core/src/commands/migrate/settings/settings.test.ts b/packages/cli-core/src/commands/migrate/settings/settings.test.ts new file mode 100644 index 000000000..90838c5f3 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/settings/settings.test.ts @@ -0,0 +1,341 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { setMode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { MIGRATE_ENV_FILE } from "../lib/env-file.ts"; +import { loadSettings, saveSettings } from "../lib/settings.ts"; +import { clear } from "./clear.ts"; +import { list } from "./list.ts"; +import { displayValue, findSetting, suggestSettingName } from "./registry.ts"; +import { set } from "./set.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let configDir: string; +let originalCwd: string; + +const envFileContent = () => fs.readFileSync(path.join(workDir, MIGRATE_ENV_FILE), "utf-8"); + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-cmd-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-settings-cfg-")); + _setConfigDir(configDir); + process.chdir(workDir); +}); + +afterAll(() => { + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.rmSync(path.join(workDir, MIGRATE_ENV_FILE), { force: true }); + fs.rmSync(path.join(workDir, ".gitignore"), { force: true }); +}); + +afterEach(() => { + process.exitCode = 0; +}); + +describe("displayValue", () => { + const signerKey = findSetting("firebase-signer-key")!; + + // No part of the value, at any length — the same `[REDACTED]` that + // `clerk users create --dry-run` prints for a password. + test.each([["short"], ["0123456789"], ["aVeryLongSignerKeyValue123456"]])( + "withholds the credential %p entirely", + (value) => { + expect(displayValue(signerKey, value)).toBe("[REDACTED]"); + }, + ); + + test("shows a setting that is not a credential", () => { + expect(displayValue(findSetting("transformer")!, "firebase")).toBe("firebase"); + }); +}); + +describe("set", () => { + test("writes a credential to the gitignored env file, not the CLI config", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + + expect(envFileContent()).toContain("CLERK_FIREBASE_SIGNER_KEY=aVeryLongSignerKeyValue123456"); + expect(await loadSettings()).toEqual({}); + expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toContain(MIGRATE_ENV_FILE); + }); + + test("writes project state to the CLI config, not the env file", async () => { + await set("transformer", "firebase"); + + expect(await loadSettings()).toEqual({ transformer: "firebase" }); + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + }); + + test("keeps the settings it is not changing", async () => { + await saveSettings({ transformer: "clerk", file: "users.json" }); + await set("file", "other.json"); + + expect(await loadSettings()).toEqual({ transformer: "clerk", file: "other.json" }); + }); + + test("stores a boolean setting as a boolean", async () => { + await set("skip-unsupported-providers", "true"); + expect(await loadSettings()).toEqual({ skipUnsupportedProviders: true }); + }); + + test.each([ + ["firebase-rounds", "zero", /positive whole number/], + ["skip-unsupported-providers", "yes", /true or false/], + ])("rejects an invalid value for %s", async (name, value, message) => { + await expect(set(name, value)).rejects.toThrow(message); + }); + + test("names the valid settings when given an unknown one", async () => { + await expect(set("nope", "x")).rejects.toThrow(/firebase-signer-key/); + }); + + // A run would fail on it later; failing at write time keeps the bad value out + // of the file entirely. + test("writes nothing when the value is rejected", async () => { + await expect(set("firebase-rounds", "-1")).rejects.toThrow(); + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + }); +}); + +describe("list", () => { + test("names the source each value resolved from", async () => { + await set("transformer", "firebase"); + await set("firebase-salt-separator", "Bw=="); + captured.clear(); + + await list(); + + expect(captured.err).toContain("clerk config"); + expect(captured.err).toContain(MIGRATE_ENV_FILE); + }); + + test("redacts a credential but not the rest", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + await set("transformer", "firebase"); + captured.clear(); + + await list(); + + expect(captured.err).toContain("[REDACTED]"); + expect(captured.err).not.toContain("aVeryLongSignerKeyValue123456"); + expect(captured.err).toContain("firebase"); + }); + + // --json is what gets piped into a ticket or a CI log. + test("redacts in JSON output too", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + captured.clear(); + + await list({ json: true }); + + expect(captured.out).not.toContain("aVeryLongSignerKeyValue123456"); + expect(JSON.parse(captured.out)).toContainEqual( + expect.objectContaining({ name: "firebase-signer-key", value: "[REDACTED]", secret: true }), + ); + }); + + // The names are kebab-case because they mirror the `migrate import` flags; the + // description column is what makes the list readable. + test("explains each setting in prose", async () => { + await list(); + + expect(captured.err).toContain("Source platform the export came from"); + expect(captured.err).toContain("Export file to import users from"); + }); + + test("carries the description into JSON too", async () => { + await list({ json: true }); + + expect(JSON.parse(captured.out)).toContainEqual( + expect.objectContaining({ name: "file", description: "Export file to import users from" }), + ); + }); + + // Colouring before padding counts the ANSI bytes towards the column width, + // which pulls later columns left on exactly the rows that have a value. + test("starts the description at one column, set or not", async () => { + await set("transformer", "supabase"); + captured.clear(); + + await list(); + + // eslint-disable-next-line no-control-regex + const plain = captured.err.replaceAll(/\u001B\[\d+m/g, ""); + const columnOf = (description: string) => + plain + .split("\n") + .find((row) => row.includes(description)) + ?.indexOf(description); + + expect(columnOf("Source platform the export came from")).toBe( + columnOf("Export file to import users from") as number, + ); + }); + + test("marks everything as unset in a fresh project", async () => { + await list({ json: true }); + expect(JSON.parse(captured.out).every((entry: { set: boolean }) => !entry.set)).toBe(true); + }); + + // A listing is where someone lands before they know what to type, so it + // closes by naming the two commands that change what it just showed — + // the same next-steps block `mcp list` and `whoami` end on. + test("closes with next steps", async () => { + setMode("human"); + await list(); + setMode("agent"); + + expect(captured.err).toContain("clerk migrate settings set "); + expect(captured.err).toContain("clerk migrate settings clear"); + }); + + test("counts how many are set", async () => { + await set("transformer", "firebase"); + captured.clear(); + + await list(); + + expect(captured.err).toContain("1 of 8 settings set"); + }); + + // Firebase's own names for these, and what every guide tells you to paste + // into `.env`. Reporting "not set" for a value the import would read is the + // listing being wrong about the project rather than strict about it. + test("reads a credential written under the name Firebase uses", async () => { + fs.writeFileSync(path.join(workDir, ".env.local"), "ROUNDS=8\n"); + captured.clear(); + + await list(); + + fs.rmSync(path.join(workDir, ".env.local")); + // Named alongside the file: `ROUNDS` may mean something else in this app. + expect(captured.err).toContain(".env.local (ROUNDS)"); + }); +}); + +describe("clear", () => { + test("empties both stores", async () => { + await set("transformer", "firebase"); + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + + await clear({ yes: true }); + + expect(await loadSettings()).toEqual({}); + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + }); + + test("says so rather than claiming to have cleared nothing", async () => { + await clear({ yes: true }); + expect(captured.err).toContain("No migration settings to clear"); + }); + + test("leaves settings the migration does not own", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "OTHER=keep\n"); + await set("firebase-rounds", "8"); + + await clear({ yes: true }); + + expect(envFileContent()).toBe("OTHER=keep\n"); + }); + + // It destroys credentials, like `logs clean` destroys logs and `migrate + // delete` destroys users — and those two both refuse rather than assume. + // Proceeding here because nobody could be asked is the one reading of + // silence that cannot be undone. + test("refuses rather than assuming when it cannot prompt", async () => { + await set("transformer", "firebase"); + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + + await expect(clear({})).rejects.toThrow(/cannot prompt here\. Pass -y to confirm/); + + expect(await loadSettings()).toMatchObject({ transformer: "firebase" }); + expect(envFileContent()).toContain("CLERK_FIREBASE_SIGNER_KEY"); + }); +}); + +describe("clear ", () => { + test("drops one config setting and keeps the rest", async () => { + await set("transformer", "firebase"); + await set("file", "users.json"); + + await clear({ yes: true }, "file"); + + expect(await loadSettings()).toEqual({ transformer: "firebase" }); + }); + + test("drops one credential and keeps the rest of the env file", async () => { + await set("firebase-signer-key", "aVeryLongSignerKeyValue123456"); + await set("firebase-rounds", "8"); + + await clear({ yes: true }, "firebase-signer-key"); + + expect(envFileContent()).toContain("CLERK_FIREBASE_ROUNDS=8"); + expect(envFileContent()).not.toContain("CLERK_FIREBASE_SIGNER_KEY"); + }); + + // Clearing only the prefixed name would report success and leave the next run + // reading the alias. + test("drops every spelling the setting answers to", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "ROUNDS=8\nOTHER=keep\n"); + + await clear({ yes: true }, "firebase-rounds"); + + expect(envFileContent()).toBe("OTHER=keep\n"); + }); + + test("says so when the setting was not set here", async () => { + await clear({ yes: true }, "firebase-rounds"); + expect(captured.err).toContain("firebase-rounds"); + expect(captured.err).toContain("is not set here"); + + captured.clear(); + await clear({ yes: true }, "transformer"); + expect(captured.err).toContain("transformer"); + expect(captured.err).toContain("is not set here"); + }); + + // `log-dir` is remembered in the config but yields to an env var, so half a + // clear would report success and leave the run reading the same directory. + test("clears a setting that lives in both stores", async () => { + fs.writeFileSync(path.join(workDir, MIGRATE_ENV_FILE), "CLERK_MIGRATE_LOG_DIR=./env-logs\n"); + await saveSettings({ logDir: "./saved-logs", transformer: "firebase" }); + + await clear({ yes: true }, "log-dir"); + + expect(fs.existsSync(path.join(workDir, MIGRATE_ENV_FILE))).toBe(false); + expect(await loadSettings()).toEqual({ transformer: "firebase" }); + }); + + test("rejects a name that is not a setting", async () => { + await expect(clear({ yes: true }, "nope")).rejects.toThrow(/Unknown setting "nope"/); + }); +}); + +describe("suggestSettingName", () => { + // `.choices()` rejects before the action runs, so this is the only thing + // standing between a one-character miss and a bare list of eight names. + test.each([ + ["logs-dir", "log-dir"], + ["log_dir", "log-dir"], + ["firebase-round", "firebase-rounds"], + ["transfomer", "transformer"], + ])("%s -> %s", (typo, expected) => { + expect(suggestSettingName(typo)).toBe(expected); + }); + + test.each(["banana", "secret", ""])("says nothing for %p", (unrelated) => { + expect(suggestSettingName(unrelated)).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/auth0.ts b/packages/cli-core/src/commands/migrate/transformers/auth0.ts new file mode 100644 index 000000000..c1595d139 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/auth0.ts @@ -0,0 +1,43 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification } from "./shared.ts"; + +/** + * Auth0 → Clerk transformer. + * + * Works with Auth0's Export Users API. `user_id` is a `provider|id` string + * (`auth0|abc123`, `github|12345`) and is carried through as the Clerk user's + * `external_id`. + * + * Auth0 does not include password hashes in a standard export — they have to + * be requested from Auth0 support. When present they are bcrypt (`$2a$`/`$2b$`, + * 10 rounds), which is why `passwordHasher` defaults to `bcrypt`. + */ +const auth0Transformer = { + key: "auth0", + label: "Auth0", + description: + "Works with Auth0's Export Users API. Password hashes require a support request to Auth0.", + transformer: { + user_id: "userId", + email: "email", + email_verified: "emailVerified", + username: "username", + given_name: "firstName", + family_name: "lastName", + phone_number: "phone", + phone_verified: "phoneVerified", + passwordHash: "password", + user_metadata: "publicMetadata", + app_metadata: "privateMetadata", + created_at: "createdAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "boolean"); + routeByVerification(user, "phone", "phoneVerified", "boolean"); + }, + defaults: { + passwordHasher: "bcrypt" as const, + }, +} satisfies TransformerRegistryEntry; + +export default auth0Transformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/authjs.ts b/packages/cli-core/src/commands/migrate/transformers/authjs.ts new file mode 100644 index 000000000..3391b1804 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/authjs.ts @@ -0,0 +1,38 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, splitName } from "./shared.ts"; + +/** + * Auth.js (formerly NextAuth) → Clerk transformer. + * + * Auth.js has no export tool and no fixed user table, so this assumes the + * common shape: `SELECT id, name, email, email_verified, created_at FROM users`. + * A different schema means editing the mapping below or supplying a custom + * transformer file. + * + * `email_verified` is a nullable timestamp rather than a boolean — any value + * means verified. + * + * No password default: Auth.js's core is passwordless (OAuth and email links), + * so users arrive without a digest and are imported with + * `skip_password_requirement`. + */ +const authjsTransformer = { + key: "authjs", + label: "Auth.js (NextAuth)", + description: + "Assumes an export of `SELECT id, name, email, email_verified, created_at FROM users`. `name` is split into firstName and lastName.", + transformer: { + id: "userId", + email: "email", + email_verified: "emailVerified", + name: "name", + created_at: "createdAt", + updated_at: "updatedAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "timestamp"); + splitName(user); + }, +} satisfies TransformerRegistryEntry; + +export default authjsTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/betterauth.ts b/packages/cli-core/src/commands/migrate/transformers/betterauth.ts new file mode 100644 index 000000000..f32d2f31c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/betterauth.ts @@ -0,0 +1,46 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, splitName } from "./shared.ts"; + +/** + * Better Auth → Clerk transformer. + * + * Works with `clerk migrate export betterauth`, which joins the user table + * with the credential account row to pick up the bcrypt `password_hash`. + * + * Better Auth plugins add columns Clerk has no equivalent for + * (`display_username`, `role`, `ban_reason`, `two_factor_enabled`). They need + * no handling: the schema strips anything it does not declare. `banned` is the + * exception, because that one *is* a Clerk field. + */ +const betterAuthTransformer = { + key: "betterauth", + label: "Better Auth", + description: + "Works with the Better Auth export. Supports bcrypt passwords and the admin plugin's banned flag.", + transformer: { + user_id: "userId", + email: "email", + email_verified: "emailVerified", + name: "name", + password_hash: "password", + username: "username", + phone_number: "phone", + phone_number_verified: "phoneVerified", + created_at: "createdAt", + updated_at: "updatedAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "boolean"); + routeByVerification(user, "phone", "phoneVerified", "boolean"); + splitName(user); + + // Only carry `banned` when it is actually true — Better Auth writes false + // for every user that was never banned, and sending that to Clerk is noise. + if (user.banned !== true) delete user.banned; + }, + defaults: { + passwordHasher: "bcrypt" as const, + }, +} satisfies TransformerRegistryEntry; + +export default betterAuthTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/clerk.ts b/packages/cli-core/src/commands/migrate/transformers/clerk.ts new file mode 100644 index 000000000..8fb094839 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/clerk.ts @@ -0,0 +1,45 @@ +import type { TransformerRegistryEntry } from "../types.ts"; + +/** + * Clerk → Clerk transformer, for moving users between Clerk instances + * (typically development → production). + * + * Maps the Dashboard's user export format onto the import schema. + */ +const clerkTransformer = { + key: "clerk", + label: "Clerk", + description: + "Migrate between Clerk instances (e.g. development to production, or to another Clerk application). Export your users from the Clerk Dashboard first.", + transformer: { + id: "userId", + primary_email_address: "email", + verified_email_addresses: "emailAddresses", + unverified_email_addresses: "unverifiedEmailAddresses", + first_name: "firstName", + last_name: "lastName", + password_digest: "password", + password_hasher: "passwordHasher", + primary_phone_number: "phone", + verified_phone_numbers: "phoneNumbers", + unverified_phone_numbers: "unverifiedPhoneNumbers", + username: "username", + totp_secret: "totpSecret", + backup_codes_enabled: "backupCodesEnabled", + backup_codes: "backupCodes", + public_metadata: "publicMetadata", + unsafe_metadata: "unsafeMetadata", + private_metadata: "privateMetadata", + // Account state a Dashboard export carries and `POST /v1/users` accepts. + // Unmapped, these survive the export and are then silently stripped at + // validation — losing original signup dates on a dev → prod migration. + created_at: "createdAt", + legal_accepted_at: "legalAcceptedAt", + banned: "banned", + create_organization_enabled: "createOrganizationEnabled", + create_organizations_limit: "createOrganizationsLimit", + delete_self_enabled: "deleteSelfEnabled", + }, +} satisfies TransformerRegistryEntry; + +export default clerkTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/firebase.ts b/packages/cli-core/src/commands/migrate/transformers/firebase.ts new file mode 100644 index 000000000..9c9d09c6e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/firebase.ts @@ -0,0 +1,122 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import type { PreTransformResult, TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, splitName, toIsoDate } from "./shared.ts"; + +/** + * Column order of `firebase auth:export --format=csv`, which writes no header + * row. Without these the CSV parser would treat the first user as the header. + */ +const FIREBASE_CSV_HEADERS = + "localId,email,emailVerified,passwordHash,passwordSalt,displayName,photoUrl," + + "googleId,googleEmail,googleDisplayName,googlePhotoUrl," + + "facebookId,facebookEmail,facebookDisplayName,facebookPhotoUrl," + + "twitterId,twitterEmail,twitterDisplayName,twitterPhotoUrl," + + "githubId,githubEmail,githubDisplayName,githubPhotoUrl," + + "createdAt,lastSignedInAt,phoneNumber,disabled,customAttributes,providerUserInfo"; + +/** + * Firebase → Clerk transformer. + * + * Handles both shapes `firebase auth:export` produces: a headerless CSV, and + * JSON wrapped in `{ users: [...] }`. + * + * Firebase's scrypt is a modified variant, so Clerk needs the project's four + * hash parameters alongside each digest. They arrive on the run's + * {@link TransformContext} from `--firebase-*` flags or the matching + * `CLERK_FIREBASE_*` environment variables; they are never persisted. + * + * See https://clerk.com/docs/guides/development/migrating/firebase + */ +const firebaseTransformer = { + key: "firebase", + label: "Firebase", + description: + "Works with `firebase auth:export` (CSV or JSON). Requires the project's four password hash parameters to migrate passwords.", + + preTransform: (filePath: string, fileType: string): PreTransformResult => { + if (fileType === "text/csv") { + // Written to the OS temp dir rather than the user's cwd: this is a + // parsing artifact, not a migration output like ./logs. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-firebase-")); + const withHeaders = path.join(tmpDir, path.basename(filePath)); + fs.writeFileSync( + withHeaders, + `${FIREBASE_CSV_HEADERS}\n${fs.readFileSync(filePath, "utf-8")}`, + ); + return { filePath: withHeaders }; + } + + if (fileType === "application/json") { + const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + if (Array.isArray(parsed)) return { filePath, data: parsed as Record[] }; + + const users = (parsed as { users?: unknown })?.users; + if (Array.isArray(users)) return { filePath, data: users as Record[] }; + + throw new CliError( + "Invalid Firebase JSON export: expected `{ users: [...] }` or an array of users.", + { code: ERROR_CODE.INVALID_JSON }, + ); + } + + return { filePath }; + }, + + transformer: { + localId: "userId", + email: "email", + emailVerified: "emailVerified", + passwordHash: "passwordHash", + passwordSalt: "salt", + phoneNumber: "phone", + displayName: "name", + }, + + postTransform: (user, context) => { + const passwordHash = user.passwordHash; + const salt = user.salt; + + if (passwordHash && salt) { + const config = context.firebaseHashConfig; + if (!config) { + throw new CliError( + "This export contains Firebase password hashes, which need the project's hash parameters to import.\n" + + "Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters, then pass:\n" + + " --firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost", + { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: "https://clerk.com/docs/guides/development/migrating/firebase", + }, + ); + } + + // Clerk's scrypt_firebase hasher expects every parameter inline: + // hash$salt$signerKey$saltSeparator$rounds$memCost + user.password = [ + passwordHash, + salt, + config.base64_signer_key, + config.base64_salt_separator, + config.rounds, + config.mem_cost, + ].join("$"); + + delete user.passwordHash; + delete user.salt; + } + + routeByVerification(user, "email", "emailVerified", "boolean"); + // Firebase exports timestamps as Unix milliseconds, often as strings. + user.createdAt = toIsoDate(user.createdAt, true); + splitName(user); + }, + + defaults: { + passwordHasher: "scrypt_firebase" as const, + }, +} satisfies TransformerRegistryEntry; + +export default firebaseTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/list.test.ts b/packages/cli-core/src/commands/migrate/transformers/list.test.ts new file mode 100644 index 000000000..a297ec919 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/list.test.ts @@ -0,0 +1,171 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { list, wrapText } from "./list.ts"; +import { transformers } from "./registry.ts"; + +const captured = useCaptureLog(); + +const stripAnsi = (value: string) => value.replace(/\[[0-9;]*m/g, ""); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-tlist-"))); + process.chdir(workDir); + fs.writeFileSync( + path.join(workDir, "custom.ts"), + `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId" }, + };`, + ); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("human output", () => { + test.each([...transformers])("lists the $key transformer with its label", async (transformer) => { + await list(); + expect(captured.err).toContain(transformer.key); + expect(captured.err).toContain(transformer.label); + }); + + // Two normalizations: `log.info` auto-highlights backticked spans, so the + // rendered description carries colour codes the source string does not, and + // descriptions are wrapped to the terminal width across several indented + // lines. Collapsing whitespace compares the words, not the layout. + const collapse = (value: string) => stripAnsi(value).replace(/\s+/g, " "); + + test.each([...transformers])("includes the $key description", async (transformer) => { + await list(); + expect(collapse(captured.err)).toContain(collapse(transformer.description)); + }); + + test("counts the built-ins", async () => { + await list(); + expect(captured.err).toContain(`${transformers.length} built-in transformers`); + }); + + // A compiled binary has no source tree to grep, so the way to extend it has + // to be discoverable from the list itself. + test("says how to add one when none is loaded", async () => { + await list(); + expect(captured.err).toContain("--transformer-file"); + }); + + test("appends a custom transformer and names its source", async () => { + await list({ transformerFile: "./custom.ts" }); + + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("custom — ./custom.ts"); + expect(captured.err).toContain("plus 1 loaded from --transformer-file"); + }); + + test("drops the how-to hint once one is loaded", async () => { + await list({ transformerFile: "./custom.ts" }); + expect(captured.err).not.toContain("Migrating from something else?"); + }); +}); + +describe("--json", () => { + test("emits every built-in on stdout", async () => { + await list({ json: true }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed).toHaveLength(transformers.length); + expect(parsed.map((entry) => entry.key)).toEqual(transformers.map((entry) => entry.key)); + }); + + test("reports key, label, description and the userId source field", async () => { + await list({ json: true }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed[0]).toMatchObject({ + key: "clerk", + label: "Clerk", + built_in: true, + maps_to_user_id: "id", + }); + }); + + test("marks a custom transformer as not built in", async () => { + await list({ json: true, transformerFile: "./custom.ts" }); + + const parsed = JSON.parse(captured.out) as Record[]; + expect(parsed.at(-1)).toMatchObject({ + key: "myplatform", + built_in: false, + source: "./custom.ts", + maps_to_user_id: "account_ref", + }); + }); +}); + +describe("a bad --transformer-file", () => { + test("fails rather than listing only the built-ins", async () => { + await expect(list({ transformerFile: "./nope.ts" })).rejects.toThrow(CliError); + }); +}); + +describe("human-mode frame", () => { + let originalMode: Mode; + + beforeAll(() => { + originalMode = getMode(); + setMode("human"); + }); + + afterAll(() => { + setMode(originalMode); + }); + + // Reading a static registry is not a run: there is no progress to bracket, + // and the gutter's `│` would sit in front of every wrapped line. + test("prints no intro/outro gutter", async () => { + await list(); + + expect(captured.err).not.toContain("┌"); + expect(captured.err).not.toContain("└"); + expect(stripAnsi(captured.err)).toContain("Transformers:"); + }); + + test("--json stays on stdout only", async () => { + await list({ json: true }); + + expect(() => JSON.parse(captured.out)).not.toThrow(); + expect(captured.err).toBe(""); + }); +}); + +describe("wrapText", () => { + test("breaks on whitespace within the width", () => { + expect(wrapText("one two three four", 9)).toEqual(["one two", "three", "four"]); + }); + + // `log.info` pairs backticks per line, so a span split across two lines + // leaves one unmatched backtick on each and colours the wrong half of both. + test("never breaks inside a backticked span", () => { + const lines = wrapText("Assumes an export of `SELECT id, name FROM users`.", 30); + + expect(lines).toContain("`SELECT id, name FROM users`."); + for (const line of lines) { + expect((line.match(/`/g) ?? []).length % 2).toBe(0); + } + }); + + test("gives an over-long word its own line rather than dropping it", () => { + expect(wrapText("short supercalifragilistic", 8)).toEqual(["short", "supercalifragilistic"]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/list.ts b/packages/cli-core/src/commands/migrate/transformers/list.ts new file mode 100644 index 000000000..993ee4838 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/list.ts @@ -0,0 +1,120 @@ +/** + * `clerk migrate transformers list` — which source platforms are available. + * + * New in the CLI. The standalone tool's interactive picker was the only place + * these were listed, which was fine when the user had the source tree to grep. + * A compiled binary's users have neither, so the list is a command. + */ + +import { bold, cyan } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import type { TransformerRegistryEntry } from "../types.ts"; +import { loadCustomTransformer } from "./load-custom.ts"; +import { transformers } from "./registry.ts"; + +export type TransformersListOptions = { + json?: boolean; + transformerFile?: string; +}; + +type Listed = TransformerRegistryEntry & { builtIn: boolean; source?: string }; + +function toJson(entries: Listed[]) { + return entries.map((entry) => ({ + key: entry.key, + label: entry.label, + description: entry.description, + built_in: entry.builtIn, + ...(entry.source ? { source: entry.source } : {}), + maps_to_user_id: + Object.entries(entry.transformer).find(([, target]) => target === "userId")?.[0] ?? null, + })); +} + +/** + * Capped, not just measured: a description that rewrapped differently on every + * terminal makes two runs of the same command look like different output. 80 is + * the same width `--help` lays itself out at. + */ +const MAX_WIDTH = 80; + +function outputWidth(): number { + return Math.min(process.stderr.columns || MAX_WIDTH, MAX_WIDTH); +} + +/** + * A run of non-space characters, except that a backticked span counts as one + * character run even when it contains spaces. Keeps `SELECT a, b FROM users` + * whole: `log.info` pairs backticks per line, so a span broken across two lines + * leaves an unmatched backtick on each and colours the wrong half of both. + */ +const WORD = /(?:`[^`]*`|\S)+/g; + +/** + * Wraps on whitespace. Safe to measure raw because the backtick spans + * `log.info` highlights keep their backticks — the colour it adds is invisible + * to width, and nothing here is coloured before wrapping. + */ +export function wrapText(text: string, width: number): string[] { + const lines: string[] = []; + let line = ""; + + for (const word of text.match(WORD) ?? []) { + if (!line) line = word; + else if (line.length + 1 + word.length <= width) line += ` ${word}`; + else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + + return lines; +} + +export async function list(options: TransformersListOptions = {}): Promise { + const entries: Listed[] = transformers.map((entry) => ({ ...entry, builtIn: true })); + + if (options.transformerFile) { + const custom = await loadCustomTransformer(options.transformerFile); + entries.push({ ...custom, builtIn: false, source: options.transformerFile }); + } + + if (options.json) { + log.data(JSON.stringify(toJson(entries), null, 2)); + return; + } + + const width = outputWidth(); + + // No gutter: this reads a static registry, it does not run anything. The + // frame belongs on `migrate import`, where there is progress to bracket. + for (const line of wrapText( + "A transformer maps one platform's export onto the fields Clerk imports. " + + "Pass the one your export came from as `--transformer `.", + width, + )) { + log.info(line); + } + log.blank(); + + log.info(bold("Transformers:")); + for (const entry of entries) { + const suffix = entry.builtIn ? "" : ` (custom — ${entry.source})`; + log.info(` ${cyan(bold(entry.key))} ${entry.label}${suffix}`); + for (const line of wrapText(entry.description, width - 4)) { + log.info(` ${line}`); + } + log.blank(); + } + + const custom = entries.length - transformers.length; + log.info( + `${transformers.length} built-in transformer${transformers.length === 1 ? "" : "s"}` + + (custom > 0 ? ` plus ${custom} loaded from --transformer-file` : ""), + ); + + if (custom === 0) { + log.info("Migrating from something else? Write a transformer and pass --transformer-file."); + } +} diff --git a/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts b/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts new file mode 100644 index 000000000..5dff6ad0b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/load-custom.test.ts @@ -0,0 +1,222 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { loadCustomTransformer, validateTransformer } from "./load-custom.ts"; +import { __resetCustomTransformersForTesting } from "./registry.ts"; + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-custom-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +afterEach(() => { + __resetCustomTransformersForTesting(); +}); + +/** + * Writes a transformer file with a unique name. + * + * Names must not repeat: a dynamic `import()` caches by URL, so reusing one + * would silently return the previous test's module. + */ +function writeTransformer(source: string, ext = "ts"): string { + const name = `custom-${counter++}.${ext}`; + fs.writeFileSync(path.join(workDir, name), source); + return `./${name}`; +} + +const VALID = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email" }, +};`; + +describe("loadCustomTransformer", () => { + test("loads a user-authored TypeScript transformer", async () => { + const entry = await loadCustomTransformer(writeTransformer(VALID)); + + expect(entry).toMatchObject({ + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId", contact_email: "email" }, + }); + }); + + test("loads plain JavaScript too", async () => { + const entry = await loadCustomTransformer(writeTransformer(VALID, "js")); + expect(entry.key).toBe("myplatform"); + }); + + // The file is the user's own code; the CLI must transpile whatever they wrote. + test("transpiles TypeScript syntax the runtime has to strip", async () => { + const entry = await loadCustomTransformer( + writeTransformer(` + interface Entry { key: string; label: string; transformer: Record } + const mapping = { my_id: "userId" } as const; + const custom: Entry = { key: "tsplatform", label: "TS", transformer: { ...mapping } }; + export default custom satisfies Entry; + `), + ); + expect(entry.key).toBe("tsplatform"); + }); + + test("carries the optional hooks through", async () => { + const entry = await loadCustomTransformer( + writeTransformer(`export default { + key: "hooked", label: "Hooked", + transformer: { id: "userId" }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { user.firstName = "set"; }, + };`), + ); + + expect(entry.defaults).toEqual({ passwordHasher: "bcrypt" }); + const user: Record = {}; + entry.postTransform?.(user, {}); + expect(user.firstName).toBe("set"); + }); + + test("supplies a description when the author omitted one", async () => { + const entry = await loadCustomTransformer( + writeTransformer( + `export default { key: "bare", label: "Bare", transformer: { id: "userId" } };`, + ), + ); + expect(entry.description).toBe("Custom transformer"); + }); + + test("reports a path that is not there", async () => { + await expect(loadCustomTransformer("./nope.ts")).rejects.toThrow(/No transformer file at/); + }); + + test("reports a directory given instead of a file", async () => { + fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); + await expect(loadCustomTransformer("./adir")).rejects.toThrow(/is a directory/); + }); + + test("reports a file that does not parse, quoting the syntax error", async () => { + await expect( + loadCustomTransformer(writeTransformer("export default { key: ,,, }")), + ).rejects.toThrow(/Could not load/); + }); + + test("reports a file that throws while loading", async () => { + await expect( + loadCustomTransformer(writeTransformer(`throw new Error("boom"); export default {};`)), + ).rejects.toThrow(/Could not load .*boom/s); + }); + + test("points at a named export when the default is missing", async () => { + const file = writeTransformer( + `export const myPlatform = { key: "x", label: "X", transformer: { a: "userId" } };`, + ); + + await expect(loadCustomTransformer(file)).rejects.toThrow( + /has no default export.*`myPlatform`.*did you mean `export default`/s, + ); + }); + + test("reports a missing default with no named exports to suggest", async () => { + await expect(loadCustomTransformer(writeTransformer("const unused = 1;"))).rejects.toThrow( + /has no default export\.$/m, + ); + }); +}); + +describe("validateTransformer", () => { + const valid = { + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId" }, + }; + + test("accepts a minimal valid entry", () => { + expect(validateTransformer(valid, "f.ts").key).toBe("myplatform"); + }); + + test.each([ + ["a null default export", null, /is null, not an object/], + ["a number default export", 42, /is number, not an object/], + ["a string default export", "nope", /is string, not an object/], + ])("rejects %s", (_label, value, expected) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(expected); + }); + + test.each([ + ["key", { ...valid, key: undefined }], + ["key", { ...valid, key: "" }], + ["key", { ...valid, key: " " }], + ["key", { ...valid, key: 7 }], + ["label", { ...valid, label: undefined }], + ["label", { ...valid, label: "" }], + ])("rejects a bad %s naming the field", (field, value) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test("rejects a non-string description", () => { + expect(() => validateTransformer({ ...valid, description: 7 }, "f.ts")).toThrow( + /`description` must be a string/, + ); + }); + + test.each([ + ["missing", { ...valid, transformer: undefined }], + ["null", { ...valid, transformer: null }], + ["an array", { ...valid, transformer: [] }], + ["a string", { ...valid, transformer: "id" }], + ])("rejects a transformer mapping that is %s", (_label, value) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(/`transformer`|`transformer\./); + }); + + test("names the offending entry when a mapping target is not a field name", () => { + expect(() => + validateTransformer({ ...valid, transformer: { account_ref: "userId", bad: 7 } }, "f.ts"), + ).toThrow(/`transformer.bad` must map to a Clerk field name, got number/); + }); + + // Without it the import runs to completion and creates every user with no + // external_id — which is what makes a migration reversible. + test("rejects a mapping with no userId target", () => { + expect(() => validateTransformer({ ...valid, transformer: { a: "email" } }, "f.ts")).toThrow( + /no source field maps to `userId`/, + ); + }); + + test.each([ + ["defaults", { ...valid, defaults: "nope" }], + ["preTransform", { ...valid, preTransform: "nope" }], + ["postTransform", { ...valid, postTransform: 7 }], + ])("rejects a %s of the wrong type", (field, value) => { + expect(() => validateTransformer(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test.each([["clerk"], ["auth0"], ["supabase"]])( + "rejects %s, which would shadow a built-in", + (key) => { + expect(() => validateTransformer({ ...valid, key }, "f.ts")).toThrow( + /already a built-in transformer/, + ); + }, + ); + + test("names the file in every message, so the author knows which one", () => { + expect(() => validateTransformer({}, "./their-file.ts")).toThrow(/\.\/their-file\.ts/); + }); + + test("raises CliError, so the global handler formats it", () => { + expect(() => validateTransformer({}, "f.ts")).toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/transformers/load-custom.ts b/packages/cli-core/src/commands/migrate/transformers/load-custom.ts new file mode 100644 index 000000000..1ac82a719 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/load-custom.ts @@ -0,0 +1,164 @@ +/** + * Loading a user-authored transformer at runtime. + * + * In the standalone migration-tool, supporting a new platform meant adding a + * file to `src/transformers/` and one line to the registry — the user had the + * source tree. A compiled binary has neither a source tree to edit nor a way + * for an end user to rebuild it, so `--transformer-file` restores that + * extensibility by importing a file from the user's own project instead. + * + * **Verified before this was built on:** a `bun build --compile` executable can + * `import()` an arbitrary external `.ts` file at runtime, including TypeScript + * that needs transpiling. Bun's transpiler is part of the runtime, not only the + * bundler. Confirmed with a throwaway compiled binary on darwin-arm64, + * linux-arm64 (glibc), linux-arm64-musl and linux-x64. + * + * The file is user-supplied code the CLI executes, so its shape is validated + * up front and rejected with a specific message rather than crashing deep in + * the transform pipeline on a missing field. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import type { TransformerRegistryEntry } from "../types.ts"; +import { transformers } from "./registry.ts"; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; + +function invalid(problem: string, file: string): never { + throw new CliError(`${file} is not a valid transformer: ${problem}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); +} + +/** + * Checks a loaded value against the registry entry shape. + * + * Every failure names the specific field and what was wrong with it — the + * author is writing this file by hand against a shape they cannot see. + * + * @param file - Path as the user typed it, for the error message. + */ +export function validateTransformer(value: unknown, file: string): TransformerRegistryEntry { + if (value === null || typeof value !== "object") { + invalid(`the default export is ${value === null ? "null" : typeof value}, not an object`, file); + } + + const entry = value as Record; + + for (const field of ["key", "label"] as const) { + if (typeof entry[field] !== "string" || entry[field].trim().length === 0) { + invalid(`\`${field}\` must be a non-empty string`, file); + } + } + + if (entry.description !== undefined && typeof entry.description !== "string") { + invalid("`description` must be a string when present", file); + } + + // Arrays are objects, and an author who wrote `transformer: []` should hear + // that rather than the downstream "no field maps to userId". + if ( + entry.transformer === null || + typeof entry.transformer !== "object" || + Array.isArray(entry.transformer) + ) { + invalid("`transformer` must be an object mapping source fields to Clerk fields", file); + } + + const mapping = entry.transformer as Record; + for (const [source, target] of Object.entries(mapping)) { + if (typeof target !== "string" || target.length === 0) { + invalid( + `\`transformer.${source}\` must map to a Clerk field name, got ${typeof target}`, + file, + ); + } + } + + // Without this the import runs to completion and creates every user with no + // external_id, which is what makes a migration re-runnable and reversible. + if (!Object.values(mapping).includes("userId")) { + invalid( + "no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id", + file, + ); + } + + if ( + entry.defaults !== undefined && + (entry.defaults === null || typeof entry.defaults !== "object" || Array.isArray(entry.defaults)) + ) { + invalid("`defaults` must be an object when present", file); + } + + for (const hook of ["preTransform", "postTransform"] as const) { + if (entry[hook] !== undefined && typeof entry[hook] !== "function") { + invalid(`\`${hook}\` must be a function when present`, file); + } + } + + if (transformers.some((builtIn) => builtIn.key === entry.key)) { + invalid( + `\`key\` is "${String(entry.key)}", which is already a built-in transformer. Choose another key`, + file, + ); + } + + return { + ...(entry as unknown as TransformerRegistryEntry), + description: (entry.description as string | undefined) ?? "Custom transformer", + }; +} + +/** + * Imports and validates a user-authored transformer. + * + * @throws CliError when the path is missing, the module fails to load, or the + * exported value does not match the registry entry shape. + */ +export async function loadCustomTransformer(file: string): Promise { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No transformer file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + if (fs.statSync(resolved).isDirectory()) { + throw new CliError(`${resolved} is a directory, not a transformer file.`, { + code: ERROR_CODE.USAGE_ERROR, + }); + } + + let module: Record; + try { + // A file URL rather than a bare path: an absolute POSIX path happens to + // work, but a Windows path (`C:\...`) is not a valid import specifier. + module = (await import(Bun.pathToFileURL(resolved).href)) as Record; + } catch (error) { + throw new CliError( + `Could not load ${file}: ${(error as Error).message}\n` + + "The file must be valid JavaScript or TypeScript that this CLI can import.", + { code: ERROR_CODE.USAGE_ERROR, docsUrl: DOCS_URL }, + ); + } + + if (module.default === undefined) { + // Point at what they probably meant rather than just restating the rule. + const named = Object.keys(module).filter((key) => key !== "default"); + const hint = + named.length > 0 + ? ` Found named export${named.length === 1 ? "" : "s"} ${named.map((n) => `\`${n}\``).join(", ")} — did you mean \`export default\`?` + : ""; + throw new CliError(`${file} has no default export.${hint}`, { + code: ERROR_CODE.USAGE_ERROR, + docsUrl: DOCS_URL, + }); + } + + return validateTransformer(module.default, file); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/registry.ts b/packages/cli-core/src/commands/migrate/transformers/registry.ts new file mode 100644 index 000000000..ae2f133b8 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/registry.ts @@ -0,0 +1,76 @@ +/** + * Transformer registry. + * + * `migrate import` reads this array to resolve `--transformer` and to list the + * valid choices in help output and tab-completion. + * + * To add a platform: create `transformers/.ts` exporting a + * `TransformerRegistryEntry`, then add it to the array below. + */ + +import type { TransformerRegistryEntry } from "../types.ts"; +import auth0Transformer from "./auth0.ts"; +import authjsTransformer from "./authjs.ts"; +import betterAuthTransformer from "./betterauth.ts"; +import clerkTransformer from "./clerk.ts"; +import firebaseTransformer from "./firebase.ts"; +import supabaseTransformer from "./supabase.ts"; +import workosTransformer from "./workos.ts"; + +export const transformers: TransformerRegistryEntry[] = [ + clerkTransformer, + auth0Transformer, + authjsTransformer, + betterAuthTransformer, + firebaseTransformer, + supabaseTransformer, + workosTransformer, +]; + +/** + * Transformers loaded from a user's `--transformer-file` for this invocation. + * + * Kept beside the built-ins rather than pushed into them, so the shipped list + * is never mutated and `--transformer`'s choices stay exactly the built-in + * keys. One CLI invocation loads at most one, so this holding a single entry is + * the normal case; the array shape just avoids a special case in the lookups. + */ +const customTransformers: TransformerRegistryEntry[] = []; + +export function registerCustomTransformer(entry: TransformerRegistryEntry): void { + customTransformers.push(entry); +} + +/** Test-only: drops anything a previous test registered. */ +export function __resetCustomTransformersForTesting(): void { + customTransformers.length = 0; +} + +/** Built-ins plus whatever `--transformer-file` loaded. */ +export function allTransformers(): TransformerRegistryEntry[] { + return [...transformers, ...customTransformers]; +} + +/** + * The built-in keys, for `--transformer`'s choices and tab-completion. + * + * Deliberately excludes custom transformers: they are selected by path via + * `--transformer-file`, and Commander resolves these choices once at + * registration time, before any file could have been loaded. + */ +export function transformerKeys(): string[] { + return transformers.map((entry) => entry.key); +} + +/** + * Looks up a transformer by key, custom ones included. + * + * @throws Error when no transformer is registered under that key. + */ +export function getTransformer(key: string): TransformerRegistryEntry { + const transformer = allTransformers().find((entry) => entry.key === key); + if (!transformer) { + throw new Error(`Transformer not found for key: ${key}`); + } + return transformer; +} diff --git a/packages/cli-core/src/commands/migrate/transformers/shared.ts b/packages/cli-core/src/commands/migrate/transformers/shared.ts new file mode 100644 index 000000000..c4040230a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/shared.ts @@ -0,0 +1,93 @@ +/** + * Helpers shared by more than one transformer. + * + * Every source platform records verification as a sibling field of the + * identifier, and several ship a single `name` string where Clerk wants a + * first/last pair — so both live here rather than being copied five times. + */ + +/** + * How a platform records that an identifier is verified. + * + * - `boolean` — a true/false flag (Auth0, Better Auth, Firebase). A CSV export + * turns these into the *strings* `"true"`/`"false"`, so `"false"` must not + * be mistaken for a truthy value. + * - `timestamp` — a nullable confirmation time (Auth.js `email_verified`, + * Supabase `email_confirmed_at`). Any real value means verified. + */ +export type VerificationStyle = "boolean" | "timestamp"; + +/** CSV exports write SQL NULL as one of these rather than an empty cell. */ +const NULLISH_STRINGS = new Set(["", "null", "nil", "undefined", "\\n"]); + +export function isVerified(value: unknown, style: VerificationStyle): boolean { + if (value === null || value === undefined) return false; + + if (style === "boolean") { + return value === true || value === 1 || value === "true" || value === "1"; + } + + if (value instanceof Date) return !Number.isNaN(value.getTime()); + if (typeof value === "number") return true; + return typeof value === "string" && !NULLISH_STRINGS.has(value.trim().toLowerCase()); +} + +/** + * Routes an identifier to its verified or unverified field, then drops the + * platform's verification marker. + * + * An unverified identifier must not go on `POST /v1/users`'s primary field: + * Clerk creates those verified, which would silently promote an address the + * source platform never confirmed. + */ +export function routeByVerification( + user: Record, + field: "email" | "phone", + verifiedField: string, + style: VerificationStyle, +): void { + const value = user[field]; + if (value && !isVerified(user[verifiedField], style)) { + user[field === "email" ? "unverifiedEmailAddresses" : "unverifiedPhoneNumbers"] = value; + delete user[field]; + } + delete user[verifiedField]; +} + +/** + * Splits a single display name into `firstName` and `lastName`. + * + * Only splits when there are at least two words — a one-word name would + * otherwise produce a first name with no last name, which several instance + * configurations reject. + */ +export function splitName(user: Record, field = "name"): void { + const name = user[field]; + if (!name || typeof name !== "string") return; + + const parts = name.trim().split(/\s+/); + if (parts.length > 1) { + user.firstName = parts[0]; + user.lastName = parts.slice(1).join(" "); + } + delete user[field]; +} + +/** + * Converts a source timestamp to ISO 8601, leaving it untouched when it does + * not parse so the schema reports it as a validation failure with the original + * value visible in the log. + * + * @param epochMillis - Treat a bare number (or numeric string) as Unix + * milliseconds, which is how Firebase exports timestamps. + */ +export function toIsoDate(value: unknown, epochMillis = false): unknown { + if (value === undefined || value === null || value === "") return value; + + const parsed = + epochMillis && (typeof value === "number" || /^\d+$/.test(String(value))) + ? new Date(Number(value)) + : new Date(String(value)); + + return Number.isNaN(parsed.getTime()) ? value : parsed.toISOString(); +} diff --git a/packages/cli-core/src/commands/migrate/transformers/supabase.ts b/packages/cli-core/src/commands/migrate/transformers/supabase.ts new file mode 100644 index 000000000..1efde4b6a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/supabase.ts @@ -0,0 +1,70 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification, toIsoDate } from "./shared.ts"; + +/** + * Supabase Auth → Clerk transformer. + * + * Works with a `auth.users` export, per + * https://supabase.com/docs/guides/auth/managing-user-data#exporting-users + * + * Supabase records verification as a nullable confirmation timestamp + * (`email_confirmed_at`) rather than a boolean, and stores timestamps in + * PostgreSQL's format (`2024-06-29 20:25:06.126079+00`). + */ + +/** Discord writes display names as `name#0`; the suffix reads as a URL to Clerk. */ +const DISCORD_DISCRIMINATOR = /#\d+$/; + +function stripDiscriminator(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + return value.replace(DISCORD_DISCRIMINATOR, "").trim() || undefined; +} + +const supabaseTransformer = { + key: "supabase", + label: "Supabase", + description: + "Works with a Supabase `auth.users` export. Use --skip-unsupported-providers to drop users whose only social provider is not enabled in Clerk.", + transformer: { + id: "userId", + email: "email", + email_confirmed_at: "emailConfirmedAt", + first_name: "firstName", + last_name: "lastName", + encrypted_password: "password", + phone: "phone", + phone_confirmed_at: "phoneConfirmedAt", + raw_user_meta_data: "publicMetadata", + created_at: "createdAt", + }, + postTransform: (user) => { + user.createdAt = toIsoDate(user.createdAt); + routeByVerification(user, "email", "emailConfirmedAt", "timestamp"); + routeByVerification(user, "phone", "phoneConfirmedAt", "timestamp"); + + // A basic SQL export has no first_name/last_name columns; the name lives in + // user metadata instead, under whichever key the provider happened to use. + if (!user.firstName && user.publicMetadata && typeof user.publicMetadata === "object") { + const meta = user.publicMetadata as Record; + const displayName = stripDiscriminator(meta.display_name ?? meta.first_name ?? meta.name); + if (displayName) { + const parts = displayName.split(/\s+/); + user.firstName = parts[0]; + if (parts.length > 1 && !user.lastName) user.lastName = parts.slice(1).join(" "); + } + } + + for (const field of ["firstName", "lastName"] as const) { + if (typeof user[field] === "string") { + const cleaned = stripDiscriminator(user[field]); + if (cleaned) user[field] = cleaned; + else delete user[field]; + } + } + }, + defaults: { + passwordHasher: "bcrypt" as const, + }, +} satisfies TransformerRegistryEntry; + +export default supabaseTransformer; diff --git a/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts b/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts new file mode 100644 index 000000000..b0005a471 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/transformers.test.ts @@ -0,0 +1,455 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { getLogDir } from "../lib/logger.ts"; +import { loadUsersFromFile, transformUsers } from "../lib/transform.ts"; +import type { FirebaseHashConfig } from "../types.ts"; +import { getTransformer, transformerKeys, transformers } from "./registry.ts"; +import { isVerified } from "./shared.ts"; + +const DATE_TIME = "2026-01-01T00:00:00"; + +const FIREBASE_HASH: FirebaseHashConfig = { + base64_signer_key: "SIGNERKEY==", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, +}; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-transformers-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +/** Writes `records` to a uniquely-named file and loads it through `key`. */ +async function load(key: string, records: unknown, ext = "json", context = {}) { + const file = `${key}-${Math.abs(JSON.stringify(records).length)}-${ext}.${ext}`; + fs.writeFileSync( + path.join(workDir, file), + typeof records === "string" ? records : JSON.stringify(records), + ); + return loadUsersFromFile(file, key, DATE_TIME, { context }); +} + +const one = (key: string, record: Record, context = {}) => + transformUsers([record], key, DATE_TIME, { validate: false, context }).transformedData[0] as + | Record + | undefined; + +describe("registry", () => { + test("registers all seven platforms", () => { + expect(transformerKeys()).toEqual([ + "clerk", + "auth0", + "authjs", + "betterauth", + "firebase", + "supabase", + "workos", + ]); + }); + + test.each([...transformers])("$key maps a source field to userId", (transformer) => { + expect(Object.values(transformer.transformer)).toContain("userId"); + }); + + test.each([...transformers])("$key carries a label and description", (transformer) => { + expect(transformer.label.length).toBeGreaterThan(0); + expect(transformer.description.length).toBeGreaterThan(0); + }); + + test("throws for an unregistered key", () => { + expect(() => getTransformer("okta")).toThrow(/Transformer not found/); + }); +}); + +describe("isVerified", () => { + // A CSV export stringifies everything, so the boolean style must read + // "false" as false. Treating it as truthy would mark unconfirmed addresses + // verified on import — the exact thing the routing exists to prevent. + test.each([ + [true, true], + ["true", true], + [1, true], + ["1", true], + [false, false], + ["false", false], + [0, false], + ["0", false], + ["", false], + [null, false], + [undefined, false], + ])("boolean style: %p -> %p", (value, expected) => { + expect(isVerified(value, "boolean")).toBe(expected); + }); + + // The timestamp style is presence-based: any real confirmation time counts, + // and SQL NULL arrives from a CSV export as one of several spellings. + test.each([ + ["2024-06-29 20:25:06+00", true], + ["2024-01-15T10:30:00.000Z", true], + ["", false], + [" ", false], + ["null", false], + ["NULL", false], + ["\\N", false], + [null, false], + [undefined, false], + ])("timestamp style: %p -> %p", (value, expected) => { + expect(isVerified(value, "timestamp")).toBe(expected); + }); +}); + +describe("auth0", () => { + const base = { user_id: "auth0|abc", email: "a@x.dev", passwordHash: "$2b$10$hash" }; + + test("maps identity, name and metadata onto the Clerk schema", async () => { + const { users } = await load("auth0", [ + { ...base, email_verified: true, given_name: "Ada", family_name: "Lovelace" }, + ]); + expect(users[0]).toMatchObject({ + userId: "auth0|abc", + email: "a@x.dev", + firstName: "Ada", + lastName: "Lovelace", + password: "$2b$10$hash", + passwordHasher: "bcrypt", + }); + }); + + test.each([ + [true, "email", undefined], + [false, undefined, "a@x.dev"], + [undefined, undefined, "a@x.dev"], + ])("email_verified=%p routes the address correctly", (verified, kept, unverified) => { + const user = one("auth0", { ...base, email_verified: verified }); + expect(user?.email).toBe(kept ? "a@x.dev" : undefined); + expect(user?.unverifiedEmailAddresses).toBe(unverified); + }); + + test("routes an unverified phone away from the primary field", () => { + const user = one("auth0", { ...base, phone_number: "+15555550100", phone_verified: false }); + expect(user?.phone).toBeUndefined(); + expect(user?.unverifiedPhoneNumbers).toBe("+15555550100"); + }); + + test("drops the platform's verification markers", () => { + const user = one("auth0", { ...base, email_verified: true, phone_verified: true }); + expect("emailVerified" in (user ?? {})).toBe(false); + expect("phoneVerified" in (user ?? {})).toBe(false); + }); + + test("keeps user_metadata public and app_metadata private", async () => { + const { users } = await load("auth0", [ + { + ...base, + email_verified: true, + user_metadata: { theme: "dark" }, + app_metadata: { plan: "pro" }, + }, + ]); + expect(users[0]?.publicMetadata).toEqual({ theme: "dark" }); + expect(users[0]?.privateMetadata).toEqual({ plan: "pro" }); + }); +}); + +describe("workos", () => { + const base = { id: "user_01ABC", email: "a@x.dev" }; + + test("maps identity, name and metadata onto the Clerk schema", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, first_name: "Ada", last_name: "Lovelace" }, + ]); + expect(users[0]).toMatchObject({ + userId: "user_01ABC", + email: "a@x.dev", + firstName: "Ada", + lastName: "Lovelace", + }); + }); + + test.each([ + [true, "email", undefined], + [false, undefined, "a@x.dev"], + [undefined, undefined, "a@x.dev"], + ])("email_verified=%p routes the address correctly", (verified, kept, unverified) => { + const user = one("workos", { ...base, email_verified: verified }); + expect(user?.email).toBe(kept ? "a@x.dev" : undefined); + expect(user?.unverifiedEmailAddresses).toBe(unverified); + }); + + test("keeps metadata public", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, metadata: { plan: "pro" } }, + ]); + expect(users[0]?.publicMetadata).toEqual({ plan: "pro" }); + }); + + // No other transformer omits it. WorkOS never returns a digest, so naming a + // hasher would imply a password column that cannot exist. + test("names no password hasher, because WorkOS returns no hashes", () => { + expect(getTransformer("workos").defaults).toBeUndefined(); + }); + + // The export carries these so whoever runs the migration can see who used + // social sign-in; Clerk's import has no field for them. + test("drops OAuth identities carried through from the export", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, identities: [{ provider: "GoogleOAuth", idp_id: "1" }] }, + ]); + expect(users[0]?.userId).toBe("user_01ABC"); + expect("identities" in (users[0] ?? {})).toBe(false); + }); +}); + +describe("authjs", () => { + const base = { id: "cuid1", email: "a@x.dev" }; + + test("treats a confirmation timestamp as verified", () => { + const user = one("authjs", { ...base, email_verified: "2024-01-15T10:30:00.000Z" }); + expect(user?.email).toBe("a@x.dev"); + expect(user?.unverifiedEmailAddresses).toBeUndefined(); + }); + + test.each([[null], [""], [undefined]])("treats email_verified=%p as unverified", (value) => { + const user = one("authjs", { ...base, email_verified: value }); + expect(user?.unverifiedEmailAddresses).toBe("a@x.dev"); + }); + + test.each([ + ["Jane Doe", "Jane", "Doe"], + ["Mary Jane Watson", "Mary", "Jane Watson"], + [" Ada Lovelace ", "Ada", "Lovelace"], + ])("splits %p into %p / %p", (name, firstName, lastName) => { + const user = one("authjs", { ...base, name }); + expect(user?.firstName).toBe(firstName); + expect(user?.lastName).toBe(lastName); + }); + + test("leaves a single-word name unsplit rather than inventing a last name", () => { + const user = one("authjs", { ...base, name: "Prince" }); + expect(user?.firstName).toBeUndefined(); + expect(user?.lastName).toBeUndefined(); + expect("name" in (user ?? {})).toBe(false); + }); + + test("imports without a password, since Auth.js core is passwordless", async () => { + const { users } = await load("authjs", [{ ...base, email_verified: "2024-01-01" }]); + expect(users[0]?.password).toBeUndefined(); + expect(users[0]?.passwordHasher).toBeUndefined(); + }); +}); + +describe("betterauth", () => { + const base = { user_id: "ba1", email: "a@x.dev", email_verified: true }; + + test("maps the credential hash and defaults the hasher to bcrypt", async () => { + const { users } = await load("betterauth", [{ ...base, password_hash: "$2a$10$hash" }]); + expect(users[0]).toMatchObject({ password: "$2a$10$hash", passwordHasher: "bcrypt" }); + }); + + test("routes an unverified phone", () => { + const user = one("betterauth", { + ...base, + phone_number: "+15555550100", + phone_number_verified: false, + }); + expect(user?.unverifiedPhoneNumbers).toBe("+15555550100"); + }); + + test.each([ + [true, true], + [false, undefined], + [undefined, undefined], + ])("banned=%p is carried through as %p", (banned, expected) => { + expect(one("betterauth", { ...base, banned })?.banned).toBe(expected as boolean | undefined); + }); + + test("drops plugin-only columns during validation", async () => { + const { users } = await load("betterauth", [ + { ...base, role: "admin", display_username: "ADA", two_factor_enabled: true }, + ]); + const user = users[0] as Record; + expect("role" in user).toBe(false); + expect("display_username" in user).toBe(false); + expect("two_factor_enabled" in user).toBe(false); + }); +}); + +describe("firebase", () => { + const base = { localId: "fb1", email: "a@x.dev", emailVerified: true }; + const withHash = { ...base, passwordHash: "SGFzaA==", salt: "U2FsdA==" }; + + test("builds the scrypt digest Clerk expects, parameters inline", async () => { + const { users } = await load("firebase", { users: [withHash] }, "json", { + firebaseHashConfig: FIREBASE_HASH, + }); + expect(users[0]?.password).toBe("SGFzaA==$U2FsdA==$SIGNERKEY==$Bw==$8$14"); + expect(users[0]?.passwordHasher).toBe("scrypt_firebase"); + }); + + test("refuses to import hashes without the project's hash parameters", async () => { + await expect(load("firebase", { users: [withHash] })).rejects.toThrow( + /Firebase password hashes/, + ); + }); + + test("imports a passwordless export with no hash parameters at all", async () => { + const { users } = await load("firebase", { users: [base] }); + expect(users).toHaveLength(1); + expect(users[0]?.password).toBeUndefined(); + }); + + test("unwraps the { users: [...] } export shape", async () => { + const { users } = await load("firebase", { users: [base, { ...base, localId: "fb2" }] }); + expect(users.map((u) => u.userId)).toEqual(["fb1", "fb2"]); + }); + + test("accepts a bare array too", async () => { + const { users } = await load("firebase", [base]); + expect(users).toHaveLength(1); + }); + + test("rejects a JSON export that is neither", async () => { + await expect(load("firebase", { records: [] })).rejects.toThrow(CliError); + }); + + test("prepends headers to a headerless CSV export", async () => { + const csv = "fb9,a@x.dev,true,,,Ada Lovelace,,,,,,,,,,,,,,,,,,1704067200000,,,,,\n"; + const { users } = await load("firebase", csv, "csv"); + expect(users[0]).toMatchObject({ userId: "fb9", email: "a@x.dev", firstName: "Ada" }); + }); + + test.each([ + ["1704067200000", "2024-01-01T00:00:00.000Z"], + [1704067200000, "2024-01-01T00:00:00.000Z"], + ])("converts the Unix-millisecond createdAt %p", (createdAt, expected) => { + expect(one("firebase", { ...base, createdAt })?.createdAt).toBe(expected); + }); + + test.each([ + [true, true], + ["true", true], + [false, false], + ["false", false], + ])("emailVerified=%p keeps the address primary: %p", (emailVerified, verified) => { + const user = one("firebase", { ...base, emailVerified }); + expect(user?.email !== undefined).toBe(verified); + }); +}); + +describe("supabase", () => { + const base = { id: "sb1", email: "a@x.dev", email_confirmed_at: "2024-06-29 20:25:06.126079+00" }; + + test("maps the bcrypt password and converts the PostgreSQL timestamp", async () => { + const { users } = await load("supabase", [ + { ...base, encrypted_password: "$2b$10$hash", created_at: "2024-06-29 20:25:06.126079+00" }, + ]); + expect(users[0]).toMatchObject({ + password: "$2b$10$hash", + passwordHasher: "bcrypt", + createdAt: "2024-06-29T20:25:06.126Z", + }); + }); + + test.each([ + ["2024-06-29 20:25:06+00", true], + [null, false], + ["", false], + ])("email_confirmed_at=%p means verified: %p", (confirmedAt, verified) => { + const user = one("supabase", { ...base, email_confirmed_at: confirmedAt }); + expect(user?.email !== undefined).toBe(verified); + }); + + test("falls back to user metadata for a missing first name", () => { + const user = one("supabase", { + ...base, + raw_user_meta_data: { display_name: "Ada Lovelace" }, + }); + expect(user?.firstName).toBe("Ada"); + expect(user?.lastName).toBe("Lovelace"); + }); + + test("prefers explicit name columns over metadata", () => { + const user = one("supabase", { + ...base, + first_name: "Grace", + raw_user_meta_data: { display_name: "Ada Lovelace" }, + }); + expect(user?.firstName).toBe("Grace"); + }); + + test.each([ + ["ada#0", "ada"], + ["ada#1234", "ada"], + ])("strips the Discord discriminator from %p", (displayName, expected) => { + const user = one("supabase", { ...base, raw_user_meta_data: { display_name: displayName } }); + expect(user?.firstName).toBe(expected); + }); + + test("drops a name that was nothing but a discriminator", () => { + const user = one("supabase", { ...base, first_name: "#0" }); + expect(user?.firstName).toBeUndefined(); + }); +}); + +describe("invalid records", () => { + const INVALID: [string, Record][] = [ + ["auth0", { user_id: "a1" }], + ["authjs", { id: "a2" }], + ["betterauth", { user_id: "a3" }], + ["firebase", { localId: "a4" }], + ["supabase", { id: "a5" }], + ]; + + test.each(INVALID)( + "%s logs a user with no identifier instead of crashing", + async (key, record) => { + fs.rmSync(getLogDir(), { recursive: true, force: true }); + + const { users, validationFailed } = await load(key, [ + record, + { ...record, ...identifierFor(key) }, + ]); + + expect(validationFailed).toBe(1); + expect(users).toHaveLength(1); + + const logged = fs + .readdirSync(getLogDir()) + .flatMap((name) => + fs.readFileSync(path.join(getLogDir(), name), "utf-8").trim().split("\n"), + ) + .map((line) => JSON.parse(line) as Record); + expect(logged.some((entry) => entry.status === "fail")).toBe(true); + }, + ); + + test.each(INVALID)("%s logs a malformed email rather than sending it", async (key, record) => { + const { users, validationFailed } = await load(key, [ + { ...record, ...identifierFor(key, "not-an-email") }, + ]); + expect(validationFailed).toBe(1); + expect(users).toHaveLength(0); + }); +}); + +/** The per-platform source field that becomes a Clerk identifier. */ +function identifierFor(key: string, email = "ok@x.dev"): Record { + if (key === "auth0") return { email, email_verified: true }; + if (key === "authjs") return { email, email_verified: "2024-01-01" }; + if (key === "betterauth") return { email, email_verified: true }; + if (key === "firebase") return { email, emailVerified: true }; + return { email, email_confirmed_at: "2024-01-01 00:00:00+00" }; +} diff --git a/packages/cli-core/src/commands/migrate/transformers/workos.ts b/packages/cli-core/src/commands/migrate/transformers/workos.ts new file mode 100644 index 000000000..9e5b8e349 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/transformers/workos.ts @@ -0,0 +1,39 @@ +import type { TransformerRegistryEntry } from "../types.ts"; +import { routeByVerification } from "./shared.ts"; + +/** + * WorkOS → Clerk transformer. + * + * Works with WorkOS's User Management API. `id` is a `user_…` string and is + * carried through as the Clerk user's `external_id`. + * + * **There is no `passwordHasher` default here, and that is deliberate.** Every + * other transformer names the hasher its platform ships so a digest can be + * verified; WorkOS returns no digest to verify. It accepts password hashes on + * import and never gives them back, and its TOTP secrets are returned on enrol + * only — so a WorkOS migration moves identities, not credentials. Naming a + * hasher here would imply a password column that cannot exist. + * + * WorkOS has no phone number and no username, which is why the map is short: + * those fields have nothing to come from. + */ +const workosTransformer = { + key: "workos", + label: "WorkOS", + description: + "Works with WorkOS's User Management API. WorkOS returns no password hashes, so imported users sign in by reset or SSO.", + transformer: { + id: "userId", + email: "email", + email_verified: "emailVerified", + first_name: "firstName", + last_name: "lastName", + metadata: "publicMetadata", + created_at: "createdAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "boolean"); + }, +} satisfies TransformerRegistryEntry; + +export default workosTransformer; diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts new file mode 100644 index 000000000..904267037 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -0,0 +1,179 @@ +/** + * Shared types for `clerk migrate`. + * + * Ported from the standalone migration-tool's `src/types.ts`. The Clerk API + * error shape is declared locally rather than imported from `@clerk/types`, + * because this command family talks to BAPI through `lib/bapi.ts` instead of + * `@clerk/backend`. + */ + +import type * as z from "zod"; +import type { userSchema } from "./validator.ts"; + +/** + * Password hashing algorithms Clerk can verify on import. + * + * When migrating users with existing passwords, the source platform's hasher + * must be named so Clerk can validate the digest instead of rejecting it. + */ +export const PASSWORD_HASHERS = [ + "argon2i", + "argon2id", + "awscognito", + "bcrypt", + "bcrypt_peppered", + "bcrypt_sha256_django", + "hmac_sha256_utf16_b64", + "md5", + "md5_salted", + "pbkdf2_sha1", + "pbkdf2_sha256", + "pbkdf2_sha256_django", + "pbkdf2_sha512", + "pbkdf2_sha512_hex", + "scrypt_firebase", + "scrypt_werkzeug", + "sha256", + "sha256_salted", + "md5_phpass", + "ldap_ssha", + "sha512_symfony", +] as const; + +/** A user that has passed schema validation and is ready to import. */ +export type User = z.infer; + +/** Union of all registered transformer keys (e.g. `"clerk"`). */ +export type TransformerKey = string; + +/** + * One error entry as returned in a Clerk API error response body. + * + * Local mirror of `@clerk/types`' `ClerkAPIError` covering only the fields the + * migration logs read. + */ +export type ClerkApiError = { + code: string; + message: string; + longMessage?: string; +}; + +/** A failed user-creation attempt, as handed to the error logger. */ +export type ErrorPayload = { + userId: string; + status: string; + errors: ClerkApiError[]; +}; + +/** A user that failed schema validation before any API call was made. */ +export type ValidationErrorPayload = { + error: string; + path: (string | number)[]; + userId: string; + row: number; +}; + +/** A formatted error line as written to the NDJSON log. */ +export type ErrorLog = { + type: string; + userId: string; + status: string; + error: string | undefined; +}; + +/** One import attempt as written to the NDJSON log. */ +export type ImportLogEntry = { + userId: string; + status: "success" | "error"; + clerkUserId?: string; + error?: string; + code?: string; +}; + +/** One exported user as written to the NDJSON log. */ +export type ExportLogEntry = { + /** The source platform's ID for this user. */ + userId: string; + status: "success" | "error"; + error?: string; +}; + +/** One deletion attempt as written to the NDJSON log. */ +export type DeleteLogEntry = { + /** The source platform's ID — the Clerk user's `external_id`. */ + userId: string; + clerkUserId?: string; + status: "success" | "error"; + error?: string; + code?: string; +}; + +/** Totals for a completed import run. */ +export type ImportSummary = { + totalProcessed: number; + successful: number; + failed: number; + validationFailed: number; + errorBreakdown: Map; +}; + +/** + * Firebase's scrypt parameters, needed to rebuild a password hash Clerk can + * verify. + * + * Found in the Firebase console under Authentication → Users → (⋮) → Password + * hash parameters. All four are required together; a partial set produces a + * digest that silently fails every sign-in. + */ +export type FirebaseHashConfig = { + base64_signer_key: string; + base64_salt_separator: string; + rounds: number; + mem_cost: number; +}; + +/** + * Per-run values a transformer may need but cannot read from the user record. + * + * Passed to `postTransform` rather than held in module state so two runs in one + * process — or two test files — cannot see each other's configuration. + */ +export type TransformContext = { + firebaseHashConfig?: FirebaseHashConfig; +}; + +/** + * Result of a transformer's `preTransform` hook. + * + * @property filePath - Path to read from; may differ from the input (e.g. a + * temp file with generated CSV headers). + * @property data - Users already extracted from a wrapper object, when the + * source format nests them. + */ +export type PreTransformResult = { + filePath: string; + data?: Record[]; +}; + +/** + * A platform transformer: how to get from one source export shape to Clerk's + * import shape. + * + * @property transformer - Source field path → Clerk field name. + * @property defaults - Values merged into every user from this platform. + * @property preTransform - Runs before field mapping. + * @property postTransform - Mutates a user after field mapping, given the + * run's {@link TransformContext}. + */ +export type TransformerRegistryEntry = { + key: string; + label: string; + description: string; + transformer: Record; + defaults?: Record; + preTransform?: ( + filePath: string, + fileType: string, + ) => PreTransformResult | Promise; + postTransform?: (user: Record, context: TransformContext) => void; +}; diff --git a/packages/cli-core/src/commands/migrate/validator.test.ts b/packages/cli-core/src/commands/migrate/validator.test.ts new file mode 100644 index 000000000..d398a4dc9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/validator.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, test } from "bun:test"; +import { PASSWORD_HASHERS } from "./types.ts"; +import { userSchema } from "./validator.ts"; + +const base = { userId: "user_1", email: "a@example.com" }; + +describe("userSchema identifiers", () => { + const IDENTIFIER_CASES = [ + ["email", { email: "a@example.com" }, true], + ["emailAddresses array", { emailAddresses: ["a@example.com"] }, true], + ["unverified email", { unverifiedEmailAddresses: ["a@example.com"] }, true], + ["phone", { phone: "+15555550100" }, true], + ["unverified phone", { unverifiedPhoneNumbers: ["+15555550100"] }, true], + ["username", { username: "alice" }, true], + ["nothing", {}, false], + ["empty email array", { email: [] }, false], + ["empty username", { username: "" }, false], + ] as const; + + test.each([...IDENTIFIER_CASES])( + "accepts a user identified by %s: %p -> %p", + (_label, fields, ok) => { + expect(userSchema.safeParse({ userId: "user_1", ...fields }).success).toBe(ok); + }, + ); + + test("reports the identifier failure against the email path", () => { + const result = userSchema.safeParse({ userId: "user_1" }); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues[0]?.path).toEqual(["email"]); + }); +}); + +describe("userSchema passwords", () => { + test("rejects a password without a hasher", () => { + const result = userSchema.safeParse({ ...base, password: "digest" }); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues[0]?.path).toEqual(["passwordHasher"]); + }); + + test("accepts a password with a valid hasher", () => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: "bcrypt" }).success, + ).toBe(true); + }); + + test("rejects an unknown hasher", () => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: "rot13" }).success, + ).toBe(false); + }); + + test.each([...PASSWORD_HASHERS])("accepts the %s hasher", (hasher) => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: hasher }).success, + ).toBe(true); + }); +}); + +describe("userSchema field types", () => { + const FIELD_CASES = [ + ["valid email", { email: "a@example.com" }, true], + ["malformed email", { email: "not-an-email" }, false], + ["email array with one bad entry", { email: ["a@example.com", "nope"] }, false], + ["userId missing", { userId: undefined }, false], + ["valid createdAt", { createdAt: "2024-01-01T00:00:00Z" }, true], + ["unparseable createdAt", { createdAt: "yesterday" }, false], + ["integer org limit", { createOrganizationsLimit: 3 }, true], + ["fractional org limit", { createOrganizationsLimit: 1.5 }, false], + ["metadata object", { publicMetadata: { plan: "pro" } }, true], + ["metadata string", { publicMetadata: "pro" }, false], + ["backupCodes array", { backupCodes: ["a", "b"] }, true], + ] as const; + + test.each([...FIELD_CASES])("%s -> %p", (_label, fields, ok) => { + expect(userSchema.safeParse({ ...base, ...fields }).success).toBe(ok); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/validator.ts b/packages/cli-core/src/commands/migrate/validator.ts new file mode 100644 index 000000000..2ebc31166 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/validator.ts @@ -0,0 +1,99 @@ +/** + * Zod schema every user is validated against before it reaches BAPI. + * + * Ported from the standalone migration-tool's `src/migrate/validator.ts`. + * + * ============================================================================ + * ONLY EDIT THIS IF YOU ARE ADDING A NEW FIELD. + * Adding support for a new source platform means adding a transformer, not + * touching the schema. + * ============================================================================ + */ + +import * as z from "zod"; +import { PASSWORD_HASHERS } from "./types.ts"; + +const metadataSchema = z.record(z.string(), z.unknown()); + +const dateStringSchema = z.string().refine((value) => !Number.isNaN(new Date(value).getTime()), { + message: "Expected a valid date string", +}); + +/** Zod enum of the password hashers Clerk accepts on import. */ +export const passwordHasherEnum = z.enum(PASSWORD_HASHERS); + +/** + * Validates user data before sending it to Clerk. + * + * Everything is optional except: + * - `userId`, required for tracking, logging and `--resume-after` + * - `passwordHasher`, required whenever `password` is present + * - at least one identifier (email, phone or username) + * + * Identifier fields accept either a single value or an array. + */ +export const userSchema = z + .object({ + userId: z.string(), + // Email fields + email: z.union([z.email(), z.array(z.email())]).optional(), + emailAddresses: z.union([z.email(), z.array(z.email())]).optional(), + unverifiedEmailAddresses: z.union([z.email(), z.array(z.email())]).optional(), + // Phone fields + phone: z.union([z.string(), z.array(z.string())]).optional(), + phoneNumbers: z.union([z.string(), z.array(z.string())]).optional(), + unverifiedPhoneNumbers: z.union([z.string(), z.array(z.string())]).optional(), + // User info + username: z.string().optional(), + firstName: z.string().optional(), + lastName: z.string().optional(), + // Password + password: z.string().optional(), + passwordHasher: passwordHasherEnum.optional(), + // 2FA + totpSecret: z.string().optional(), + backupCodesEnabled: z.boolean().optional(), + backupCodes: z.array(z.string()).optional(), + // Metadata + unsafeMetadata: metadataSchema.optional(), + publicMetadata: metadataSchema.optional(), + privateMetadata: metadataSchema.optional(), + // Additional Clerk API fields + banned: z.boolean().optional(), + bypassClientTrust: z.boolean().optional(), + createOrganizationEnabled: z.boolean().optional(), + createOrganizationsLimit: z.number().int().optional(), + createdAt: dateStringSchema.optional(), + deleteSelfEnabled: z.boolean().optional(), + legalAcceptedAt: dateStringSchema.optional(), + skipLegalChecks: z.boolean().optional(), + skipPasswordChecks: z.boolean().optional(), + }) + .refine((data) => !data.password || data.passwordHasher, { + message: "passwordHasher is required when password is provided", + path: ["passwordHasher"], + }) + .refine( + (data) => { + const hasValue = (field: unknown): boolean => { + if (!field) return false; + if (typeof field === "string") return field.length > 0; + if (Array.isArray(field)) return field.length > 0; + return false; + }; + return ( + hasValue(data.email) || + hasValue(data.emailAddresses) || + hasValue(data.unverifiedEmailAddresses) || + hasValue(data.phone) || + hasValue(data.phoneNumbers) || + hasValue(data.unverifiedPhoneNumbers) || + hasValue(data.username) + ); + }, + { + message: + "User must have at least one identifier (email, phone, unverified email, unverified phone, or username)", + path: ["email"], + }, + ); diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts new file mode 100644 index 000000000..cee841da4 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -0,0 +1,268 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +useCaptureLog(); + +type Prompt = { message: string; default?: string; validate?: (v?: string) => string | undefined }; +type SelectPrompt = Prompt & { choices: { name: string; value: string }[] }; + +// Registered at file top, before the wizard (or anything it imports) loads. +// This file is the only consumer of the mocked prompt modules. +const mockSelect = mock(async (_config: SelectPrompt) => undefined as unknown); +const mockText = mock(async (_config: Prompt) => "" as unknown); + +mock.module("../../lib/listage.ts", () => ({ + ...listageStubs, + select: (config: SelectPrompt) => mockSelect(config), +})); + +mock.module("../../lib/prompts.ts", () => ({ + confirm: async () => true, + text: (config: Prompt) => mockText(config), + password: async () => "", + editor: async () => "{}", +})); + +const { runWizard, throwAgentFlagsRequired } = await import("./wizard.ts"); +const { saveSettings } = await import("./lib/settings.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); + +let workDir: string; +let configDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-config-")); + _setConfigDir(configDir); + process.chdir(workDir); + fs.writeFileSync(path.join(workDir, "users.json"), "[]"); + fs.writeFileSync(path.join(workDir, "other.csv"), ""); +}); + +afterAll(() => { + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + mockSelect.mockReset(); + mockText.mockReset(); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); +}); + +/** The config object the wizard passed to its Nth `text`/`select` prompt. */ +const textCall = (index: number): Prompt | undefined => mockText.mock.calls[index]?.[0]; +const selectCall = (index: number): SelectPrompt | undefined => mockSelect.mock.calls[index]?.[0]; + +describe("transformer picker", () => { + test("is built from the registry, so every platform appears", async () => { + mockSelect.mockResolvedValue("auth0"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.choices.map((choice) => choice.value)).toEqual([ + "clerk", + "auth0", + "authjs", + "betterauth", + "firebase", + "supabase", + "workos", + ]); + }); + + test("labels each choice with the transformer's display name", async () => { + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.choices.map((choice) => choice.name)).toContain("Better Auth"); + }); + + test("is skipped when --transformer was already passed", async () => { + mockText.mockResolvedValue("users.json"); + + const result = await runWizard({ transformer: "clerk" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(result.transformer).toBe("clerk"); + }); +}); + +describe("defaults from the previous run", () => { + test("pre-selects the last transformer and pre-fills the last file", async () => { + await saveSettings({ transformer: "supabase", file: "other.csv" }); + mockSelect.mockResolvedValue("supabase"); + mockText.mockResolvedValue("other.csv"); + + await runWizard({}); + + expect(selectCall(0)?.default).toBe("supabase"); + expect(textCall(0)?.default).toBe("other.csv"); + }); + + test("offers no default when nothing has been saved", async () => { + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.default).toBeUndefined(); + expect(textCall(0)?.default).toBeUndefined(); + }); + + // A saved key from a build that has since dropped that transformer would + // otherwise pre-select a value the picker cannot offer. + test("ignores a saved transformer that is no longer registered", async () => { + await saveSettings({ transformer: "okta" }); + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(selectCall(0)?.default).toBeUndefined(); + }); +}); + +describe("file prompt validation", () => { + const validate = async () => { + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("users.json"); + await runWizard({}); + return textCall(0)?.validate; + }; + + test.each([ + ["users.json", undefined], + ["other.csv", undefined], + ])("accepts %s", async (file, expected) => { + expect((await validate())?.(file)).toBe(expected as undefined); + }); + + test("rejects an empty answer", async () => { + expect((await validate())?.("")).toMatch(/required/); + }); + + test("rejects a file that does not exist", async () => { + expect((await validate())?.("missing.json")).toMatch(/File not found/); + }); + + test("rejects an unsupported extension", async () => { + fs.writeFileSync(path.join(workDir, "notes.txt"), ""); + expect((await validate())?.("notes.txt")).toMatch(/\.json or \.csv/); + }); +}); + +describe("firebase hash parameters", () => { + test("are asked for when the firebase transformer is picked", async () => { + mockSelect.mockResolvedValue("firebase"); + mockText + .mockResolvedValueOnce("users.json") + .mockResolvedValueOnce("SIGNER") + .mockResolvedValueOnce("Bw==") + .mockResolvedValueOnce("8") + .mockResolvedValueOnce("14"); + + const result = await runWizard({}); + + expect(result.firebaseHashConfig).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + // Pressing enter through the signer key is how a user says "this export has + // no passwords" — the remaining three would be meaningless without it. + test("stop being asked when the signer key is left blank", async () => { + mockSelect.mockResolvedValue("firebase"); + mockText.mockResolvedValueOnce("users.json").mockResolvedValueOnce(" "); + + const result = await runWizard({}); + + expect(result.firebaseHashConfig).toBeUndefined(); + expect(mockText).toHaveBeenCalledTimes(2); + }); + + // The signer key is a Firebase secret, so it is never written to disk and so + // there is nothing to offer back. A repeat run passes it as a flag or env var. + test("are never pre-filled, because they are not saved", async () => { + mockSelect.mockResolvedValue("firebase"); + mockText + .mockResolvedValueOnce("users.json") + .mockResolvedValueOnce("SIGNER") + .mockResolvedValueOnce("Bw==") + .mockResolvedValueOnce("8") + .mockResolvedValueOnce("14"); + + await runWizard({}); + + expect(textCall(1)?.default).toBeUndefined(); + expect(textCall(3)?.default).toBeUndefined(); + }); + + test("are not asked for on a non-firebase transformer", async () => { + mockSelect.mockResolvedValue("auth0"); + mockText.mockResolvedValue("users.json"); + + await runWizard({}); + + expect(mockText).toHaveBeenCalledTimes(1); + }); + + test("are not asked for when the flags already supplied them", async () => { + mockSelect.mockResolvedValue("firebase"); + mockText.mockResolvedValue("users.json"); + + const config = { + base64_signer_key: "FLAG", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }; + const result = await runWizard({ firebaseHashConfig: config }); + + expect(mockText).toHaveBeenCalledTimes(1); + expect(result.firebaseHashConfig).toEqual(config); + }); + + test.each([["0"], ["-1"], ["1.5"], ["many"]])("rejects %p as a rounds value", async (value) => { + mockSelect.mockResolvedValue("firebase"); + mockText + .mockResolvedValueOnce("users.json") + .mockResolvedValueOnce("SIGNER") + .mockResolvedValueOnce("Bw==") + .mockResolvedValueOnce("8") + .mockResolvedValueOnce("14"); + + await runWizard({}); + + expect(textCall(3)?.validate?.(value)).toMatch(/positive whole number/); + }); +}); + +describe("throwAgentFlagsRequired", () => { + test.each([ + [{ transformer: true, file: true }, /--transformer and --file /], + [{ transformer: true, file: false }, /--transformer \./], + [{ transformer: false, file: true }, /--file \./], + ])("names only the flags that are missing (%p)", (missing, expected) => { + expect(() => throwAgentFlagsRequired(missing)).toThrow(expected); + }); + + test("says why it cannot prompt", () => { + expect(() => throwAgentFlagsRequired({ transformer: true, file: true })).toThrow( + /cannot prompt in agent mode/, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts new file mode 100644 index 000000000..122173046 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -0,0 +1,167 @@ +/** + * The interactive path behind a bare `clerk migrate import`. + * + * Ported from the standalone migration-tool's `src/migrate/cli.ts` interactive + * flow. The platform and file are pre-filled from the previous run, so a repeat + * migration is mostly pressing enter. Firebase's hash parameters are not: the + * signer key is a secret, and the CLI does not keep those. + * + * Agent mode never reaches here — `run` raises a usage error naming the flags + * instead, because an agent cannot answer a prompt. + */ + +import { throwUsageError } from "../../lib/errors.ts"; +import { select } from "../../lib/listage.ts"; +import { log } from "../../lib/log.ts"; +import { text } from "../../lib/prompts.ts"; +import { resolveFirebaseHashConfig, type FirebaseHashFlags } from "./lib/firebase-hash.ts"; +import { loadSettings } from "./lib/settings.ts"; +import { fileExists, getFileType } from "./lib/transform.ts"; +import { transformers } from "./transformers/registry.ts"; +import type { FirebaseHashConfig } from "./types.ts"; + +export type WizardResult = { + transformer: string; + file: string; + firebaseHashConfig?: FirebaseHashConfig; +}; + +/** Trims a description down to a single readable hint line. */ +function hint(description: string): string { + const firstSentence = description.split(". ")[0] ?? description; + return firstSentence.length > 96 ? `${firstSentence.slice(0, 93)}...` : firstSentence; +} + +async function pickTransformer(defaultKey: string | undefined): Promise { + // Built from the registry, so a new platform appears here with no second + // place to update. + return select({ + message: "Which platform are you migrating from?", + choices: transformers.map((entry) => ({ + name: entry.label, + value: entry.key, + description: hint(entry.description), + })), + default: defaultKey && transformers.some((t) => t.key === defaultKey) ? defaultKey : undefined, + }); +} + +async function askFile(defaultFile: string | undefined): Promise { + return text({ + message: "Path to the exported user file (JSON or CSV)", + default: defaultFile, + validate: (value) => { + const file = value?.trim(); + if (!file) return "A file path is required"; + if (!fileExists(file)) return `File not found: ${file}`; + if (!getFileType(file)) return "Provide a .json or .csv file"; + return undefined; + }, + }); +} + +/** + * Collects Firebase's four hash parameters. + * + * Asked as a set because a partial set produces a digest that verifies against + * nothing. Pressing enter through all four leaves the config unset, which is + * correct for an export with no password hashes. + */ +async function askFirebaseHashConfig(): Promise { + log.info( + "Firebase password hashes need the project's hash parameters. Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + ); + log.info( + "Set CLERK_FIREBASE_SIGNER_KEY, CLERK_FIREBASE_SALT_SEPARATOR, CLERK_FIREBASE_ROUNDS and CLERK_FIREBASE_MEM_COST to skip these prompts on the next run.", + ); + + const signerKey = ( + await text({ + message: "base64 signer key (leave blank if this export has no passwords)", + }) + ).trim(); + if (!signerKey) return undefined; + + const saltSeparator = ( + await text({ + message: "base64 salt separator", + validate: (value) => (value?.trim() ? undefined : "Required alongside the signer key"), + }) + ).trim(); + + return { + base64_signer_key: signerKey, + base64_salt_separator: saltSeparator, + rounds: await askNumber("rounds"), + mem_cost: await askNumber("mem cost"), + }; +} + +async function askNumber(label: string): Promise { + const answer = await text({ + message: label, + validate: (value) => { + const parsed = Number(value?.trim()); + return Number.isInteger(parsed) && parsed > 0 ? undefined : "Enter a positive whole number"; + }, + }); + return Number(answer.trim()); +} + +/** + * Fills in whichever of transformer and file were not passed as flags. + * + * @param provided - Flags the caller already supplied; those are not asked for. + */ +export async function runWizard( + provided: { + transformer?: string; + file?: string; + firebaseHashConfig?: FirebaseHashConfig; + } & FirebaseHashFlags, +): Promise { + const saved = await loadSettings(); + + const transformer = provided.transformer ?? (await pickTransformer(saved.transformer)); + const file = provided.file ?? (await askFile(saved.file)); + + let firebaseHashConfig = provided.firebaseHashConfig; + if (transformer === "firebase" && !firebaseHashConfig) { + // Looked up here rather than before the picker: until the platform is + // chosen there is no reason to read Firebase's variables at all, and a + // migration from anywhere else must not see them. + firebaseHashConfig = await resolveFirebaseHashConfig(provided, "firebase"); + } + if (transformer === "firebase" && !firebaseHashConfig) { + // Prompted, never prefilled: the signer key is a secret the CLI does not + // keep, so there is nothing to offer back. + firebaseHashConfig = await askFirebaseHashConfig(); + } + + return { transformer, file, ...(firebaseHashConfig ? { firebaseHashConfig } : {}) }; +} + +/** + * The error an agent gets instead of a prompt. + * + * Names exactly the flags that are missing, so the caller can retry without + * guessing which of the two it forgot. + */ +export function throwAgentFlagsRequired(missing: { transformer: boolean; file: boolean }): never { + const flags = [ + missing.transformer ? "--transformer " : undefined, + missing.file ? "--file " : undefined, + ].filter(Boolean); + + throwUsageError( + `\`clerk migrate import\` is interactive and cannot prompt in agent mode. Pass ${flags.join(" and ")}.`, + undefined, + undefined, + [ + { + command: `clerk migrate import -y --transformer ${transformers[0]?.key ?? "clerk"} --file users.json`, + description: "Run non-interactively", + }, + ], + ); +} diff --git a/packages/cli-core/src/lib/config.test.ts b/packages/cli-core/src/lib/config.test.ts index 03d2f4eae..faeeeb806 100644 --- a/packages/cli-core/src/lib/config.test.ts +++ b/packages/cli-core/src/lib/config.test.ts @@ -11,6 +11,9 @@ const { clearAuth, getProfile, setProfile, + getMigrationEntry, + setMigrationEntry, + getProjectKey, listProfiles, resolveProfile, resolveInstanceId, @@ -89,6 +92,42 @@ describe("config", () => { expect(await getAuth()).toBeUndefined(); }); + test("setMigrationEntry and getMigrationEntry", async () => { + expect(await getMigrationEntry("/projects/my-app")).toBeUndefined(); + await setMigrationEntry("/projects/my-app", { transformer: "clerk", file: "users.json" }); + expect(await getMigrationEntry("/projects/my-app")).toEqual({ + transformer: "clerk", + file: "users.json", + }); + expect(await getMigrationEntry("/projects/other")).toBeUndefined(); + }); + + // readConfig rebuilds the document field by field, so a key it does not know + // about is dropped on the next write rather than merely ignored. + // readConfig rebuilds the document field by field, so a section it does not + // know about is dropped on the next write rather than merely ignored. + test("migrations survive a write to another section", async () => { + await setMigrationEntry("/projects/my-app", { transformer: "clerk" }); + await setProfile("/projects/my-app", { + workspaceId: "org_abc", + appId: "app_def", + instances: { development: "ins_ghi" }, + }); + + expect(await getMigrationEntry("/projects/my-app")).toEqual({ transformer: "clerk" }); + }); + + test("getProjectKey prefers the linked profile's key over the directory", async () => { + expect(await getProjectKey("/projects/unlinked")).toBe("/projects/unlinked"); + + await setProfile("/projects/linked", { + workspaceId: "org_abc", + appId: "app_def", + instances: { development: "ins_ghi" }, + }); + expect(await getProjectKey("/projects/linked/src")).toBe("/projects/linked"); + }); + test("setProfile and getProfile", async () => { const profile = { workspaceId: "org_abc", diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index 41943b990..4d84ed82d 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -50,11 +50,21 @@ interface RelayEntry { token: string; } +/** What `clerk migrate import` last imported for a project, and how. */ +interface MigrationEntry { + transformer?: string; + file?: string; + skipUnsupportedProviders?: boolean; + /** Where this project's migration logs are written. Absent until chosen. */ + logDir?: string; +} + interface ClerkConfig { environment?: string; auth?: Record; profiles: Record; relay?: Record; + migrations?: Record; machineUuid?: string; telemetryNoticeShown?: boolean; telemetryDisabled?: boolean; @@ -93,6 +103,13 @@ function migrateRawConfig(raw: Record): ClerkConfig { config.relay = relay; } + // Not validated per entry the way `relay` is: every field is optional, so + // there is no key whose absence marks an entry as junk. A malformed one costs + // a remembered default, not a failed run. + if (raw.migrations && typeof raw.migrations === "object" && !Array.isArray(raw.migrations)) { + config.migrations = raw.migrations as Record; + } + if (raw.auth && typeof raw.auth === "object") { const auth = raw.auth as Record; if (typeof auth.userId === "string") { @@ -214,6 +231,18 @@ export async function setRelayEntry(key: string, entry: RelayEntry): Promise { + const config = await readConfig(); + return config.migrations?.[key]; +} + +export async function setMigrationEntry(key: string, entry: MigrationEntry): Promise { + const config = await readConfig(); + if (!config.migrations) config.migrations = {}; + config.migrations[key] = entry; + await writeConfig(config); +} + /** Persistent random machine id for telemetry. Generated on first use. */ export async function ensureMachineUuid(): Promise { const config = await readConfig(); @@ -308,6 +337,20 @@ export async function resolveProfile(cwd: string): Promise< return undefined; } +/** + * The key a per-project record (e.g. `migrations`) is filed under. + * + * Prefers the linked profile's own key so the record sits beside the profile it + * belongs to, and so it survives `clerk link` being re-run from a subdirectory. + * Falls back to the git remote and then the directory, because a project that + * has never been linked still deserves to be remembered. + */ +export async function getProjectKey(cwd: string): Promise { + const resolved = await resolveProfile(cwd); + if (resolved) return resolved.path; + return (await getGitNormalizedRemote(cwd)) ?? cwd; +} + const INSTANCE_ALIASES: Record = { dev: "development", development: "development", @@ -447,4 +490,4 @@ export async function resolveAppContext( }; } -export type { Auth, Profile, ClerkConfig, AppContextOptions }; +export type { Auth, Profile, ClerkConfig, MigrationEntry, AppContextOptions }; diff --git a/packages/cli-core/src/lib/constants.ts b/packages/cli-core/src/lib/constants.ts index 9e7c1c02f..e7b757ac7 100644 --- a/packages/cli-core/src/lib/constants.ts +++ b/packages/cli-core/src/lib/constants.ts @@ -55,3 +55,15 @@ export const NPM_REGISTRY_URL = "https://registry.npmjs.org/"; /** Event ingestion endpoint (telemetry-service worker → BigQuery). */ export const DEFAULT_TELEMETRY_ENDPOINT = "https://clerk-telemetry.com/v1/event"; export const TELEMETRY_TIMEOUT_MS = 1000; + +// ── Redaction ───────────────────────────────────────────────────────────── + +/** + * What a withheld secret displays as, everywhere the CLI shows one. + * + * Square brackets rather than a mask or a truncation: a row of dots or a + * head-and-tail (`aVer…3456`) reads as a value, and the reader has to work out + * that it is not one. Used by `clerk users create --dry-run` and by + * `clerk migrate settings`. + */ +export const REDACTED = "[REDACTED]"; diff --git a/packages/cli-core/src/lib/dotenv.ts b/packages/cli-core/src/lib/dotenv.ts index 2b5d0cb20..a2e0c2ec7 100644 --- a/packages/cli-core/src/lib/dotenv.ts +++ b/packages/cli-core/src/lib/dotenv.ts @@ -28,6 +28,76 @@ export async function findExistingEnvFile(cwd: string, fallback: string): Promis return fallback; } +/** + * The env files read back when resolving a value, as opposed to written to. + * + * Deliberately shorter than {@link ENV_FILE_CANDIDATES}: the runtime has + * already loaded every `.env*` variant it recognises into `process.env`, which + * {@link findEnvValue} checks first. This list only has to cover the case where + * the CLI's own process did not load the file — a different cwd at startup, or + * a runtime with no dotenv support. + */ +const ENV_FILES = [".env", ".env.local"]; + +export interface FindEnvValueOptions { + /** Injectable in tests; defaults to the real environment. */ + env?: Record; + /** Lowest priority first — a later file overrides an earlier one. */ + files?: readonly string[]; +} + +export interface LocatedEnvValue { + value: string; + /** Which of `names` supplied it — the caller may have passed several aliases. */ + name: string; + /** Where it came from, for `--verbose` (`CLERK_SECRET_KEY env var`, `.env.local`). */ + source: string; +} + +/** + * Looks for a value under any of `names`, in the order the app itself would + * resolve one: the environment first, then env files with a later file + * overriding an earlier one. + * + * This is the CLI's one way to read a project-level setting. Reading + * `process.env` directly instead skips the file fallback and reports no source, + * so a command that does it cannot explain where its input came from. + */ +export async function findEnvValue( + cwd: string, + names: string[], + options: FindEnvValueOptions = {}, +): Promise { + const { env = process.env, files = ENV_FILES } = options; + + for (const name of new Set(names)) { + const value = env[name]; + if (value) return { value, name, source: `${name} env var` }; + } + + // Priority is by name, not by position: the framework-specific name beats + // the generic fallback even when the generic one appears later in the same + // file. Within one name, a later file still overrides an earlier one. + const foundByName = new Map(); + for (const envFile of files) { + const file = Bun.file(join(cwd, envFile)); + if (!(await file.exists())) continue; + + for (const line of parseEnvFile(await file.text())) { + if (line.type !== "entry" || !line.value) continue; + if (names.includes(line.key)) { + foundByName.set(line.key, { value: line.value, name: line.key, source: envFile }); + } + } + } + + for (const name of names) { + const located = foundByName.get(name); + if (located) return located; + } + return undefined; +} + export type EnvLine = | { type: "comment"; raw: string } | { type: "blank" } diff --git a/packages/cli-core/src/lib/errors.ts b/packages/cli-core/src/lib/errors.ts index 247e02776..c984ba4d0 100644 --- a/packages/cli-core/src/lib/errors.ts +++ b/packages/cli-core/src/lib/errors.ts @@ -96,6 +96,8 @@ export const ERROR_CODE = { INSTALLER_NOT_FOUND: "installer_not_found", /** The npm registry was unreachable. */ REGISTRY_UNREACHABLE: "registry_unreachable", + /** A request never reached the server — DNS, refused connection, no route. */ + NETWORK_UNREACHABLE: "network_unreachable", /** Production instance was created but came back without a domain. */ DEPLOY_DOMAIN_MISSING: "deploy_domain_missing", /** Local publishable key and secret key address different applications. */ diff --git a/packages/cli-core/src/lib/fetch.test.ts b/packages/cli-core/src/lib/fetch.test.ts index 505131b9e..9ad2432d6 100644 --- a/packages/cli-core/src/lib/fetch.test.ts +++ b/packages/cli-core/src/lib/fetch.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { _resetUserAgentCache, loggedFetch } from "./fetch.ts"; import { _resetInterruptState, abortInFlight, beginInterrupt, interruptSignal } from "./signals.ts"; import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from "./config.ts"; +import { CliError } from "./errors.ts"; const originalFetch = globalThis.fetch; @@ -34,6 +35,31 @@ describe("loggedFetch", () => { expect(init.headers.get("User-Agent")).toBe("Custom/1.0"); }); + test("reports a connection failure as a CliError naming the host", async () => { + globalThis.fetch = mock(async () => { + // Bun's own shape for DNS failures, refused connections and no-route. + const error: NodeJS.ErrnoException = new Error( + "Unable to connect. Is the computer able to access the url?", + ); + error.code = "ConnectionRefused"; + throw error; + }) as unknown as typeof fetch; + + const failure = loggedFetch("https://example.test/x", { tag: "test" }); + await expect(failure).rejects.toThrow(/Could not reach example\.test/); + await expect(failure).rejects.toBeInstanceOf(CliError); + }); + + test("leaves a non-connection failure alone", async () => { + globalThis.fetch = mock(async () => { + throw new DOMException("The operation was aborted.", "AbortError"); + }) as unknown as typeof fetch; + + await expect(loggedFetch("https://example.test/x", { tag: "test" })).rejects.toThrow( + /operation was aborted/, + ); + }); + test("preserves other caller-provided headers", async () => { globalThis.fetch = mock( async () => new Response("ok", { status: 200 }), diff --git a/packages/cli-core/src/lib/fetch.ts b/packages/cli-core/src/lib/fetch.ts index 1d73ea9ff..36289e164 100644 --- a/packages/cli-core/src/lib/fetch.ts +++ b/packages/cli-core/src/lib/fetch.ts @@ -8,6 +8,7 @@ * every network error. See `.claude/rules/debug-logging.md`. */ +import { CliError, ERROR_CODE } from "./errors.ts"; import { log } from "./log.ts"; import { interruptSignal } from "./signals.ts"; import { withNetworkAccess } from "./host-execution.ts"; @@ -74,6 +75,23 @@ function interruptSignalFor( return own ? AbortSignal.any([own, interruptSignal()]) : interruptSignal(); } +/** + * Bun reports every connection-level failure — DNS, refused, no route — as a + * bare `Error` reading "Unable to connect. Is the computer able to access the + * url?", which names neither the host nor what wanted it, and which the global + * handler can only render as `unexpected_error`. Everything else, an aborted + * request included, is left exactly as thrown. + */ +function asConnectionError(error: unknown, url: string): unknown { + if ((error as NodeJS.ErrnoException | null)?.code !== "ConnectionRefused") return error; + + const host = URL.parse(url)?.host ?? url; + return new CliError( + `Could not reach ${host}. Check your network connection (or VPN) and try again.`, + { code: ERROR_CODE.NETWORK_UNREACHABLE }, + ); +} + export async function loggedFetch(url: URL | string, options: LoggedFetchInit): Promise { const { tag, bestEffort, ignoreInterrupt, ...init } = options; const method = init.method ?? "GET"; @@ -85,7 +103,9 @@ export async function loggedFetch(url: URL | string, options: LoggedFetchInit): const response = await withNetworkAccess( { operation: "connect", target: urlStr, label: tag, bestEffort }, async () => fetch(url, { ...init, headers, signal }), - ); + ).catch((error: unknown) => { + throw asConnectionError(error, urlStr); + }); if (!response.ok) { // Clone so the caller can still consume the body for error construction. const body = await response.clone().text(); diff --git a/packages/cli-core/src/lib/git.ts b/packages/cli-core/src/lib/git.ts index da4f4e955..697a57de8 100644 --- a/packages/cli-core/src/lib/git.ts +++ b/packages/cli-core/src/lib/git.ts @@ -1,4 +1,4 @@ -import { resolve } from "node:path"; +import { join, resolve } from "node:path"; import { log } from "./log.ts"; const $ = Bun.$; @@ -100,3 +100,22 @@ export function normalizeGitRemoteUrl(raw: string): string { return url.toLowerCase(); } + +/** + * Adds `entry` to the project's `.gitignore` unless it is already listed. + * + * The CLI writes files into a user's repository that must not be committed — + * the keyless breadcrumb, and the migration settings file. Creating one without + * this is how a live credential ends up in a tracked file. + */ +export async function ensureGitignoreEntry(cwd: string, entry: string): Promise { + const gitignorePath = join(cwd, ".gitignore"); + const content = await Bun.file(gitignorePath) + .text() + .catch(() => ""); + const lines = content.split("\n").map((l) => l.trim()); + if (lines.includes(entry)) return; + const separator = content && !content.endsWith("\n") ? "\n" : ""; + await Bun.write(gitignorePath, `${content}${separator}${entry}\n`); + log.debug(`git: added ${entry} to .gitignore`); +} diff --git a/packages/cli-core/src/lib/keyless-target.ts b/packages/cli-core/src/lib/keyless-target.ts index 99a63edbb..6eca393b1 100644 --- a/packages/cli-core/src/lib/keyless-target.ts +++ b/packages/cli-core/src/lib/keyless-target.ts @@ -11,7 +11,7 @@ import { join } from "node:path"; import { bapiRequest } from "./bapi.ts"; import { resolveAppContext, resolveProfile } from "./config.ts"; import { getStoredSession, hasAccountCredentials, type OAuthSession } from "./credential-store.ts"; -import { parseEnvFile } from "./dotenv.ts"; +import { findEnvValue } from "./dotenv.ts"; import { CliError, ERROR_CODE, throwUsageError } from "./errors.ts"; import { decodePublishableKey } from "./fapi.ts"; import { detectPublishableKeyName, detectSecretKeyName } from "./framework.ts"; @@ -38,8 +38,6 @@ export type InstanceTarget = | { kind: "account"; ctx: AccountContext; label: string } | { kind: "keyless"; keyless: KeylessTarget; label: string }; -const ENV_FILES = [".env", ".env.local"]; - /** * Where the Clerk SDKs park the keys for a keyless app they created themselves * (running `next dev` with no keys configured). Shape: @@ -71,45 +69,6 @@ export async function readSdkKeylessApp( } } -interface LocatedKey { - value: string; - source: string; -} - -/** - * Looks for a key under any of `names`, in the order the app itself would - * resolve one: the environment first, then env files with a later file - * overriding an earlier one. - */ -async function findKeyInProject(cwd: string, names: string[]): Promise { - for (const name of new Set(names)) { - const value = process.env[name]; - if (value) return { value, source: `${name} env var` }; - } - - // Priority is by name, not by position: the framework-specific name beats - // the generic fallback even when the generic one appears later in the same - // file. Within one name, a later file still overrides an earlier one. - const foundByName = new Map(); - for (const envFile of ENV_FILES) { - const file = Bun.file(join(cwd, envFile)); - if (!(await file.exists())) continue; - - for (const line of parseEnvFile(await file.text())) { - if (line.type !== "entry" || !line.value) continue; - if (names.includes(line.key)) { - foundByName.set(line.key, { value: line.value, source: envFile }); - } - } - } - - for (const name of names) { - const located = foundByName.get(name); - if (located) return located; - } - return undefined; -} - /** * The instance secret key a keyless project keeps locally. Falls back to the * keys an SDK created for itself, which it only does when nothing else supplies @@ -117,7 +76,7 @@ async function findKeyInProject(cwd: string, names: string[]): Promise { const names = [await detectSecretKeyName(cwd), "CLERK_SECRET_KEY"]; - const located = await findKeyInProject(cwd, names); + const located = await findEnvValue(cwd, names); const found = located ? { secretKey: located.value, source: located.source } @@ -136,7 +95,7 @@ async function sdkKeylessTarget(cwd: string): Promise /** The publishable key a keyless project holds locally, when one can be found. */ export async function findLocalPublishableKey(cwd: string): Promise { const names = [await detectPublishableKeyName(cwd), "CLERK_PUBLISHABLE_KEY"]; - const located = await findKeyInProject(cwd, names); + const located = await findEnvValue(cwd, names); return located?.value ?? (await readSdkKeylessApp(cwd))?.publishableKey; } diff --git a/packages/cli-core/src/lib/keyless.ts b/packages/cli-core/src/lib/keyless.ts index e8c769e2b..5994010f5 100644 --- a/packages/cli-core/src/lib/keyless.ts +++ b/packages/cli-core/src/lib/keyless.ts @@ -5,6 +5,7 @@ import { detectPublishableKeyName, detectSecretKeyName, detectEnvFile } from "./ import { parseEnvFile, mergeEnvVars, serializeEnvFile } from "./dotenv.ts"; import { BapiError } from "./errors.ts"; import { loggedFetch } from "./fetch.ts"; +import { ensureGitignoreEntry } from "./git.ts"; import { log } from "./log.ts"; const BREADCRUMB_DIR = ".clerk"; @@ -113,18 +114,6 @@ function breadcrumbPath(cwd: string): string { return join(cwd, BREADCRUMB_DIR, BREADCRUMB_FILE); } -async function ensureGitignoreEntry(cwd: string, entry: string): Promise { - const gitignorePath = join(cwd, ".gitignore"); - const content = await Bun.file(gitignorePath) - .text() - .catch(() => ""); - const lines = content.split("\n").map((l) => l.trim()); - if (lines.includes(entry)) return; - const separator = content && !content.endsWith("\n") ? "\n" : ""; - await Bun.write(gitignorePath, `${content}${separator}${entry}\n`); - log.debug(`Added ${entry} to .gitignore`); -} - export async function writeKeylessBreadcrumb(cwd: string, claimToken: string): Promise { await ensureGitignoreEntry(cwd, BREADCRUMB_DIR + "/"); await mkdir(join(cwd, BREADCRUMB_DIR), { recursive: true }); diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index e447ddd2e..35c34dfdb 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -71,6 +71,27 @@ export const NEXT_STEPS = { "Run `clerk apps list` to see your other applications", "Run `clerk config pull` to inspect the live configuration of this instance", ], + MIGRATE_DONE: [ + "Run `clerk migrate logs list` to inspect the import log", + "Run `clerk migrate delete` to undo this migration", + ], + // `logs list` only names the file; after a partial import the operator needs + // the failures themselves, which live one line per user in that file. + MIGRATE_DONE_WITH_ERRORS: (logFile: string) => [ + `Run \`grep '"status":"error"' ${logFile}\` to see every user that failed and why`, + "Run `clerk migrate delete` to undo this migration", + ], + MIGRATE_DELETE: ["Run `clerk migrate logs list` to inspect the deletion log"], + MIGRATE_SETTINGS: [ + "Run `clerk migrate settings set ` to change one", + "Run `clerk migrate settings clear ` to forget one", + "Run `clerk migrate settings clear` to forget them all, credentials included", + ], + // A suggested import is worthless unless it names the transformer that reads + // this export and the file just written. + MIGRATE_EXPORT: (transformerKey: string, file: string) => [ + `Run \`clerk migrate import --transformer ${transformerKey} --file ${file}\` to import them`, + ], } as const; /** diff --git a/packages/cli-core/src/lib/prompts-instructions.test.ts b/packages/cli-core/src/lib/prompts-instructions.test.ts new file mode 100644 index 000000000..b159c1a2e --- /dev/null +++ b/packages/cli-core/src/lib/prompts-instructions.test.ts @@ -0,0 +1,29 @@ +/** + * The multiselect footer hack, checked against the real @clack/prompts. + * + * Kept out of `prompts.test.ts`, which mocks the whole module — the one thing + * worth verifying here is that the real export is still a live array clack + * reads at render time. A clack upgrade that froze it, replaced it, or rendered + * a copy would drop `a: all` from the legend silently, and nothing else in the + * suite would notice. + */ + +import { test, expect } from "bun:test"; +import { MULTISELECT_INSTRUCTIONS } from "@clack/prompts"; + +// Importing for the module-level side effect is the point. +await import("./prompts.ts"); + +const legend = () => MULTISELECT_INSTRUCTIONS.join(" • ").replaceAll(/\[[0-9;]*m/g, ""); + +test("the multiselect legend advertises select-all", () => { + expect(legend()).toContain("a: all"); +}); + +test("confirm stays last, where readers expect it", () => { + expect(legend().endsWith("Enter: confirm")).toBe(true); +}); + +test("the keys clack actually binds are the ones named", () => { + expect(legend()).toBe("↑/↓ to navigate • Space: select • a: all • Enter: confirm"); +}); diff --git a/packages/cli-core/src/lib/prompts.ts b/packages/cli-core/src/lib/prompts.ts index 5d7ea6633..e4a0a9503 100644 --- a/packages/cli-core/src/lib/prompts.ts +++ b/packages/cli-core/src/lib/prompts.ts @@ -7,17 +7,35 @@ import { confirm as clackConfirm, isCancel, + MULTISELECT_INSTRUCTIONS, text as clackText, password as clackPassword, multiselect as clackMultiselect, type Option as ClackOption, } from "@clack/prompts"; import { editAsync } from "external-editor"; +import { dim } from "./color.ts"; import { throwUserAbort } from "./errors.ts"; import { ttyContext } from "./listage.ts"; import { log } from "./log.ts"; import { whileAwaitingUser } from "./signals.ts"; +/** + * Advertise select-all in the multiselect footer. + * + * `MultiSelectPrompt` binds `a` to toggle every option (and `i` to invert), but + * clack's instruction footer has never listed them and takes no override — the + * array below is the only seam, and it is read fresh on every render. So a + * genuinely useful key stays undiscoverable unless each call site spells it out + * in its own message, which is worse: it is a property of the prompt, not of + * any one question. + * + * Inserted second-to-last so `Enter: confirm` stays where readers expect it. + * `i` is left out deliberately — inverting is rarely what anyone wants, and a + * four-item legend stops being scannable. + */ +MULTISELECT_INSTRUCTIONS.splice(MULTISELECT_INSTRUCTIONS.length - 1, 0, `${dim("a:")} all`); + type ValidationResult = string | Error | true | undefined; type Validate = (value: string | undefined) => ValidationResult | Promise; type SyncValidate = (value: string | undefined) => string | Error | undefined; diff --git a/packages/cli-core/src/lib/spinner.ts b/packages/cli-core/src/lib/spinner.ts index 2222cb838..886248b33 100644 --- a/packages/cli-core/src/lib/spinner.ts +++ b/packages/cli-core/src/lib/spinner.ts @@ -107,7 +107,9 @@ export async function withGutter( let nextSteps: readonly string[] | undefined; const controls: GutterControls = { setNextSteps(steps) { - nextSteps = steps; + // Empty is ignored rather than stored: `outro([])` would render the + // "Next steps" header with no bullets under it. Matches printNextSteps. + if (steps.length > 0) nextSteps = steps; }, }; diff --git a/packages/cli-core/src/lib/users.ts b/packages/cli-core/src/lib/users.ts index d43a767e6..5d9345f18 100644 --- a/packages/cli-core/src/lib/users.ts +++ b/packages/cli-core/src/lib/users.ts @@ -1,8 +1,8 @@ import { bapiRequest } from "./bapi.ts"; +import { REDACTED } from "./constants.ts"; import { ERROR_CODE, throwUsageError } from "./errors.ts"; const USERS_INVALID_JSON_MESSAGE = "User payload must be a JSON object."; -const REDACTED = "[REDACTED]"; const DIRECT_REDACT_KEYS = new Set(["password", "code"]); const OBJECT_REDACT_KEYS = new Set(["private_metadata", "unsafe_metadata"]); diff --git a/packages/cli-core/src/test/integration/lib/harness.ts b/packages/cli-core/src/test/integration/lib/harness.ts index 6a99749fb..f4132d348 100644 --- a/packages/cli-core/src/test/integration/lib/harness.ts +++ b/packages/cli-core/src/test/integration/lib/harness.ts @@ -96,6 +96,7 @@ mock.module( getGitRepoIdentifier: async () => mockState.gitRepoIdentifier, getGitNormalizedRemote: async () => mockState.gitNormalizedRemote, normalizeGitRemoteUrl: (url: string) => url, + ensureGitignoreEntry: async () => {}, }) satisfies typeof import("../../../lib/git.ts"), ); @@ -115,7 +116,7 @@ mock.module( // ── Prompt queue (drives lib/prompts.ts and lib/listage.ts mocks) ──────────── -type PromptType = "select" | "search" | "input" | "confirm" | "password" | "editor"; +type PromptType = "select" | "search" | "input" | "confirm" | "password" | "editor" | "multiselect"; const promptQueues: Record = { select: [], @@ -124,6 +125,7 @@ const promptQueues: Record = { confirm: [], password: [], editor: [], + multiselect: [], }; function dequeuePrompt(name: PromptType) { @@ -164,6 +166,7 @@ export const mockPrompts = { input: (...responses: string[]) => promptQueues.input.push(...responses), password: (...responses: string[]) => promptQueues.password.push(...responses), editor: (...responses: string[]) => promptQueues.editor.push(...responses), + multiselect: (...responses: unknown[][]) => promptQueues.multiselect.push(...responses), }; function resetPromptQueues() { @@ -203,8 +206,12 @@ mock.module("../../../lib/listage.ts", () => ({ }, })); +// Every export of the real module must appear here: a missing one is a module +// link error at import time, not a failed prompt, so it takes down every test +// in the file the moment any command imports it. mock.module("../../../lib/prompts.ts", () => ({ confirm: dequeuePrompt("confirm"), + multiselect: dequeuePrompt("multiselect"), text: dequeuePrompt("input"), password: dequeuePrompt("password"), editor: dequeuePrompt("editor"), diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index b45e49c42..7cb412560 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -1,7 +1,8 @@ import { Writable } from "node:stream"; -import { afterEach, beforeEach, type spyOn } from "bun:test"; +import { afterAll, afterEach, beforeAll, beforeEach, type spyOn } from "bun:test"; import { type CapturedLogs, setActiveCapture } from "../../lib/log.ts"; import { setUiOutput } from "../../lib/ui.ts"; +import { _resetLogDir } from "../../commands/migrate/lib/logger.ts"; export function capturedOutput(spy: ReturnType): string { return spy.mock.calls.map((c: unknown[]) => c[0]).join("\n"); @@ -146,6 +147,9 @@ export const configStubs = { listProfiles: noop, getRelayEntry: noop, setRelayEntry: noop, + getMigrationEntry: noop, + setMigrationEntry: noop, + getProjectKey: async () => "", resolveProfile: noop, resolveProfileOrAutolink: noop, resolveInstanceId: () => ({ id: "", label: "" }), @@ -219,9 +223,14 @@ export const gitStubs = { * Stubs for `lib/prompts.ts` — the @clack/prompts-backed wrapper. Default * responses return benign values so tests can mock the module without * configuring each prompt explicitly. + * + * Must cover every export of the real module: an omission is a module link + * error at import time, which takes down the whole test file rather than + * failing one prompt. */ export const libPromptsStubs = { confirm: async () => true, + multiselect: async () => [], text: async () => "", password: async () => "", editor: async () => "{}", @@ -243,3 +252,32 @@ type FetchImpl = (input: string | URL | Request, init?: RequestInit) => Promise< export function stubFetch(impl: FetchImpl): void { globalThis.fetch = impl as typeof fetch; } + +/** + * Settles the migration log directory for a whole test file. + * + * `migrate import`, `export` and `delete` ask a human where logs should go the + * first time a project runs one. A test that flips to human mode to exercise + * something else — next steps, a wizard — would stop on that question and, + * where `prompts.ts` is mocked, silently eat the answer meant for another + * prompt. Pinning the environment variable answers it before it is asked, the + * same way an operator who exported one never sees it. + */ +export function useMigrateLogDir(dir = "./logs"): void { + let original: string | undefined; + + beforeAll(() => { + original = process.env.CLERK_MIGRATE_LOG_DIR; + process.env.CLERK_MIGRATE_LOG_DIR = dir; + }); + + // Resolution is cached per process, and each test file runs under its own + // temporary cwd, so the cached absolute path has to go with it. + beforeEach(() => _resetLogDir()); + + afterAll(() => { + if (original === undefined) delete process.env.CLERK_MIGRATE_LOG_DIR; + else process.env.CLERK_MIGRATE_LOG_DIR = original; + _resetLogDir(); + }); +} diff --git a/scripts/check-bun-version.ts b/scripts/check-bun-version.ts index a05b2d4b7..74f576892 100644 --- a/scripts/check-bun-version.ts +++ b/scripts/check-bun-version.ts @@ -9,6 +9,13 @@ * producing hundreds of order-dependent failures. Bun does not enforce * `engines.bun` at install time, so this preflight fails loudly instead. * + * A second, lower constraint rides along: the DB-backed `clerk migrate export` + * commands read MySQL through `Bun.sql` rather than `mysql2`. Verified against + * MySQL 8.4 -- the adapter landed in Bun 1.2.21, but VARBINARY/BLOB columns + * came back as lossily decoded strings until 1.3.6, which would silently + * corrupt exported password hashes. The 1.3.13 floor above already covers it; + * do not drop below 1.3.6 if the `--parallel` requirement ever goes away. + * * Usage: * bun run scripts/check-bun-version.ts */ diff --git a/testing.md b/testing.md new file mode 100644 index 000000000..6a475e97c --- /dev/null +++ b/testing.md @@ -0,0 +1,1802 @@ +# `clerk migrate` — Manual Test Plan + +Test plan for the `ra/integrate-migration-tool-into-cli` PR (40 commits, ~21.5k +lines added). Covers every export platform, every import path, logs, settings, +delete, transformers, and the shared CLI changes the PR made outside `migrate/`. + +Check a box when the behavior is confirmed. `[x]` = passed, `[-]` = skipped +(say why), `[!]` = failed (file an issue and link it). + +**Section index** + +| # | Section | Needs | +| --- | ------------------------------------------------------------------------------------------- | -------------------------- | +| 0 | [Setup](#0-setup) | — | +| 1 | [Command surface and help](#1-command-surface-and-help) | nothing external | +| 2 | [Export — shared behavior](#2-export--shared-behavior-all-platforms) | any one platform | +| 3 | [Export — Clerk](#3-export--clerk) | a Clerk account | +| 4 | [Export — Auth0](#4-export--auth0) | an Auth0 tenant | +| 5 | [Export — WorkOS](#5-export--workos) | a WorkOS tenant | +| 6 | [Export — Firebase](#6-export--firebase) | a Firebase project | +| 7 | [Export — Supabase](#7-export--supabase) | a Supabase Postgres | +| 8 | [Export — Auth.js](#8-export--authjs) | a Postgres/MySQL/SQLite DB | +| 9 | [Export — Better Auth](#9-export--better-auth) | a Postgres/MySQL/SQLite DB | +| 10 | [Import — core](#10-import--core) | a Clerk dev instance | +| 11 | [Import — interactive wizard](#11-import--interactive-wizard) | a Clerk dev instance | +| 12 | [Import — Firebase hash parameters](#12-import--firebase-hash-parameters) | a Firebase export | +| 13 | [Import — readiness report](#13-import--readiness-report) | a Clerk dev instance | +| 14 | [Import — throughput and the dev user limit](#14-import--throughput-and-the-dev-user-limit) | a Clerk dev instance | +| 15 | [Import — custom transformers](#15-import--custom-transformers) | nothing external | +| 16 | [Transformers list](#16-transformers-list) | nothing external | +| 17 | [Settings](#17-settings) | nothing external | +| 18 | [Logs](#18-logs) | a previous run | +| 19 | [Delete](#19-delete) | a previous import | +| 20 | [Cross-cutting](#20-cross-cutting) | — | +| 21 | [CI and repo hygiene](#21-ci-and-repo-hygiene) | — | + +--- + +## 0. Setup + +Run everything from the worktree root: +`/Users/royanger/clerk/clk/.clk/features/integrate-migration-tool-into-cli/cli` + +Two ways to invoke the CLI. **Test both at least once** — the compiled binary is +what ships, and several behaviors (env files, native modules, macros) differ +between them. + +```sh +# From source — fast iteration +bun run dev -- migrate --help +# or, without the wrapper's arg-forwarding quirks: +cd packages/cli-core && bun run ./src/cli.ts migrate --help + +# Compiled binary — what users actually get +bun run build:compile +./packages/cli-core/dist/clerk migrate --help +``` + +Throughout this document `clerk` means whichever of those you are using. + +- [ ] **0.1** `bun install` completes cleanly in this worktree. +- [ ] **0.2** `bun --version` is >= 1.3.13 (the `engines.bun` floor). Ideally + > = 1.3.6 is already implied; MySQL binary columns need it. +- [ ] **0.3** `bun run build:compile` produces `packages/cli-core/dist/clerk` + and `./packages/cli-core/dist/clerk --version` prints a `-dev..` + version. +- [ ] **0.4** Work in a **scratch directory**, not the CLI checkout — several + commands write `./exports`, `./logs` and `./.env.clerk-migrate` into the cwd, + and settings are keyed per project. +- [ ] **0.5** Have a **throwaway Clerk development instance** for imports. Do + not point `migrate import` at anything you care about. + +### Modes + +| Mode | How to get it | Effect | +| ------- | ------------------------------------------------------- | --------------------------------------- | +| Human | a TTY, no flag | prompts allowed | +| Agent | `--mode agent`, or `CLERK_MODE=agent`, or piping stdout | never prompts; JSON errors | +| Non-TTY | `clerk … \| cat` | treated as agent for prompting purposes | + +`--verbose` is a global flag and shows which file each resolved value came from. + +--- + +## 1. Command surface and help + +- [ ] **1.1** `clerk --help` lists a `migrate` row with the description + "Migrate users into Clerk from another auth provider or another Clerk instance". +- [ ] **1.2** `clerk migrate` on its own prints **help**, not an import. It is a + group, not a command. (Guards `9a96503` — `run` used to be `isDefault`.) +- [ ] **1.3** `clerk migrate --help` lists exactly: `delete`, `export`, `import`, + `logs`, `settings`, `transformers` (plus `help`), and an Examples block. +- [ ] **1.4** **Removed command:** `clerk migrate run` fails with + `error: unknown command 'run'`. (Breaking change in `9a96503`.) +- [ ] **1.5** **Removed flag:** `clerk migrate import --clerk-secret-key sk_test_x …` + fails with `error: unknown option '--clerk-secret-key'`. +- [ ] **1.6** `--help` works on every leaf: `migrate export firebase --help`, + `migrate logs list --help`, `migrate settings clear --help`, etc. Each shows + its own usage line and an Examples block. +- [ ] **1.7** Shell completion reads the registry: + `clerk __complete migrate ""` lists the six subcommands, and + `clerk __complete migrate import --transformer ""` lists all **seven** + transformers (`clerk auth0 authjs betterauth firebase supabase workos`). +- [ ] **1.8** `clerk migrate export --help` lists all seven platforms, each with + its `default: ./exports/-export-.json` note. +- [ ] **1.9** Every `migrate` command's human output is wrapped in the gutter + frame (`┌ … │ … └`), including `logs` and `transformers`. Nothing prints + flush-left. (Guards `922890a`, `05b6536`.) +- [ ] **1.10** Cancelling any prompt with Ctrl-C closes the frame with + `└ Paused`, not `└ Failed`. +- [ ] **1.11** Spinner messages all end in `...` and vanish on completion — no + separate "done" message. (Guards `05b6536`.) +- [ ] **1.12** Status icons are `✓` / `✗` / `!` everywhere. No `●` or `○`. +- [ ] **1.13** Pluralization: run something that yields exactly 1 user and + confirm "1 user", not "1 user(s)". + +--- + +## 2. Export — shared behavior (all platforms) + +Run these once against any one platform (Clerk is easiest), then spot-check on +the others. + +### The platform picker + +- [ ] **2.1** `clerk migrate export` with no platform opens + `Which platform are you exporting from?` with **seven** rows in registry + order: Clerk, Auth0, Supabase, Auth.js (NextAuth), Firebase, Better Auth, + WorkOS — each with its description hint. +- [ ] **2.2** Choosing a row runs that platform's export and forwards global + options. +- [ ] **2.3** Ctrl-C at the picker exits **0** with no error text. +- [ ] **2.4** Agent mode / piped stdout: usage error, exit 2 — + `` `clerk migrate export` needs a platform and cannot prompt here. Name one: +clerk, auth0, supabase, authjs, firebase, betterauth, workos. `` with one + example per platform. +- [ ] **2.5** `clerk migrate export okta` → Commander unknown-command error. +- [ ] **2.6** The picker passes the **parent** command's options, so a + picker-driven run never sees `--db-url` etc. Confirm it still prompts. + +### Where the file goes (guards `35c2b09`) + +- [ ] **2.7** No `--output`, human: one prompt **`Save the export to:`**, + prefilled with `exports/-export-.json`. Enter accepts. +- [ ] **2.8** The stamp is **local time, to the minute**, ISO 8601 basic — no + seconds. +- [ ] **2.9** Typing over the prefill saves elsewhere. Answer is trimmed. +- [ ] **2.10** An empty answer is rejected with `A path is required`. +- [ ] **2.11** The prompt appears **before** any users are fetched, so a long + export can be left unattended. Watch the ordering. +- [ ] **2.12** `--output somewhere/mine.json` skips the prompt and resolves + against **cwd**, not `exports/` — `exports/` is not created. +- [ ] **2.13** `--output ../elsewhere/users.json` escapes the project and works. +- [ ] **2.14** Missing parent directories are created. +- [ ] **2.15** Agent mode takes the proposed path silently. +- [ ] **2.16** Two exports **one minute apart** produce two files. Two in the + **same minute** collide — confirm the second overwrites (documented + trade-off, but worth seeing). +- [ ] **2.17** The file is a pretty-printed JSON **array** (2-space indent). + +### Coverage report and next steps + +- [ ] **2.18** With ≥1 user: a `Field coverage` block, one row per field, icons + `✓` when all, `!` when partial, `✗` when zero. +- [ ] **2.19** Closing line `Exported N users to ` — singular + "user" at 1. +- [ ] **2.20** Human runs close with `└ Next steps` → ``Run `clerk migrate +import --transformer --file ` to import them``. The path is + **relative** when inside cwd, absolute when outside. +- [ ] **2.21** Zero users: no coverage table, warning + `No users found to export. Wrote an empty file to .`, file contains + `[]`, and **no Next steps block at all**. (Guards `05b6536`.) +- [ ] **2.22** Agent mode: no gutter, no next steps, but coverage + the + `Exported N users` line still print. +- [ ] **2.23** Ctrl-C mid-export closes with `└ Paused` + + `→ Run this command again to continue.`, exit 130. +- [ ] **2.24** A mid-run failure closes with `└ Failed`. + +### The log-directory question (guards `1c029c8`) + +- [ ] **2.25** First export in a **fresh project** (no `CLERK_MIGRATE_LOG_DIR`, + no saved `log-dir`), human: prompt `Where should migration logs be saved?` + defaulting to `./logs`. +- [ ] **2.26** The answer is saved and printed: + ``Saving migration logs to ./logs. Change it with `clerk migrate settings set +log-dir `.`` +- [ ] **2.27** The **second** run in the same project does **not** ask. +- [ ] **2.28** Agent mode / non-TTY takes `./logs` and saves **nothing** — the + question stays open for the first interactive run. Verify by running agent + mode first, then human. +- [ ] **2.29** `CLERK_MIGRATE_LOG_DIR=/tmp/mylogs` skips the prompt entirely. +- [ ] **2.30** Every export writes `/export-.log` as NDJSON, one + `{"userId":…,"status":"success"}` per user. +- [ ] **2.31** A log directory that cannot be written only **warns** + (`Could not write migration log: …`) — it never aborts the export. + +### `-y` on exports (added in this branch) + +Every export subcommand now takes `-y, --yes`. It means "do not prompt": the +credential-retry loop fails outright instead of re-asking, and the log-directory +question takes `./logs` without saving. + +- [ ] **2.32** `clerk migrate export --help` lists + `-y, --yes Do not prompt: require --output, and fail on a rejected credential`. + Check all seven. +- [ ] **2.33** `-y` with a **rejected** credential fails on the first attempt — + no re-prompt, exit 2. Compare against the same run without `-y`, which + re-asks. +- [ ] **2.34** `-y` takes `./logs` without asking and **without saving**, so a + later run without `-y` still gets the question. +- [ ] **2.35** **`-y` with no `--output` fails instead of asking or guessing.** + `clerk migrate export supabase -y` in a human TTY → + `` `clerk migrate export supabase` needs an export location and will not +prompt for one with -y. Pass --output, then run it again. `` exit **2**. +- [ ] **2.36** The error hands back a command that **actually runs**. Copy the + example line, paste it, and confirm the export completes to that path: + `clerk migrate export supabase -y --output exports/supabase-export-.json` +- [ ] **2.37** The error names the platform you actually ran (try `firebase`), + not a hardcoded one. +- [ ] **2.38** `-y` **together with** `--output` never errors — the question was + already answered. +- [ ] **2.39** **Agent mode still defaults**, even with `-y`: + `clerk --mode agent migrate export supabase -y --db-url …` writes to the + proposed path with no error. An agent passing `-y` reflexively must not turn + a working export into a usage failure. + +--- + +## 3. Export — Clerk + +`clerk migrate export clerk [-o ] [--secret-key ] [--app ] [--instance ]` + +The interesting part is **source-instance resolution** (`8f374a9`): the linked +project is normally the migration's _destination_, so it is never taken +silently. + +### Source resolution + +- [ ] **3.1** `--secret-key sk_test_…` runs **unquestioned** — no picker. The + gutter reads `Exporting from the resolved instance.` +- [ ] **3.2** A key not starting `sk_` is rejected. +- [ ] **3.3** **Linked directory, no flags, human:** a searchable prompt opens — + `What Clerk instance do you want to export users from?` — with one row per + **instance**, rendered `my-app - Production instance (ins_abc)`. +- [ ] **3.4** Rows for the **currently resolved application come first**, then + every other app's instances. Taking the expected one is a single Enter. +- [ ] **3.5** Typing filters case-insensitively on the rendered label — an app + name, the word `production`, and a raw `ins_…` id all narrow it. +- [ ] **3.6** The picker has **no** `+ Create a new application` row. +- [ ] **3.7** After picking, there is **no second prompt** for the instance. + The gutter prints `Exporting from (production).` +- [ ] **3.8** Account with **zero instances** (or a degraded Platform API): + no picker, an info block listing `--secret-key`, `--app`/`--instance` and + `clerk link`, then exit **0** with no error. +- [ ] **3.9** Agent mode with anything resolvable: no picker, resolved key used + silently. +- [ ] **3.10** Agent mode, unlinked, no key: `No secret key found. Provide one +via: …`, exit 2. + +### Three resolution edges that look like bugs — confirm and decide + +- [x] **3.11 `--app` / `--instance` skip the picker** (fixed in this branch). + `clerk migrate export clerk --instance prod --output prod-users.json` in a + human TTY must run with **no prompt**. Same for `--app` alone and for the + two together. This is what makes the export scriptable outside agent mode. +- [x] **3.12 An exported `CLERK_SECRET_KEY` is honoured** (fixed in this + branch). Export with `CLERK_SECRET_KEY` set from a **linked** directory: + no picker, and the export reads the instance that key names. Previously + the picker opened and overrode it, making this the one command where + exporting the variable did less than not exporting it. +- [ ] **3.13 The "nothing to resolve" path uses the application picker.** + Unlinked directory, no key, no flags, human: the **application** picker opens + (`Select a Clerk application to use:`) **including `+ Create a new +application`**, then an instance picker when the app has more than one. + The README now documents this. Confirm it matches, and **decide whether + offering "create a new application" on an export is acceptable** — picking it + would export a brand-new empty app. +- [ ] **3.14** Keyless / accountless app in the directory: the picker opens + (tier 2 behavior). +- [ ] **3.15** Not logged in, linked dir, human: a plain auth error propagates — + **not** the friendly "Export from a different instance…" block. + +### Fetch and output + +- [ ] **3.16** Spinner `Fetching users from Clerk...` updating to + `Fetching users from Clerk: N so far...`. Requests are + `GET /v1/users?limit=500&offset=0`, `offset=500`, … +- [ ] **3.17** An instance whose user count is an **exact multiple of 500** + makes one extra request (the loop breaks only on a short page). Check with + `--verbose`. +- [ ] **3.18** Output fields per user: `id`, `primary_email_address`, + `verified_email_addresses[]`, `unverified_email_addresses[]`, the same trio + for phones, `username`, `first_name`, `last_name`, non-empty metadata + objects, `banned` (only when true), `create_organization_enabled`, + `create_organizations_limit`, `delete_self_enabled`, and `created_at` / + `legal_accepted_at` converted from Unix ms to RFC3339. +- [ ] **3.19** The primary identifier is **excluded** from the additional list. + With no primary flagged, the first verified address is promoted. +- [ ] **3.20** Empty metadata objects are omitted entirely. +- [ ] **3.21** Coverage rows: email / phone / username / first name / last name + / `have a password (not exportable — see below)`. +- [ ] **3.22** The password warning prints: + `Clerk's API never returns password digests, TOTP secrets or backup codes…` + — and is **suppressed** for a zero-user export. +- [ ] **3.23** A revoked secret key surfaces a 401. There is **no retry + prompt** here — `export clerk` is the one API export without `withInputRetry`. +- [ ] **3.24** 429 handling: up to 5 retries honouring `Retry-After`. Note that + **no `onRetry` is passed**, so the spinner sits silently for up to ~50s. + **Confirm whether that is acceptable.** +- [ ] **3.25** Round trip: export a **dev** instance, import into **prod**, + confirm the picker prevented you exporting the destination by accident. + +--- + +## 4. Export — Auth0 + +`clerk migrate export auth0 [--domain ] [--client-id ] [--client-secret ] [-o ]` + +**Setup:** a machine-to-machine application with the `read:users` scope +(_Applications → APIs → Auth0 Management API → Machine to Machine Applications_). + +- [ ] **4.1** All three flags given: no credential prompts. +- [ ] **4.2** `AUTH0_DOMAIN` / `AUTH0_CLIENT_ID` / `AUTH0_CLIENT_SECRET` from the + shell work. Flags beat env. +- [ ] **4.3** The same three read from `.env.clerk-migrate`, `.env.local` and + `.env` (highest to lowest). `--verbose` prints + `migrate: AUTH0_DOMAIN from ` naming the file. (Guards `2f22633` — + the compiled binary does not autoload dotenv.) +- [ ] **4.4** **Domain normalization:** `https://t.auth0.com/`, + `http://t.auth0.com/` and ` t.auth0.com ` all normalize to `t.auth0.com`. + Confirm the token request goes to `https://t.auth0.com/oauth/token` with + audience `https://t.auth0.com/api/v2/`. +- [ ] **4.5** Nothing supplied, human: an info line naming the M2M setup, then + prompts **only for what is missing** — domain (text), client ID (text), + client secret (**masked**). +- [ ] **4.6** Empty answers are rejected: `A domain is required`, + `A client ID is required`, `A client secret is required`. +- [ ] **4.7** Partially supplied: only the missing prompts appear. +- [ ] **4.8** Agent mode with nothing supplied names **every** missing + credential at once: + `Missing: --domain (or AUTH0_DOMAIN), --client-id (or AUTH0_CLIENT_ID), --client-secret (or AUTH0_CLIENT_SECRET).` + Exit 2. Not one per run. +- [ ] **4.9** Agent mode partially supplied names only the remainder. + +### Credential retry (guards `563ff35`) + +- [ ] **4.10** Wrong secret, human: error + `Auth0 rejected the credentials (): ` plus the `read:users` + advice, then **all three prompts are asked again** (Auth0's error names no + field). A correct set completes the export. +- [ ] **4.11** The export continues against whichever credential set worked. +- [ ] **4.12** Ctrl-C at the retry prompt exits the loop cleanly (exit 0, + `└ Paused`). +- [ ] **4.13** Agent mode: the error is thrown immediately, **no retry**, exit 2. +- [ ] **4.14** A 200 response with no `access_token` gives the same rejection + error ending `: no access token returned`. +- [ ] **4.15** A 403 on token exchange mentions `read:users`. +- [ ] **4.16** A failure **listing users** (not authenticating) gives + `Auth0 returned listing users: ` and is **not** retried with a + new credential. + +### Pagination and the 1000 ceiling + +- [ ] **4.17** Requests are + `GET https:///api/v2/users?page=0&per_page=100&include_totals=true`, + then `page=1`, … stopping on a short page. +- [ ] **4.18** A tenant with **≥1000 users** warns: + `Auth0 only pages through the first 1000 users on this endpoint`, plus + `, and this tenant reports ` when the total exceeds 1000, then + `. Exported what is reachable; use Auth0's bulk user export job for the rest.` + The export continues with exactly 1000 users. +- [ ] **4.19** A tenant of **exactly 1000**: confirm the warning fires at 1000 + and no 11th request is made. + +### Output + +- [ ] **4.20** Fields copied when truthy: `user_id`, `email`, `username`, + `given_name`, `family_name`, `phone_number`, `created_at`. `email_verified` + and `phone_verified` are copied **on presence**, so `false` survives. + Non-empty `user_metadata` / `app_metadata` kept. +- [ ] **4.21** Dropped: `identities`, `logins_count`, `last_login`, + `multifactor`. +- [ ] **4.22** Coverage rows: email / phone / username / first name / last name. + **No password row.** +- [ ] **4.23** Password warning: `Auth0's Management API does not return +password hashes. Request a password hash export from Auth0 support and add a +`passwordHash` field to each user before importing…` with the docs link. +- [ ] **4.24** `--verbose` shows the token POST and the users GET. **The client + secret must not appear in any log line** — it is in the POST body; confirm + nothing prints it. + +--- + +## 5. Export — WorkOS + +`clerk migrate export workos [--api-key ] [--with-identities] [-o ]` + +New in `8893e60`. WorkOS returns **no passwords and no TOTP secrets**. + +- [ ] **5.1** `--api-key sk_…` runs with no prompt. The key is trimmed. +- [ ] **5.2** `WORKOS_API_KEY` from the shell or any `.env*` file works. Flag + beats env. +- [ ] **5.3** Nothing supplied, human: info line + ``WorkOS needs a secret API key, the one starting `sk_`. Find it in the WorkOS +dashboard under API Keys.`` then a **masked** prompt + `WorkOS secret API key (sk_…)`. Empty → `An API key is required`. +- [ ] **5.4** Agent mode: usage error naming + `Missing: --api-key (or WORKOS_API_KEY).`, exit 2. +- [ ] **5.5** Rejected key, human: `WorkOS returned 401 listing users: ` + plus ``Check that the key is a secret key (`sk_…`) for the right environment, +and that it has not been revoked.`` then **only that one prompt** returns. +- [ ] **5.6** A non-JSON error body (a proxy HTML page) is surfaced verbatim, + not swallowed. An empty body reads `no detail returned`. +- [ ] **5.7** Agent mode: thrown immediately, no retry. +- [ ] **5.8** **The page fetched to prove the key is reused** — the first + `GET /user_management/users?limit=100&order=asc` is not repeated. Count + requests with `--verbose`. + +### Pagination + +- [ ] **5.9** Cursor pagination follows `list_metadata.after` until absent. + There is **no record ceiling** — a tenant larger than 1000 exports in full + (contrast Auth0). +- [ ] **5.10** A **full final page** does not cause an extra request (unlike + Clerk and Auth0). +- [ ] **5.11** Non-TTY progress on stderr every **10 pages**: + `Fetched N users from WorkOS so far...`. Needs ≥1000 users to see. + +### `--with-identities` + +- [ ] **5.12** Without the flag, human, and **>0 users**: after the user count + is known, a confirm — `Also fetch each user's OAuth providers? That is +extra request(s), and the result is report-only — Clerk's import cannot take +external accounts.` — defaulting to **No**. +- [ ] **5.13** Singular "request" when N is 1. +- [ ] **5.14** Zero users: no question asked. +- [ ] **5.15** Agent mode: no question; the flag's value is taken as-is. +- [ ] **5.16** With the flag: one `GET /user_management/users//identities` + per user, max 10 concurrent and ≥50ms apart. Spinner + `Fetching OAuth providers: done/total...`. +- [ ] **5.17** Non-TTY identity progress every **500** users. +- [ ] **5.18** The **OAuth providers** block prints under the coverage table, + **not** as coverage rows. Rows sorted by count desc, then name asc; then + `no OAuth provider N users`; then, only if a lookup failed, + `not readable N users` plus the dim caveat line. +- [ ] **5.19** A failed identity lookup does **not** abort the run. That user is + counted under `not readable` and has **no `identities` key** in the export — + not an empty array. +- [ ] **5.20** An identities response wrapped as `{ "data": [...] }` is read + correctly, not as "no providers". +- [ ] **5.21** An unknown `provider` renders as `unknown`. + +### Output and warnings + +- [ ] **5.22** Fields: `id`, `email`, `first_name`, `last_name`, `created_at` + (truthy only); `email_verified` on presence; non-empty `metadata`; + `identities` only when fetched and non-empty. Dropped: `locale`, + `profile_picture_url`, `last_sign_in_at`, `updated_at`, `external_id`. +- [ ] **5.23** Coverage always carries `have a password (WorkOS returns none)` + at **0/N**, rendering dim `✗`. +- [ ] **5.24** Credential warning: `WorkOS does not return password hashes or +TOTP secrets, and there is no export that does…` with the docs link. +- [ ] **5.25** The `workos` transformer names **no `passwordHasher`** — users + import with `skip_password_requirement` and no credential at all, never a + synthetic digest. Verify on the wire. +- [ ] **5.26** Round trip: export from WorkOS, import into Clerk, confirm a + migrated user can complete a password reset and sign in. + +--- + +## 6. Export — Firebase + +`clerk migrate export firebase [--service-account ] [-o ]` + +**Setup:** a service account key from _Project settings → Service accounts → +Generate new private key_, with the Firebase Authentication Admin role. +There is **no** `FIREBASE_SERVICE_ACCOUNT` env var — flag or prompt only. + +- [ ] **6.1** `--service-account ./sa.json` exports. The key is validated + **before any network call** — a bad path makes **zero** requests. +- [ ] **6.2** No flag, human (guards `f077786`): dim info + `Firebase console → Project settings → Service accounts → Generate new private key.` + then a **masked** prompt + `Path to the service account key file, or paste the key JSON`. +- [ ] **6.3** Answering with a **path** works. +- [ ] **6.4** Answering by **pasting the key JSON whole** works — anything + starting `{` after trim is parsed as JSON. +- [ ] **6.5** An invalid answer re-asks **in place** without leaving the prompt. +- [ ] **6.6** Agent mode: usage error naming `--service-account`, exit 2. + +### Key validation messages + +- [ ] **6.7** Missing file → `No service account file at .` +- [ ] **6.8** Not JSON → ` is not valid JSON: ` +- [ ] **6.9** Pasted non-JSON → `The pasted key is not valid JSON: ` +- [ ] **6.10** Wrong `type` (gcloud `authorized_user` creds) → `… is not a +usable service account key: its "type" is "authorized_user", not +"service_account".` +- [ ] **6.11** **Web app config pasted by mistake** → `… "project_id" is +missing`. Test each of `project_id`, `client_email`, `private_key`. +- [ ] **6.12** Mangled newlines in `private_key` → `… "private_key" does not +look like a PEM key — check its newlines survived copying`. +- [ ] **6.13** PEM body not valid base64 → `The service account's private_key is +not valid base64.` +- [ ] **6.14** Valid base64 but not PKCS8 → `The service account's private_key +could not be read: `. + +### Auth and retry + +- [ ] **6.15** Normal run: gutter `Exporting from the project.` + then spinner `Authenticating with Google...`. Token exchange is + `POST https://oauth2.googleapis.com/token` with an RS256 JWT. +- [ ] **6.16** **Revoked key, human:** `Google rejected the service account +(): ` plus `Check the key has not been revoked, and that the +service account has the Firebase Authentication Admin role.` The masked + prompt returns. +- [ ] **6.17** The run continues against the key Google **accepted**, and + `project_id` is read off **that** key, not the first one offered. + (Guards `563ff35`.) +- [ ] **6.18** Agent mode: thrown immediately, exit 2. + +### Emulator (cheapest way to test most of this) + +- [ ] **6.19** `firebase emulators:start --only auth`, seed a couple of users, + then run with `FIREBASE_AUTH_EMULATOR_HOST=127.0.0.1:9099`. The token + exchange is **skipped entirely** (bearer becomes the literal `owner`) and + Identity Toolkit calls go to `http:///identitytoolkit.googleapis.com/…`. +- [ ] **6.20** A fake-but-well-formed service account key must still be supplied + and must still pass local validation. + +### Fetch and output + +- [ ] **6.21** `GET /v1/projects//accounts:batchGet?maxResults=1000`, + paging on `nextPageToken`. Spinner `Fetching users from Firebase: N so far...`. +- [ ] **6.22** A listing failure gives `Firebase returned listing +users: ` and is **not** retried. +- [ ] **6.23** Fields: `localId`, `email`, `displayName`, `phoneNumber`, + `createdAt` (truthy only); `emailVerified` on presence. +- [ ] **6.24** **`passwordHash` + `salt` only when BOTH are present.** A user + with a hash but no salt (or the reverse) must arrive with **neither** — half + a credential produces a user nobody can sign in as. +- [ ] **6.25** A phone-only user exports as `{localId, phoneNumber}`. +- [ ] **6.26** Coverage rows: email / verified email / password hash / display + name / phone number. + +### Hash parameters + +- [ ] **6.27** Config read **permitted**, ≥1 password hash: a bold + `Password hash parameters` block, `Read from the project. Import with:`, then + the import command. +- [ ] **6.28** **The command is ONE line.** Copy it out of the gutter and paste + it into a shell — it must run. No `│` characters travel with it. + (Guards `534894f`; the old four-line version produced + `error: too many arguments for 'import'. Expected 0 arguments but got 3: │, │, │.`) +- [ ] **6.29** Defaults when absent: `rounds=8`, `memoryCost=14`. +- [ ] **6.30** Config read **denied** (403, or any non-2xx): the export still + **succeeds** and prints the "find them in the console under Authentication → + Users → (⋮) → Password hash parameters" guidance instead. `--verbose` shows + `firebase: 403 reading the project config`. +- [ ] **6.31** Config present but empty (`signIn.hashConfig` missing + `signerKey`/`saltSeparator`) is treated as denied — same guidance block. +- [ ] **6.32** **No password hashes at all:** no config call is made, and the + output is a single dim line `No password hashes in this export, so no hash +parameters are needed.` +- [ ] **6.33** **Key material never leaks.** Neither the PEM nor any fragment + appears in stdout, stderr, `--verbose` output, or the export file. +- [ ] **6.34** Ordering on screen reads sensibly: the hash block appears, then + `└ Next steps`. + +--- + +## 7. Export — Supabase + +`clerk migrate export supabase` reads `auth.users` **directly over Postgres**, +not the Admin API, because `encrypted_password` exists only in the table. + +**Setup:** a Supabase project with Auth enabled and a few users, at least one +with a password and one social-only. Get the connection string from +_Dashboard → Connect → Session pooler_. + +### Happy path + +- [ ] **7.1** `clerk migrate export supabase --db-url "postgres://postgres:…@…:5432/postgres"` + Expect: save-path prompt prefilled → export → coverage table with rows for + email / confirmed email / password hash / phone / first name / last name. +- [ ] **7.2** The run prints + `Password hashes are included — this is why the export reads the database rather than the Admin API.` +- [ ] **7.3** Open the JSON. `raw_app_meta_data` is present and carries + `providers` — this is what `--skip-unsupported-providers` reads at import. +- [ ] **7.4** `first_name` is coalesced from `display_name`, then `first_name`, + then `name` in `raw_user_meta_data`. Confirm against a user who only has + `name` set. +- [ ] **7.5** Null columns are **omitted** from each record, not written as + `null`. Dates come out as ISO strings. + +### Credential resolution + +- [ ] **7.6** `SUPABASE_DB_URL` in the shell is used when `--db-url` is absent. +- [ ] **7.7** `SUPABASE_DB_URL` in `.env`, `.env.local` and `.env.clerk-migrate` + are all read (`.env.clerk-migrate` highest). `--verbose` prints + `migrate: SUPABASE_DB_URL from ` naming the **file**. +- [ ] **7.8** A shell-exported value beats every file, and `--verbose` says so + even when a file holds the same key with a different value. +- [ ] **7.9** A garbage `SUPABASE_DB_URL` warns + `SUPABASE_DB_URL is not a valid connection string; ignoring it.` and falls + through to the prompt. +- [ ] **7.10** With nothing set, a human gets a **masked** prompt preceded by the + dim hint `Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.` +- [ ] **7.11** Agent mode with nothing set fails with + `` `clerk migrate export supabase` needs a database connection and cannot prompt here. `` + naming `--db-url` and `SUPABASE_DB_URL`. No prompt, no hang. +- [ ] **7.12** Piping stdout (`clerk migrate export supabase | cat`) behaves like + agent mode. +- [ ] **7.13** `clerk migrate export supabase -y` is rejected — export commands + have **no** `-y` flag. + +### Retry on a rejected credential (guards `94b15e8`, `563ff35`) + +- [ ] **7.14** Give a wrong password. Expect: the failure is explained, the + masked prompt **comes back**, and a correct string completes the export. +- [ ] **7.15** Three wrong answers then a right one still works (the loop is + unbounded). +- [ ] **7.16** The retry does **not** re-ask for the output path or the log + directory — those were settled before the loop. +- [ ] **7.17** A run that never succeeds leaves **no** `exports/` directory and + no partial file. +- [ ] **7.18** Ctrl-C at the re-prompt exits cleanly (`└ Paused`), no loop. +- [ ] **7.19** Agent mode gets the error once, with no retry. + +### Redaction and encoding + +- [ ] **7.20** Every error shows `postgres://***@host/db`. The password never + appears. +- [ ] **7.21** A password containing an **unencoded `@`** + (`postgres://u:pa@ss@host:5432/db`) still redacts to `postgres://***@host/db` — + split on the _last_ `@`, so no fragment leaks. (Guards `5a16189`.) +- [ ] **7.22** That same string **connects**: `normalizeConnectionString` + percent-encodes the userinfo when the raw string will not parse. +- [ ] **7.23** A password with `#`, `/` or `%` — paste a real dashboard-style + password verbatim (`postgres://postgres:aB#c%92^d@db…/postgres`). +- [ ] **7.24** An **already-encoded** password (`p%40ss`) is _not_ double-encoded. +- [ ] **7.25** `--verbose` output never contains a password or an authToken. + +### Connection-failure hints + +Trigger each and confirm the specific text, not a raw driver error: + +- [ ] **7.26** Unreachable host/port → + `Could not reach the database. Check the host and port in the connection string.` + plus the Supabase-specific IPv4/pooler advice. +- [ ] **7.27** Wrong password → + `The database rejected those credentials. Check the user and password…` +- [ ] **7.28** Wrong database / no permission → + `The auth.users table was not readable. It is created automatically when Supabase Auth is enabled.` + plus "connect as the `postgres` role". +- [ ] **7.29** Anything else → the generic + `Check the connection string, that the server is running, and that it is reachable from here.` + +### Edge cases + +- [ ] **7.30** Zero users: an empty JSON array is still written, output is + `No users found to export. Wrote an empty file to .`, **no** + coverage table and **no** next-steps block. +- [ ] **7.31** Exactly one user: "Exported 1 user to …" (singular). +- [ ] **7.32** Pointing `--db-url` at MySQL or SQLite fails in the driver — the + Supabase query is Postgres-only (`->>`). Confirm the error is legible. +- [ ] **7.33** `logs/export-.log` is written with one NDJSON line per + user and shows up in `clerk migrate logs list`. +- [ ] **7.34** Next steps name the import command with the transformer and the + file path relative to cwd. + +--- + +## 8. Export — Auth.js + +`clerk migrate export authjs`. Auth.js core stores **no passwords**. + +**Setup:** an Auth.js/NextAuth database. Test on at least two of Postgres, +MySQL and SQLite — the table-name fallback behaves differently per engine. + +- [ ] **8.1** Postgres, Prisma-style capitalized table: + `clerk migrate export authjs --db-url "postgres://…"`. Output says + `Read N rows from User.` +- [ ] **8.2** Drizzle-style lowercase table → `Read N rows from user.` +- [ ] **8.3** A `users` table → `Read N rows from users.` + (Fallback order is **`User`, then `user`, then `users`**.) +- [ ] **8.4** No user table at all → + `No Auth.js user table found. Tried User, user, users.` with the last driver + message appended. +- [ ] **8.5** A **permission** error on the first candidate aborts immediately — + it must not silently try the other two and report "no table found". +- [ ] **8.6** A schema whose user table lacks a `name` column: note what happens. + Postgres reports `column "name" does not exist`, which matches the + missing-table regex, so all three candidates get tried and you are told + `No Auth.js user table found`. **Judgement call — decide whether this is + acceptable or a bug.** +- [ ] **8.7** MySQL (case-insensitive table lookup): candidate `User` may match a + `user` table, so the reported name can differ from the real schema. Confirm it + is not misleading enough to matter. +- [ ] **8.8** Coverage rows are email / verified email / name. +- [ ] **8.9** With ≥1 user, the run warns + `Auth.js core stores no passwords — its users sign in with OAuth or email links…` +- [ ] **8.10** With 0 users that warning is **suppressed**. +- [ ] **8.11** `email_verified` is a nullable **timestamp**, not a boolean, and is + omitted when falsy. A `Date` becomes an ISO string. +- [ ] **8.12** `AUTHJS_DB_URL` resolution, prompt, agent-mode error and retry loop + behave exactly as in §7.6–7.19 (substitute the env var and the hint line + `Postgres, MySQL, libsql://… or a SQLite file — whichever your Auth.js adapter uses.`). + +--- + +## 9. Export — Better Auth + +`clerk migrate export betterauth`. Detects plugin columns from the schema. + +**Setup:** a Better Auth database. Ideally one install with plugins +(username / admin / phone-number / two-factor) and one without. + +- [ ] **9.1** `clerk migrate export betterauth --db-url "./db.sqlite"` on a plain + install prints `No plugin columns detected; exporting the core user fields.` +- [ ] **9.2** With plugins enabled, prints + `Detected plugin columns: username, banned.` — listing only what is actually + in the schema. Probed columns are `username`, `displayUsername`, + `phoneNumber`, `phoneNumberVerified`, `role`, `banned`, `banReason`, + `banExpires`, `twoFactorEnabled`. +- [ ] **9.3** That line prints on **every** run, including a zero-user export. +- [ ] **9.4** **OAuth-only user is still exported**, with no `password_hash` — + the join onto the credential account is a `LEFT JOIN` with + `providerId = 'credential'` in the `ON` clause, not the `WHERE`. +- [ ] **9.5** A user with **both** an OAuth account row and a credential row + appears **once**, with the password. +- [ ] **9.6** A user with two credential account rows — check whether they + duplicate. (Untested in the suite; probe it.) +- [ ] **9.7** Field renames on write: `id→user_id`, `emailVerified→email_verified`, + `phoneNumber→phone_number`, `phoneNumberVerified→phone_number_verified`, + `displayUsername→display_username`, `createdAt→created_at`, + `updatedAt→updated_at`. +- [ ] **9.8** Coverage rows: email / verified email / password hash / name / + username / phone number. +- [ ] **9.9** **Non-default Postgres schema.** Plugin detection is scoped to + `current_schema()` but the SELECT uses unqualified `"user"`/`"account"` + resolved through `search_path`. Put Better Auth's tables in another schema on + the search path and confirm whether plugin columns are silently dropped. +- [ ] **9.10** **MySQL password round-trip.** `password` is a binary column; + Bun < 1.3.6 decoded it lossily. Export from MySQL, then import into Clerk, and + confirm the migrated user can actually sign in with their old password. +- [ ] **9.11** MySQL 8 upper-cases `COLUMN_NAME` in `information_schema` — + detection must still work. +- [ ] **9.12** `BETTERAUTH_DB_URL` resolution, prompt, agent mode and retry behave + as §7.6–7.19. + +### SQLite and libsql/Turso (guards `4bf8048`) + +These apply to all three DB platforms; test them here. + +- [ ] **9.13** Relative SQLite path: `--db-url "./db.sqlite"`. +- [ ] **9.14** `file:` URL: `--db-url "file:./db.sqlite"` and + `--db-url "file:/abs/path.db"`. +- [ ] **9.15** Query suffix is stripped: `--db-url "./db.sqlite?mode=ro"`. +- [ ] **9.16** A missing SQLite file fails at **connect** time with + `Could not open the SQLite file. Check the path, and that the file exists and is readable.` + — not mid-export. +- [ ] **9.17** The file is opened **readonly**. Exporting from a DB you can read + but not write must work. A WAL-mode DB may need its `-wal`/`-shm` siblings. +- [ ] **9.18** **Turso:** `--db-url "libsql://app-org.turso.io?authToken=…"` works + and does **not** try to open a local file. +- [ ] **9.19** Token from `TURSO_AUTH_TOKEN` when the URL has none. +- [ ] **9.20** Token from `LIBSQL_AUTH_TOKEN`. +- [ ] **9.21** Self-hosted sqld with auth disabled needs **no** token. +- [ ] **9.22** A **typo'd Turso database name** returns 404 and gets its own hint: + `No database at that libsql host. Check the database name in the URL — `turso db show ` prints the URL to use.` + — not the generic "check the host" advice. (This is the specific case + `94b15e8` called out.) +- [ ] **9.23** A **bad token** returns 401 with + `The libsql server rejected that token. Append ?authToken=… to the URL, or set TURSO_AUTH_TOKEN…` +- [ ] **9.24** The authToken is redacted in every error and in `--verbose`: + `libsql://app.turso.io?authToken=***`. +- [ ] **9.25** **BigInt ids.** A libsql/SQLite table with an integer id past + 2^53 (`9007199254740993`) decodes to a `BigInt`, which `JSON.stringify` + throws on. Confirm what the user sees — the error is **not** a `CliError`, so + it escapes the retry loop and may surface raw. **Likely bug; verify.** + +### URL-shape validation (all three platforms) + +- [ ] **9.26** Accepted: `postgresql://`, `postgres://`, `mysql://`, `mysql2://`, + `libsql://`, anything starting `file:`, anything ending `.sqlite` / `.sqlite3` + / `.db`, anything starting `./`. +- [ ] **9.27** `mysql2://` passes validation and is handed verbatim to `Bun.sql`. + Confirm Bun actually accepts that scheme, or that the failure is legible. +- [ ] **9.28** Bare `postgres://` (no host) is **rejected** by the validator. +- [ ] **9.29** `/var/lib/app/mydb` (absolute, no extension) is **rejected** as + "does not look like a connection string" even though it is a valid SQLite + file. **Judgement call — decide whether this is acceptable.** +- [ ] **9.30** `http://x/y.db` is accepted and treated as a _filename_, failing + with "Could not open the SQLite file". Confirm the message is not confusing. +- [ ] **9.31** An invalid `--db-url` gives the usage error + `--db-url does not look like a connection string. Expected postgres://…, mysql://…, libsql://… or a SQLite file path.` + followed by `If the password contains @, # or /, URL-encode it.` + +### Performance + +- [ ] **9.32** Export ~100k rows. Everything is buffered in memory and + serialized with 2-space indent in a single write; libsql pulls the whole + result set in one HTTP response. Note peak memory and wall time. + +--- + +## 10. Import — core + +`clerk migrate import` reads an export, maps it onto Clerk's user schema, +validates every record, and creates the users through the Backend API. + +**Setup:** a throwaway Clerk **development** instance, and a small JSON export +(3–5 users) plus the same data as CSV. + +### Targeting and the sign-in gate (guards `0faebbc`) + +`ensureImportTarget` is the **first** thing the command does — before the wizard, +so you never pick a platform and a file only to be told to log in. + +- [ ] **10.1** Signed out, unlinked directory, **non-interactive**: fails with + ``Not logged in, so there is no Clerk instance to import into. Run `clerk auth +login`, then `clerk link`.`` and makes **zero** HTTP requests. +- [ ] **10.2** Signed out, **interactive**: prints `Not logged in. Signing in +first...` and runs the login flow inline, then continues. +- [ ] **10.3** Signed in but unlinked, interactive: prints `This directory isn't +linked to a Clerk application. Linking one first...` and runs `clerk link`. +- [ ] **10.4** Signed in, unlinked, non-interactive: the standard + `No secret key found. Provide one via: …` error. +- [ ] **10.5** Each of these short-circuits the gate entirely: `--secret-key`, + `--app`, `CLERK_SECRET_KEY`, or a local keyless key + (`.env.local` / `.clerk/.tmp/keyless.json`). +- [ ] **10.6** Instance type is read **from the key**: `sk_live_…` → production, + anything else → development. Confirm via the throughput defaults (§14). + +### Required flags and usage errors + +- [ ] **10.7** `-t/--transformer` only accepts the seven registry keys. + `-t okta` → Commander: `error: option '-t, --transformer ' +argument 'okta' is invalid. Allowed choices are clerk, auth0, …` +- [ ] **10.8** Missing `--file` → `Missing required option --file (path to a +JSON or CSV export).`, exit 2. +- [ ] **10.9** Nonexistent file → `File not found: `, exit 1. +- [ ] **10.10** Wrong extension (`users.txt`) → `Unsupported file type for +. Provide a .json or .csv file.`, exit 2. `users.CSV` **works** + (case-insensitive). A file with no extension is unsupported. +- [ ] **10.11** `--transformer` **and** `--transformer-file` together → + `--transformer and --transformer-file both name a transformer. Pass one or the +other.` before any request. +- [ ] **10.12** `--firebase-rounds abc` → `Invalid --firebase-rounds value +"abc". Must be an integer.`; `--firebase-rounds 0` → `Must be >= 1.` +- [ ] **10.13** JSON that is not a top-level array → + `Expected users.json to contain a JSON array of users, got object.` + (Exception: a Firebase file may be `{ users: [...] }`.) + +### CSV vs JSON coercion + +Run the **same data** as JSON and as CSV and diff the resulting Clerk users. + +- [ ] **10.14** Arrays: `a@x.dev,b@x.dev` and `["a@x.dev"]` and `a|b` all become + arrays. Entries trimmed, empties filtered, whitespace-only → field **deleted**. +- [ ] **10.15** Booleans: `true/1/yes/y` → true; `false/0/no/n` → false, + case-insensitive. A CSV `"false"` must be read as **false**, not as a + non-empty string. +- [ ] **10.16** Metadata columns holding JSON are parsed; `""` and null are + **deleted**, not sent as null; an unparseable string stays a string and fails + validation. +- [ ] **10.17** Dates: a `Date`, epoch-ms number, or parseable string → ISO 8601. + `"yesterday"` is left as-is and fails validation. Empty → deleted. +- [ ] **10.18** `createOrganizationsLimit`: numeric string → number; empty → + deleted; `1.5` fails validation (must be an integer). +- [ ] **10.19** Unmapped source fields pass through unchanged; values equal to + `""`, `"{}"` or `null` are dropped. +- [ ] **10.20** Firebase CSV (headerless) gets its headers prepended + automatically and imports. +- [ ] **10.21** `--transformer clerk` only: `email` + `emailAddresses` are merged + and deduped into one `email` array, and anything already verified is removed + from `unverifiedEmailAddresses` (the key disappears if nothing is left). Same + for phones. + +### Validation + +- [ ] **10.22** A user with **no identifier at all** (no email, phone, unverified + variant, or username) is logged as a validation failure and skipped — + `username: ""` and `email: []` do **not** count. +- [ ] **10.23** `password` without `passwordHasher` fails validation. +- [ ] **10.24** An invalid email **anywhere in the array** fails the whole user. +- [ ] **10.25** Validation failures are logged and the run **continues**, with a + warning: `3 users failed validation and will be skipped. See /logs/import-.log.` +- [ ] **10.26** **Unrecognized hasher aborts the whole run**, exit 2, **zero API + calls**: `Invalid password hasher "rot13" on user u1 (row 1). Expected one of: +argon2i, argon2id, awscognito, …` +- [ ] **10.27** All 21 hashers are accepted. Spot-check `scrypt_firebase`, + `bcrypt`, `ldap_ssha`, `sha512_symfony`. + +### `--resume-after` + +- [ ] **10.28** `-r ` drops everyone up to **and including** that ID + and prints `Resuming after ( skipped).` It matches the **source** ID + (the future `external_id`), not a Clerk ID. +- [ ] **10.29** Matching the **last** user leaves an empty set → + `No users left to import.` and a clean exit 0. +- [ ] **10.30** **An ID that is not in the file aborts**, exit 2: + `Could not find user ID "zz" in the import file.` — it must not silently + re-import everything. + +### `--require-password` + +- [ ] **10.31** Keeps only users with a truthy `password` and prints + `--require-password: skipping 1 user without a password.` (pluralized). +- [ ] **10.32** It also flips `skipPasswordRequirement` to **false**, so a + passwordless user that sneaks through is rejected by Clerk rather than created + without a password. Verify on the wire. + +### `--skip-unsupported-providers` (Supabase only) + +- [ ] **10.33** With a **non-Supabase** transformer: no-op plus + `--skip-unsupported-providers only applies to supabase exports; ignoring.` +- [ ] **10.34** Spinner `Checking enabled providers...` → `GET /v1/domains` → + FAPI `/v1/environment`. +- [ ] **10.35** Settings unreadable → **nobody dropped**, warning `Could not read +the instance's enabled providers; importing every user. Re-run with --verbose +for details.` +- [ ] **10.36** All providers enabled → `Every provider in this export is enabled +in Clerk; no users skipped.` +- [ ] **10.37** Some disabled but everyone has another route → `discord, github +not enabled in Clerk, but every user has another way to sign in; none skipped.` +- [ ] **10.38** A user whose **entire** provider list is disabled is skipped: + `--skip-unsupported-providers: skipping 1 user whose only provider is not +enabled in Clerk (discord: 1).` +- [ ] **10.39** `email`, `phone` and `anonymous_users` never count as disabled. A + user with no provider data is never dropped. +- [ ] **10.40** Provider aliases map correctly: `azure→oauth_microsoft`, + `twitter→oauth_x`, `slack_oidc→oauth_slack`, `fly→oauth_fly`. +- [ ] **10.41** **The flag is persisted** for the next run — and is persisted + **even on a non-Supabase transformer** where it did nothing. Check + `clerk migrate settings` afterwards. **Suspected bug; confirm.** + +### Additional identifiers + +- [ ] **10.42** Only the **first** verified email and phone go on + `POST /v1/users`. Every additional verified identifier, and every unverified + one, is attached afterwards via `POST /v1/email_addresses` / + `POST /v1/phone_numbers` with `primary: false`. +- [ ] **10.43** Unverified lists are deduped **and** filtered against the + verified set — an already-verified address is never re-attached as unverified. +- [ ] **10.44** An attachment failure is **logged and the user still counts as + imported**. Log line has `status: "additional_email_error"` / + `"additional_phone_error"`. +- [ ] **10.45** A user identified only by `username` produces no + `email_address` / `phone_number` key at all. +- [ ] **10.46** `external_id` is **always** set to `userId`. This is what makes + the migration re-runnable and what `migrate delete` matches on. Verify on + every created user. +- [ ] **10.47** Every other field is **omitted when absent**, never sent as null. + +### Summary, next steps and exit codes + +- [ ] **10.48** Pre-import line: `Importing 2 users via the clerk transformer +into My App (development).` With `--secret-key` it reads `… into the resolved +instance (dev).` +- [ ] **10.49** Summary block shows Total / Imported / Failed / Failed validation + (the last only when > 0), an `Error breakdown:` grouped by normalized message, + and `Log: `. +- [ ] **10.50** Error normalization: `["last_name" "first_name"]` and + `["first_name" "last_name"]` group into **one** bucket. +- [ ] **10.51** **Blocked-SMS note:** an error containing `Phone numbers from +this country` adds a dev-specific note pointing at Clerk's test phone numbers + (prod gets the Dashboard SMS settings link instead). Neither says "contact + support" first. (Guards `0f35744`.) +- [ ] **10.52** **Quota note:** `You have reached your limit of N users` on a dev + instance adds the development-quota explanation. Nothing on prod. +- [ ] **10.53** Clean run next steps: `logs list` + `migrate delete`. +- [ ] **10.54** Run **with failures**: the first step becomes + ``Run `grep '"status":"error"' ` to see every user that failed and +why``. (Guards `0f35744`.) +- [ ] **10.55** Exit codes: 0 on success and on every declined prompt; **1** when + any user failed; **2** for usage errors; 130 for Ctrl-C. +- [ ] **10.56** `/import-.log` is NDJSON with synchronous appends + — kill the run with Ctrl-C mid-import and confirm every processed user is + still recorded, then resume with `--resume-after `. +- [ ] **10.57** A log **write failure** never aborts the import — it warns + `Could not write migration log: ` and continues. + +--- + +## 11. Import — interactive wizard + +Triggered when `--transformer` **or** `--file` is missing, in a human TTY. + +- [ ] **11.1** Prompt order: platform → file → (Firebase only) signer key → salt + separator → rounds → mem cost. +- [ ] **11.2** `Which platform are you migrating from?` lists all seven with + descriptions truncated to 96 chars. +- [ ] **11.3** **Prefill from the last run:** the saved `transformer` and `file` + are offered as defaults. Run an import, then run `clerk migrate import` again — + it should be mostly pressing Enter. +- [ ] **11.4** A **stale** saved transformer (edit the config to `okta`) yields + **no** default rather than an error. +- [ ] **11.5** Nothing saved → both defaults undefined. +- [ ] **11.6** Anything already passed as a flag is **not** asked for. +- [ ] **11.7** File prompt validation, re-prompting in place: empty → + `A file path is required`; missing → `File not found: `; wrong + extension → `Provide a .json or .csv file`. +- [ ] **11.8** **Firebase hash parameters are never prefilled** — the signer key + is a secret and is never written to disk. Confirm after a Firebase run that + the next run still asks. +- [ ] **11.9** Choosing Firebase prints two info lines: where to find the hash + parameters, and which four env vars skip the prompts next time. +- [ ] **11.10** Leaving the signer key **blank** stops the remaining three + questions (correct for a password-free export). +- [ ] **11.11** Salt separator validation: `Required alongside the signer key`. +- [ ] **11.12** Rounds / mem cost validation: `Enter a positive whole number` — + rejects `0`, `-1`, `1.5`, `many`. +- [ ] **11.13** Choosing a **non-Firebase** platform never reads any Firebase env + var at all. (Guards `85afd33`.) +- [ ] **11.14** **Agent mode never prompts:** bare `clerk migrate import --mode +agent` exits 2 with `` `clerk migrate import` is interactive and cannot prompt +in agent mode. Pass --transformer and --file .`` — and names + only the **missing** half when one is supplied. + +--- + +## 12. Import — Firebase hash parameters + +All four are required **as a set**. A partial set produces a well-formed digest +that verifies against nothing. + +| Flag | Primary env var | Aliases (lower priority) | +| --------------------------- | ------------------------------- | --------------------------------------------------------- | +| `--firebase-signer-key` | `CLERK_FIREBASE_SIGNER_KEY` | `FIREBASE_BASE64_SIGNER_KEY`, `BASE64_SIGNER_KEY` | +| `--firebase-salt-separator` | `CLERK_FIREBASE_SALT_SEPARATOR` | `FIREBASE_BASE64_SALT_SEPARATOR`, `BASE64_SALT_SEPARATOR` | +| `--firebase-rounds` | `CLERK_FIREBASE_ROUNDS` | `FIREBASE_ROUNDS`, `ROUNDS` | +| `--firebase-mem-cost` | `CLERK_FIREBASE_MEM_COST` | `FIREBASE_MEM_COST`, `MEM_COST` | + +- [ ] **12.1** All four as flags: import succeeds. +- [ ] **12.2** All four as `CLERK_FIREBASE_*` shell variables. +- [ ] **12.3** All four via the **Firebase-native** aliases in `.env` + (`base64_signer_key`-style names: `BASE64_SIGNER_KEY`, `ROUNDS`, …). This is + what every Firebase guide tells people to paste. (Guards `f9c0b08`.) +- [ ] **12.4** Resolution order **per parameter independently**: flag → exported + shell var → `.env.clerk-migrate` → `.env.local` → `.env`. Mix sources and + confirm with `--verbose`. +- [ ] **12.5** A `CLERK_FIREBASE_*` variable **beats** its alias in the same + file: `ROUNDS=99` + `CLERK_FIREBASE_ROUNDS=8` → `8`. +- [ ] **12.6** An empty variable (`CLERK_FIREBASE_SIGNER_KEY=""`) counts as unset. +- [ ] **12.7** **Partial set from flags → hard usage error**, exit 2, naming + every missing flag at once, raised **before anything is read** (zero requests): + `The Firebase hash parameters must be supplied together. Missing: +--firebase-salt-separator, --firebase-rounds, --firebase-mem-cost.` +- [ ] **12.8** **Partial set from env/files only → warning, run continues** + without passwords: `Ignoring an incomplete Firebase hash configuration (no +--firebase-salt-separator, …). Run `clerk migrate settings` to see what is +set.` (Guards `18cef5e`.) +- [ ] **12.9** **Mixed** (one flag + partial env) still **throws**. +- [ ] **12.10** **Gated on the transformer** (guards `85afd33`): leave a complete + or partial `CLERK_FIREBASE_*` set in `.env.clerk-migrate`, then run a + **Supabase** import. It must not warn, must not fail, and must not read those + variables at all. Even explicit `--firebase-*` flags on a Supabase run are + silently ignored. +- [ ] **12.11** A Firebase export that **has** hashes but no config resolved + fails with `This export contains Firebase password hashes, which need the +project's hash parameters to import.` plus the console path and the four flags. +- [ ] **12.12** **Digest format on the wire:** + `password_digest = hash$salt$signerKey$saltSeparator$rounds$memCost`, e.g. + `SGFzaA==$U2FsdA==$SIGNER$Bw==$8$14`, with + `password_hasher: "scrypt_firebase"`. +- [ ] **12.13** **Never persisted:** after a Firebase import, confirm the CLI + config and `.env.clerk-migrate` contain none of the four. +- [ ] **12.14** **End to end:** export from Firebase, import with the printed + command, then sign in as a migrated user with their **original password**. + This is the only test that proves the digest is right. + +--- + +## 13. Import — readiness report + +Printed immediately before the confirmation prompt. Skipped **only** for `-y`. +Reads BAPI `GET /v1/domains` → that instance's FAPI `GET /v1/environment`. + +### The report itself + +- [ ] **13.1** Header `Migration readiness`, then `N users in this file`. +- [ ] **13.2** `N failed validation and will be skipped` appears only when > 0. +- [ ] **13.3** `N without any identifier — cannot be imported` appears only when + > 0. +- [ ] **13.4** **The outcome block classifies each user exactly once**, worst + outcome first — the ✗ / ⚠ / ✓ totals must **add up to the file**. Build a file + where users overlap (some missing email AND password) and check the arithmetic. +- [ ] **13.5** A user rejected for a missing email is **not** also counted under + the missing password they happen to share. +- [ ] **13.6** `If you import them, this applies to them too:` appears nested + under the ✗ block, naming what is masked behind a rejection. +- [ ] **13.7** The password reason appends ` — they will have to reset it to +sign in` (a required password does **not** reject a user; the import sends + `skip_password_requirement`). +- [ ] **13.8** Section blocks in fixed order, only when non-empty: `Identifiers`, + `Authentication`, `Social connections`, `User model`. +- [ ] **13.9** Row forms render correctly: blocking-required + (`⚠ Email — required in Clerk, and not every user has one — 108/120 users`), + blocking-disabled (`⚠ Username — not enabled in Clerk — 6/120 users`), fine + (`✓ Email — enabled in Clerk — all users`). +- [ ] **13.10** Coverage reads `all users` when every user has the field. +- [ ] **13.11** **Section rows do not restate user counts** in a way that + contradicts the outcome block. +- [ ] **13.12** Footer is either `⚠ 3 settings need attention` + the dashboard + link, or `✓ Every field in this file is configured in Clerk`. Singular form + is `1 setting needs attention`. +- [ ] **13.13** **Social rows never contribute to the per-user outcome counts** — + provider data lives in the raw export, not the transformed user. +- [ ] **13.14** A required attribute that **every** user has does not block. An + enabled-but-optional attribute some users lack does not block. + +### Degraded mode + +- [ ] **13.15** Make the settings unreadable (bad key, or block FAPI). The report + degrades to coverage-only with + `! Could not read this instance's settings, so the checks below are coverage +only.` plus the dashboard link. +- [ ] **13.16** In degraded mode **nothing is flagged**, no multiselect is + offered, the `✓ Every field…` line is **not** printed, and the import proceeds. +- [ ] **13.17** `--verbose` shows the reason the settings could not be read. + +### The settings-change offer (guards `e9a6692`) + +- [ ] **13.18** Human TTY: `Update this instance's settings first? (enter to +skip)` with one option per flagged row, in report order. +- [ ] **13.19** **Nothing is preselected.** +- [ ] **13.20** The multiselect footer reads + `↑/↓ to navigate • Space: select • a: all • Enter: confirm`. Press `a` — every + option toggles. (Guards `68c7057`.) +- [ ] **13.21** Selecting nothing continues to the import prompt with the + instance **untouched** and **no PATCH sent**. +- [ ] **13.22** A real selection sends **one** `PATCH +/v1/platform/applications/{app}/instances/{ins}/config` and reports + `Updated 1 setting.` / `Updated 2 settings.` +- [ ] **13.23** Changes sharing a parent **merge into one branch** — select both + name fields and confirm the payload is + `{ user_model: { first_name: { required: false }, last_name: { required: false } } }`. +- [ ] **13.24** **Email and phone take TWO writes.** Enabling Email writes + `auth_email.used_for_sign_up = true` **and** + `auth_email.verification_strategies = ["email_code"]`. Without the second, + Clerk answers `422 … verifiable attributes need to have at least one +verification`. Same for phone with `phone_code`. +- [ ] **13.25** Username, password and the name fields take **one** write each. +- [ ] **13.26** Every offer label matches the table: `Make Email optional at +sign-up`, `Enable Discord sign-in`, etc. No config paths are shown. +- [ ] **13.27** **The redraw is computed from the write, not a re-fetch.** + Confirm exactly **one** `/v1/environment` request across a fix + redraw — a + second read would show stale settings and re-flag everything you just cleared. +- [ ] **13.28** **The offer repeats** while anything is still flagged, and round + 2 offers only what is left. +- [ ] **13.29** The loop ends when nothing is flagged, when you select nothing, + or when nothing is offerable for the remaining rows. +- [ ] **13.30** After the last write the footer flips to + `✓ Every field in this file is configured in Clerk`. + +### Stand-downs + +- [ ] **13.31** `-y` skips the report **entirely** — no `/v1/domains`, no + `/v1/environment`. Confirm with `--verbose`. +- [ ] **13.32** **Agent mode without `-y` still prints the report** and then + imports **with no confirmation at all**. Confirm this is intended — it is the + combination most likely to surprise. +- [ ] **13.33** Agent mode never gets the settings offer. +- [ ] **13.34** **Unresolvable instance** (bare `--secret-key` in an unlinked + dir): a warning, not a failure — `Could not resolve which instance to +configure, so nothing was changed. Link a project with `clerk link`, or pass +`--app `.` and the import continues. +- [ ] **13.35** **Keyless application:** stands down with `These settings need an +account to change. Run `clerk auth login` to claim this application…` and the + import continues. +- [ ] **13.36** Declining the final confirm after applying settings changes + leaves the **settings changed** but writes **no users**. Confirm that is + understood/acceptable. + +--- + +## 14. Import — throughput and the dev user limit + +### Rate limiting and concurrency + +| Instance | rate limit (req/s) | concurrency | +| ------------------------ | ------------------ | ----------- | +| production (`sk_live_…`) | 100 | 9 | +| development | 10 | 1 | + +- [ ] **14.1** Defaults are derived from the key prefix. Verify the pacing on a + dev instance feels like ~10 req/s. +- [ ] **14.2** `CLERK_MIGRATE_RATE_LIMIT=50` changes the pacing. +- [ ] **14.3** `CLERK_MIGRATE_CONCURRENCY_LIMIT=5` changes in-flight requests. +- [ ] **14.4** Concurrency defaults from the **resolved** rate limit, so + `CLERK_MIGRATE_RATE_LIMIT=1000` alone yields concurrency 95. +- [ ] **14.5** **Invalid values are silently ignored** — non-numeric, `0`, + negative, `NaN`, `Infinity`, empty all fall back to the default with **no + warning**. The repo's own debug-logging rule argues for a `log.warn` here. + **Confirm and decide whether to file.** +- [ ] **14.6** Rate limiting applies **per API call**, not per user — a user with + 10 extra emails cannot burst past the limit. + +### 429 handling + +- [ ] **14.7** A 429 backs off and retries up to **5 times** (6 attempts total). +- [ ] **14.8** `Retry-After` is honoured when finite and > 0; otherwise the body's + `retryAfter`; otherwise 10s. `Retry-After: 0` and `Retry-After: soon` both fall + back to the default. +- [ ] **14.9** Each backoff writes an NDJSON line with + `"status":"429_retry"` and `Rate limit hit (429), retrying in 1s (attempt 1/5)`. +- [ ] **14.10** Exhausted retries record the user as failed with `code: "429"` + and message `Rate limit exceeded after 5 retries`. +- [ ] **14.11** Progress spinner reads + `Importing users: [3/10] (2 succeeded, 1 failed)...`. + +### The development-instance user limit (guards `0f35744`) + +- [ ] **14.12** Fits within headroom (`existing + incoming <= 100`): **totally + silent**, no warning, no prompt. +- [ ] **14.13** Exceeds headroom: spinner `Checking the instance's user +count...`, `GET /v1/users/count`, then a warning naming what the instance + already holds and roughly how many will be rejected. +- [ ] **14.14** The count is **unreadable**: the `, and this one already holds N` + clause is omitted and nothing errors. +- [ ] **14.15** **It is a warning and a prompt, not a refusal** — the old + behavior was a hard refusal at 500. Human TTY gets `Continue anyway, expecting +about 1 user to be rejected?` defaulting to **No**. +- [ ] **14.16** Declining exits **0** with no error and **nothing read from + FAPI** — the readiness report never runs. +- [ ] **14.17** `-y` and agent mode **proceed on the warning alone**. +- [ ] **14.18** The final confirm restates the split: + `Import 1 user and expect 1 to fail?` rather than a number the instance will + not take. +- [ ] **14.19** **Production instances never make the count call at all.** +- [ ] **14.20** Dev instances **still make the count call under `-y`**, unlike + the readiness report which `-y` skips entirely. Confirm this asymmetry is + intended. +- [ ] **14.21** Actually exceed the limit and confirm the error breakdown shows + `You have reached your limit of N users` with the development-quota note. + +--- + +## 15. Import — custom transformers + +`clerk migrate import --transformer-file ./my-platform.ts --file users.json` + +Write a file like this to test with: + +```ts +export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform's admin console.", + transformer: { + account_ref: "userId", // required: becomes external_id + contact_email: "email", + given: "firstName", + pw_bcrypt: "password", + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { + if (!user.firstName) delete user.firstName; + }, +}; +``` + +- [ ] **15.1** A valid file loads and prints ``Loaded the `myplatform` +transformer from ./my-transformer.ts.`` +- [ ] **15.2** `defaults` and `postTransform` are honoured end to end. +- [ ] **15.3** TypeScript works — `interface`, `satisfies`, `as const` all load + (Bun transpiles at runtime). Plain `.js` works too. **Test this against the + compiled binary**, not just `bun run dev`. +- [ ] **15.4** Omitting `description` defaults to `Custom transformer`. +- [ ] **15.5** A custom key is **not** added to `--transformer`'s choices — + selection is by path only. + +### Every validation message + +- [ ] **15.6** Path missing → `No transformer file at /abs/path.ts.` +- [ ] **15.7** Path is a directory → `/abs/path is a directory, not a transformer +file.` +- [ ] **15.8** Syntax error → `Could not load ./f.ts: ` + `The file must +be valid JavaScript or TypeScript that this CLI can import.` +- [ ] **15.9** No default export but a named one → ``./f.ts has no default +export. Found named export `myPlatform` — did you mean `export default`?`` + (plural + comma list for several). +- [ ] **15.10** No default and nothing else → `./f.ts has no default export.` +- [ ] **15.11** Default is not an object → `… the default export is string, not +an object`. +- [ ] **15.12** Missing/blank `key` or `label` → ``… `key` must be a non-empty +string``. +- [ ] **15.13** `description` not a string → ``… `description` must be a string +when present``. +- [ ] **15.14** `transformer` missing / not an object / an array → ``… +`transformer` must be an object mapping source fields to Clerk fields``. +- [ ] **15.15** A mapping value that is not a string → ``… `transformer.account_ref` +must map to a Clerk field name, got number``. +- [ ] **15.16** **Nothing maps to `userId`** → ``… no source field maps to +`userId`. Every user needs one — it becomes the Clerk user's external_id``. + **This is the load-bearing check** — without it every user imports with no + `external_id` and `migrate delete` has nothing to match on. +- [ ] **15.17** `defaults` not a plain object → ``… `defaults` must be an object +when present``. +- [ ] **15.18** A hook that is not a function → ``… `preTransform` must be a +function when present``. +- [ ] **15.19** `key` collides with a built-in → ``… `key` is "clerk", which is +already a built-in transformer. Choose another key``. +- [ ] **15.20** Every message is prefixed with the path **as typed**, and all of + them fire **before any HTTP request**. +- [ ] **15.21** **Schema fields are exact.** Add a field to your transformer that + is not in the schema and confirm it is silently dropped (Zod strips unknown + keys) and never reaches Clerk. + +--- + +## 16. Transformers list + +- [ ] **16.1** `clerk migrate transformers` defaults to `list`. +- [ ] **16.2** **No gutter** — `┌` / `└` must not appear even in human mode + (this reads a static registry, it does not run anything). (Guards `f0ba767`.) +- [ ] **16.3** Output has the lead sentence, a `Transformers:` heading, seven + entries in registry order (`clerk, auth0, authjs, betterauth, firebase, +supabase, workos`), the count line, and the write-your-own hint. +- [ ] **16.4** Labels are `Clerk`, `Auth0`, `Auth.js (NextAuth)`, `Better Auth`, + `Firebase`, `Supabase`, `WorkOS`. +- [ ] **16.5** The count line says **`7 built-in transformers`**, and matches + the example block in `migrate/README.md`. +- [ ] **16.6** **Wrapping:** width is `min(terminal columns, 80)`. Resize + narrower than 80 → rewraps. Resize wider → stays capped at 80, so two runs lay + out identically. +- [ ] **16.7** A backticked span is **never split across lines** — a split would + leave an unmatched backtick colouring the wrong half of both lines. +- [ ] **16.8** An over-long single word gets its own line rather than being + dropped. +- [ ] **16.9** Descriptions are **not dimmed** — they are the whole point. +- [ ] **16.10** `--transformer-file ./my.ts` appends the custom entry as + `mykey My Label (custom — ./my-transformer.ts)`, the count becomes `7 +built-in transformers plus 1 loaded from --transformer-file`, and the + "Migrating from something else?" hint disappears. +- [ ] **16.11** A **bad** `--transformer-file` fails the whole command — it does + not degrade to listing the built-ins. +- [ ] **16.12** `--json` emits `key, label, description, built_in`, `source` + (custom only), and `maps_to_user_id` — `id` for clerk/authjs/supabase/workos, + `user_id` for auth0/betterauth, `localId` for firebase. +- [ ] **16.13** `clerk migrate transformers list --json | jq` parses — nothing + but the array on stdout. + +--- + +## 17. Settings + +Eight settings: `transformer`, `file`, `skip-unsupported-providers`, `log-dir`, +`firebase-signer-key`, `firebase-salt-separator`, `firebase-rounds`, +`firebase-mem-cost`. Names are kebab-case and **identical to the `migrate +import` flag** each one backs. + +### `settings list` + +- [ ] **17.1** Bare `clerk migrate settings` lists (read-only default) and never + prompts. +- [ ] **17.2** Both orientation lines print, then the + `SETTING / VALUE / SOURCE / DESCRIPTION` table. +- [ ] **17.3** **Column alignment holds when a value is long.** Set `file` to a + long absolute path and confirm `SOURCE` and `DESCRIPTION` still line up — + padding happens **before** colouring. (Guards `194d8f6`.) +- [ ] **17.4** An unset setting leaves the VALUE cell **empty**, not `—`, and + SOURCE reads `not set`. +- [ ] **17.5** `firebase-signer-key` always renders `[REDACTED]`, whatever its + length — never a truncation like `aVer…3456`. (Guards `b1fec90`, `f9c0b08`.) +- [ ] **17.6** Footer `N of 8 settings set. Credentials are shown redacted.` +- [ ] **17.7** Next steps (human only) name `settings set`, `settings clear +` and `settings clear`. +- [ ] **17.8** `--json` emits `name, store, description, value` (null when unset, + `[REDACTED]` for the secret), `set`, `secret`, `source`. Prose and next steps + suppressed; `| jq` parses. +- [ ] **17.9** Credentials are redacted **under `--json` too**, so the output is + safe to paste into an issue. + +### The SOURCE column (the point of the command) + +Produce each source and confirm the string: + +- [ ] **17.10** `clerk config` — set via `settings set transformer clerk`. +- [ ] **17.11** `.env.clerk-migrate` — set via `settings set firebase-rounds 8`. +- [ ] **17.12** `.env.local` — put `CLERK_FIREBASE_ROUNDS=8` there. +- [ ] **17.13** **`.env.local (ROUNDS)`** — put a bare `ROUNDS=8` in + `.env.local`. The alias is named alongside the file. +- [ ] **17.14** **`ROUNDS env var`** — export `ROUNDS=9` in the shell while + `.env.local` holds `ROUNDS=8`. Attribution is **by value**, so the file that + lost is not credited. This is exactly the case the column exists to catch. +- [ ] **17.15** `log-dir` checks the **environment before the config**: + `CLERK_MIGRATE_LOG_DIR=./x clerk migrate settings` shows `./x` with source + `CLERK_MIGRATE_LOG_DIR env var`, not the saved value. (Guards `1c029c8`.) +- [ ] **17.16** A value from the app's own `.env` is **named** but not editable + by `settings clear` — confirm the listing says where to remove it. +- [ ] **17.17** `firebase-salt-separator` is **not** marked secret and displays + in full while the signer key is redacted. **Confirm this is intended.** + +### `settings set` + +- [ ] **17.18** An env-store setting writes `=` into + `./.env.clerk-migrate`, creating the file, and prints ``Set +`firebase-signer-key` in .env.clerk-migrate (gitignored).`` +- [ ] **17.19** **`.env.clerk-migrate` is added to `./.gitignore` on creation.** + Test with a `.gitignore` that has **no trailing newline** — a newline is added + first, nothing is damaged. Run `set` twice — no duplicate entry. With no + `.gitignore` at all — one is created. (Guards `2f22633`.) +- [ ] **17.20** A config-store setting writes to the CLI config and prints ``Set +`transformer` to firebase for this project.`` +- [ ] **17.21** `skip-unsupported-providers` is stored as a real **boolean**, not + the string `"true"`. +- [ ] **17.22** **Validation runs before any write:** `settings set +firebase-rounds -1` fails with `Invalid value for firebase-rounds: Expected a +positive whole number.` and `.env.clerk-migrate` **is not created**. +- [ ] **17.23** `settings set skip-unsupported-providers maybe` → + `Expected true or false`. +- [ ] **17.24** `settings set log-dir ""` → `Expected a directory path`. +- [ ] **17.25** **File preservation:** existing comments, blank lines and key + order in `.env.clerk-migrate` survive; an existing key is updated in place + with no duplicate; **no `# Clerk` section header is ever added**, however many + times you run `set`. + +### Did-you-mean (guards `9900266`) + +- [ ] **17.26** `clerk migrate settings clear logs-dir` → + `error: command-argument value 'logs-dir' is invalid for argument 'name'. Did +you mean "log-dir"? Allowed choices are transformer, file, …` +- [ ] **17.27** Also fires for `log_dir`, `firebase-round`, `transfomer`. +- [ ] **17.28** An unrelated word (`banana`) gets the plain choices error with + **no** "Did you mean". +- [ ] **17.29** Works on both `set` and `clear`. + +### `settings clear` + +- [ ] **17.30** `settings clear ` clears **both stores** for that setting + and leaves everything else alone. +- [ ] **17.31** `log-dir` is the one that can live in both — clearing it removes + `CLERK_MIGRATE_LOG_DIR` from `.env.clerk-migrate` **and** `logDir` from the + config, so the first-run prompt returns. +- [ ] **17.32** **Aliases are dropped too:** `settings clear firebase-rounds` + removes `CLERK_FIREBASE_ROUNDS`, `FIREBASE_ROUNDS` **and** a bare `ROUNDS` + line. (Clearing one while a bare `ROUNDS` stayed behind would report the + setting cleared and leave the next run reading the old value.) +- [ ] **17.33** Confirm prompt `Clear \`\`?` defaults to **no**. +- [ ] **17.34** Clearing `file` warns first: ``\`clerk migrate delete\` uses the +saved file (users.json) to identify the users the last run created. Clearing +it leaves nothing to undo from.`` +- [ ] **17.35** Success line names the right store(s): `… from +.env.clerk-migrate.` / `… from this project's settings.` / `… from +.env.clerk-migrate and this project's settings.` +- [ ] **17.36** Nothing to clear → ``\`firebase-rounds\` is not set here. A value +coming from the app's own env files or the shell has to be removed there…`` +- [ ] **17.37** `settings clear` (no name) confirms `Clear this project's +migration settings?` default **no**, then clears the config entry and the five + prefixed env vars. +- [ ] **17.38** Output: `Cleared the saved transformer and file.` and `Removed N +credentials from .env.clerk-migrate.` Nothing set at all → `No migration +settings to clear for this project.` +- [ ] **17.39** **`.env.clerk-migrate` is deleted** once no managed lines remain + — even if only comments are left. Unrelated keys (`OTHER=keep`) keep the file + alive and are preserved byte-for-byte. +- [ ] **17.40** `.gitignore` is **not** cleaned up when the file goes. Confirm + that is acceptable. + +### Two suspected gaps — verify and decide + +- [ ] **17.41** **Alias asymmetry on the full clear.** Put a bare `ROUNDS=8` in + `.env.clerk-migrate`, then run `clerk migrate settings clear -y`. It survives, + because the full clear only strips the five `CLERK_*` names while + `clear ` strips aliases too. "Forget them all, credentials included" + should mean all. **Likely bug.** +- [x] **17.42 A bare clear refuses where it cannot ask** (fixed in this + branch). `clerk --mode agent migrate settings clear` must fail with + `… cannot prompt here. Pass -y to confirm.` and change **nothing** — + check the config entry and `.env.clerk-migrate` afterwards. `-y` still + clears. `settings clear ` deliberately still needs no `-y`: naming + the one setting is the confirmation. +- [ ] **18.1** `clerk migrate logs` defaults to `list`. +- [ ] **18.2** **It resolves the log directory but never prompts** — "where + should logs go?" is not a question to ask someone who asked to see the logs + they already have. Run it in a fresh project and confirm no prompt, and that + the setting is **not** saved (the first-run prompt must still fire later on an + import). +- [ ] **18.3** Human mode wraps in the gutter (`┌ Listing migration logs` … + `└ Done`); `--json` prints **outside** the gutter with no `┌`. + (Guards `922890a`.) +- [ ] **18.4** Empty or missing dir → `No migration logs in ./logs.` +- [ ] **18.5** Two prose lines, then `FILE / TYPE / DATE / SIZE / ENTRIES`, + newest first. **The filename leads** — it is what `convert` and `clean` talk + about. (Guards `aa62b62`.) +- [ ] **18.6** **DATE renders the filename's UTC stamp in your own timezone** + (`Feb 1, 2026 at 4:14 AM`). Run twice with different `TZ=` and confirm the + column changes. The raw `2026-01-01T12-00-00` must never appear there. +- [ ] **18.7** Unparseable stamp → the raw stamp; missing stamp → dimmed `—`. +- [ ] **18.8** SIZE: `<1024` → `N B`; `<1 MiB` → `N.N KB`; else `N.N MB`. +- [ ] **18.9** ENTRIES counts non-blank lines, malformed ones included. +- [ ] **18.10** Footer `N log files in ./logs` — **relative** when under cwd, + **absolute** when not. Test both (`CLERK_MIGRATE_LOG_DIR=/tmp/xlogs`). +- [ ] **18.11** **The legend is fixed**, listing `export`, `import` and `delete` + regardless of what is present — "what else could be here" is the other half of + the question. `unknown` is never in the legend. +- [ ] **18.12** Sorting is by timestamp **descending**, then name ascending, and + is **chronological across kinds** — a January `import-` sorts after a February + `delete-`, not grouped by prefix. +- [ ] **18.13** Unrecognised names sort **last**. +- [ ] **18.14** **Legacy filenames still classify** (guards `aa62b62`): create + `migration-.log` → TYPE `import`; `user-deletion-.log` → TYPE + `delete`. Both list, clean and convert unchanged. +- [ ] **18.15** `random.log` / `import.log` (no stamp) → kind `unknown`, empty + date, sorts last, still listed and convertible. +- [ ] **18.16** Only `*.log` is listed — subdirectories and `*.json` are skipped. + An unreadable file still lists with 0 entries. +- [ ] **18.17** `--json` fields: `name, kind, timestamp` (raw), `size_bytes, +entry_count, path` (absolute). Empty dir → `[]`, not prose. + +### `logs clean` + +- [ ] **18.18** Nothing to delete → `No migration logs to clean in .` +- [ ] **18.19** Non-interactive/agent **without `-y`** → usage error, nothing + deleted: `` `clerk migrate logs clean` deletes 2 log files from /abs/logs and +cannot prompt here. Pass -y to confirm.`` +- [ ] **18.20** Interactive: `Delete 2 log files?` default **no**. Declining + closes with `└ Paused` (not `Failed`) and every file remains. +- [ ] **18.21** `-y` deletes and prints `Deleted N log files.` +- [ ] **18.22** **Only `.log` files are touched** — converted `.json` siblings + survive. +- [ ] **18.23** A per-file unlink failure warns `Could not delete : +`, the rest continue, and the exit code is **1**. Reproduce with a + read-only log directory and check `echo $?`. + +### `logs convert` + +- [ ] **18.24** `logs convert --all` converts every file. +- [ ] **18.25** `logs convert import-2026-01-01T12-00-00.log` converts one. +- [ ] **18.26** A path is accepted and matched on **basename only** — + `./logs/import-….log` works. +- [ ] **18.27** Named file not found → `No log file named in /abs/logs.` + and the gutter closes `└ Failed`. +- [ ] **18.28** Non-interactive with no target → usage error naming **both** + alternatives, with two suggestions (`--all`, and the first available filename). +- [ ] **18.29** Human, no args → multiselect `Which log files should be converted +to JSON?`, one option per file with an `N entries` hint, newest first. + **This is the easiest place to see the new `a: all` footer.** +- [ ] **18.30** Selecting nothing aborts with `└ Paused` and writes nothing. +- [ ] **18.31** Output is `.json` beside the original, a pretty-printed + 2-space JSON array. **The original `.log` is left in place.** +- [ ] **18.32** Per-file line: `import-….log → import-….json (3 entries)`, + singular `1 entry`. +- [ ] **18.33** **Malformed line handling:** write a log with a truncated line + (`{"a":1}\n{"b":\n{"c":3}\n`). Expect a warning + `import-….log:2 is not valid JSON and was skipped — `, a trailing + `1 malformed line skipped.`, and JSON output `[{"a":1},{"c":3}]` — the good + entries still convert. +- [ ] **18.34** Line numbers are 1-indexed and **blank lines count toward the + number**. +- [ ] **18.35** Blank lines alone are skipped without any report. +- [ ] **18.36** A write failure on one file warns and sets exit code 1; other + files continue. +- [ ] **18.37** Footer `Converted N log files. Originals left in place.` + +### Where logs go (guards `1c029c8`) + +- [ ] **18.38** Resolution order: `CLERK_MIGRATE_LOG_DIR` (shell, `.env`, + `.env.local`, `.env.clerk-migrate`) → `log-dir` in the CLI config → `./logs`. +- [ ] **18.39** The first-run prompt fires from `import`, every `export +`, and `delete` — and **only** those. +- [ ] **18.40** Pressing Enter counts as an answer: `./logs` is **saved**. +- [ ] **18.41** Asked exactly once per project. +- [ ] **18.42** Agent/non-TTY takes `./logs` and saves **nothing**, so the + question stays open for the first interactive run. +- [ ] **18.43** Env beats config: `logDir: ./saved` in config plus + `CLERK_MIGRATE_LOG_DIR=./env-logs` → logs land in `./env-logs`, and + `settings list` says so. +- [ ] **18.44** The directory is resolved against **cwd**, not the binary. +- [ ] **18.45** `settings set log-dir ` pre-answers it; `settings clear +log-dir` makes the prompt return. + +### Two inconsistencies to confirm + +- [ ] **18.46** **`-y` skips the log-dir prompt** (fixed in this branch). Fresh + project, human TTY, `clerk migrate import -y -t clerk -f users.json` — no + prompt, logs land in `./logs`, and **nothing is saved**: a later run without + `-y` must still ask. Same on every `migrate export -y` and on + `clerk migrate delete -y`. +- [ ] **18.47** `logs clean` and `logs convert` print the **absolute** log dir in + their "nothing to do" messages while `logs list` prints the relative one. + Minor, but inconsistent — **still open.** + +--- + +## 19. Delete + +`clerk migrate delete` is the undo for a bad migration. It is the one command in +this tree that **destroys data in Clerk** (contrast `migrate logs clean`, which +only removes local files). + +**Run an import first**, then test against it. + +- [ ] **19.1** It matches on the `external_id` the import stamped on each user, + read from the **saved migration record** (transformer + file) in the CLI + config for this project. +- [ ] **19.2** It re-reads the export file and runs only the transformer's + **field mapping** — so a Firebase file needs **no** hash parameters to undo. +- [ ] **19.3** Lookup is `GET /v1/users?limit=100&external_id=…`, 100 IDs per + request, with 429 retries. Empty IDs skipped, duplicates deduped. +- [ ] **19.4** **Only users Clerk itself reports as carrying one of this + migration's external IDs are deleted.** Create an unrelated user in the same + instance and confirm it survives. +- [ ] **19.5** IDs with no matching user are skipped and reported — the normal + case for a partial or already-partly-undone migration. + +### Missing record + +- [ ] **19.6** No saved record → `No migration to undo: this project has no +record of a previous `clerk migrate import`.` + `Run `clerk migrate delete` +from the project you migrated from.` and **zero HTTP requests**. +- [ ] **19.7** Record present but the file is gone → `The migration file +users.json is no longer there, so the users it created cannot be identified.` + and zero requests. +- [ ] **19.8** `clerk migrate settings clear file` **disarms** delete — that is + why `clear` warns first. Confirm the sequence. +- [ ] **19.9** **The record is actually written.** Run an import, then inspect + the CLI config for a `migrations` entry. (Guards `829d979` — `setMigrationEntry` + used to mutate memory and never write.) +- [ ] **19.10** The record is keyed **by project** (linked profile → git remote → + directory), not by cwd, and lives in the CLI config — **not** a `.settings` + file in the repo being migrated. Confirm no `.settings` file appears anywhere. + (Guards `d94307e`.) + +### Confirmation and execution + +- [ ] **19.11** Warning first: `About to delete 2 users from , matched to +users.json by external ID.` plus a dimmed `1 of the file's users is not in this +instance and will be left alone.` when applicable. +- [ ] **19.12** Confirm `Permanently delete 2 users?` defaults to **false**. + Declining aborts with `└ Paused` and deletes nothing. +- [ ] **19.13** Non-interactive/agent **without `-y`** → `` `clerk migrate +delete` permanently deletes 2 users and cannot prompt here. Pass -y to +confirm.`` and **no DELETE requests are made**. +- [ ] **19.14** No IDs in the file → `No user IDs found in ; nothing to +undo.` +- [ ] **19.15** No matches in the instance → `None of the 5 users in users.json +are in . Nothing to delete.` +- [ ] **19.16** Spinners: `Finding migrated users: batch 1/3...` then + `Deleting users: [3/10] (2 deleted, 1 failed)...` +- [ ] **19.17** **A failure on one user does not abort the rest.** Delete one + user manually mid-run (or force a 422) and confirm the others still go. +- [ ] **19.18** Summary shows Deleted / Failed / an error breakdown / the log + path. +- [ ] **19.19** Exactly one `delete-.log` per run, NDJSON, carrying + **both** the source ID and the Clerk ID per line. 429 retries add + `"status":"429_retry"` lines. +- [ ] **19.20** Exit code **1** when any deletion failed, 0 otherwise (including + the "nothing to delete" paths). +- [ ] **19.21** Next steps: `Run `clerk migrate logs list` to inspect the +deletion log`. +- [ ] **19.22** Takes the same targeting flags as import: `--secret-key`, + `--app`, `--instance`. +- [ ] **19.23** **`migrate delete` can trigger the first-run log-dir prompt** + before it does anything — surprising on an undo. Confirm whether that is + acceptable. +- [ ] **19.24** **Full round trip:** import 5 users → `clerk migrate delete` → + all 5 gone, nothing else touched, exit 0. + +--- + +## 20. Cross-cutting + +### Agent mode + +- [ ] **20.1** `--mode agent`, `CLERK_MODE=agent` and a piped stdout all behave + identically. +- [ ] **20.2** `--mode banana` → `Invalid mode "banana". Must be "human" or +"agent".`, exit 2. +- [ ] **20.3** Every agent-mode error is **one JSON line on stderr** with + `code`, `message`, `docsUrl` and `examples`. +- [ ] **20.4** Clerk docs URLs get `.md` appended in agent mode. +- [ ] **20.5** `UserAbortError` (Ctrl-C, declined prompts) exits **0** with no + error text. Ctrl-C mid-command exits **130**. +- [ ] **20.6** No gutter, no next steps, no spinners in agent mode. +- [ ] **20.7** `--input-json` works on `migrate import` the way it does elsewhere. + +### Secrets never leak + +- [ ] **20.8** Run every credential-taking command with `--verbose` and grep the + output for the secret. Nothing may appear: Auth0 client secret, WorkOS API + key, DB password, Turso authToken, Firebase private key. +- [ ] **20.9** `clerk migrate settings list` and `--json` redact the signer key. +- [ ] **20.10** No secret is written to the CLI config file. +- [ ] **20.11** `.env.clerk-migrate` is gitignored the moment it is created. + +### The new `network_unreachable` error (guards `bf4d6ed`) + +- [ ] **20.12** Disable the network (or point at a dead port) and run any + networked command. Expect `Could not reach . Check your network +connection (or VPN) and try again.` — the **host is named**, and the code is + `network_unreachable`, not `unexpected_error`. +- [ ] **20.13** Ctrl-C mid-request still reads as an abort, **not** as "Could not + reach". + +### `a: all` in every multiselect (guards `68c7057`) + +- [ ] **20.14** The footer reads `↑/↓ to navigate • Space: select • a: all • +Enter: confirm` — `a: all` second-to-last, `Enter: confirm` still last. +- [ ] **20.15** Pressing `a` toggles every option. +- [ ] **20.16** `i` (invert) still works but is deliberately **not** advertised. +- [ ] **20.17** **This is global.** Check a multiselect outside `migrate` too. + +### `clerk init` detects WorkOS + +- [ ] **20.18** In a project depending on `@workos-inc/authkit-nextjs` or + `@workos-inc/node`, `clerk init` prints, in yellow, `⚠ Detected WorkOS in your +project.` and a dimmed `Migration guide:` link. (New in `8893e60`.) + +### Config durability + +- [ ] **20.19** After `clerk migrate settings set transformer clerk`, a + `migrations` key appears in the CLI config. +- [ ] **20.20** Run `clerk link` / `clerk config pull` (any other config write) + and confirm the `migrations` section **survives** — the config is rebuilt field + by field, so an unpreserved section would silently vanish. +- [ ] **20.21** Running `settings set` from a **subdirectory** of a linked + project reuses the same entry (keyed on the profile path). +- [ ] **20.22** In an unlinked repo with a git remote, the key is the normalized + remote — so **two clones of the same repo share settings**. Confirm that is + intended. + +### Gitignore hygiene (guards `3a667d7`) + +- [ ] **20.23** After running an export and downloading a Firebase key in the CLI + checkout, `git status` shows neither `exports/` nor `*service-account*.json`. +- [ ] **20.24** The migrate `logs/` **source** directory is still tracked (the + root `logs` ignore is negated for it). + +--- + +## 21. CI and repo hygiene + +- [ ] **21.1** `bun run format:check` passes. +- [ ] **21.2** `bun run lint` exits 0. Two warnings are present; one is new in + this PR — `src/commands/migrate/export/workos.test.ts:31:61 no-control-regex`. + **Decide whether to silence it.** +- [ ] **21.3** `bun run typecheck` passes. +- [ ] **21.4** `bun run test` passes. **Known:** `scripts/check-patches.test.ts` + fails in a worktree where `playwright-core` is not installed — that is a local + environment issue, not this PR. Run `bun install` and re-check. +- [ ] **21.5** `bun run test:e2e:op` passes (no migrate e2e tests exist yet — + confirm that is a deliberate gap). +- [ ] **21.6** `.changeset/migrate-cli.md` exists, targets `clerk`, and is a + `minor` bump. +- [ ] **21.7** The root `README.md` command table matches `clerk --help` exactly + (`readme.test.ts` enforces this — guards `4e5c4bd`). +- [ ] **21.8** `packages/cli-core/package.json` gained `csv-parser` and `zod`, + and the compiled binary still builds. +- [ ] **21.9** Binary size is still acceptable (~62 MB) — the PR deliberately + avoided `firebase-admin` (74 MB across 158 packages) for this reason. + +--- + +## Appendix A — Documentation drift found during planning + +File these as doc fixes; none of them change behavior. + +- [x] **A.1** ~~`migrate/README.md` says `6 built-in transformers`; there are + **7**~~ — **fixed.** WorkOS added to the example listing and the count + corrected. +- [x] **A.2** ~~`migrate/README.md`'s settings example says `4 of 7 settings +set`~~ — **fixed.** The example now shows all eight rows and reads + `6 of 8 settings set`. +- [x] **A.3** ~~`migrate/README.md` and `lib/input-retry.ts` both say `-y` + suppresses the credential-retry loop on exports, but `-y` does not exist on + any export subcommand~~ — **fixed in code.** `-y, --yes` added to all seven + export subcommands, and `withInputRetry` now honours it. +- [x] **A.4** ~~`migrate/README.md` and `lib/logger.ts` both say `-y` takes the + default log directory without asking, and it does not~~ — **fixed in code.** + `ensureLogDir` now takes `./logs` without asking and without saving when `-y` + was passed. +- [x] **A.5** ~~`migrate/README.md` says the "no link, no key, no flags" path + opens the flat instance picker~~ — **fixed.** The README and + `clerk-source.ts`'s own header comment now describe the application picker + and its `+ Create a new application` row. +- [x] **A.6** ~~`export clerk`'s own help example implies `--instance prod` runs + without a prompt~~ — **fixed.** The example is now + `clerk migrate export clerk --secret-key sk_live_… --output prod-users.json`, + and the README states that `--secret-key` is the only flag that skips the + picker. + +## Appendix B — Suspected bugs to confirm + +Each has a checkbox above; collected here so they can be triaged together. + +| # | Item | Where | +| ----- | --------------------------------------------------------------------------- | ----- | +| ~~1~~ | ~~`--app`/`--instance` open the picker~~ — **fixed** | 3.11 | +| ~~2~~ | ~~`CLERK_SECRET_KEY` discarded in a linked dir~~ — **fixed** | 3.12 | +| 3 | The unlinked-no-key path offers "Create a new application" | 3.13 | +| 4 | BigInt ids from libsql/SQLite crash `JSON.stringify` outside the retry loop | 9.25 | +| 5 | `settings clear -y` does not strip alias env vars | 17.41 | +| ~~6~~ | ~~`settings clear` in agent mode needs no `-y`~~ — **fixed** | 17.42 | +| ~~7~~ | ~~`-y` does not skip the log-dir prompt~~ — **fixed** | 18.46 | +| 8 | Invalid `CLERK_MIGRATE_*` limits are ignored with no warning | 14.5 | +| 9 | `--skip-unsupported-providers` persisted even when it did nothing | 10.41 | +| 10 | `export clerk` 429 backoff is silent for up to ~50s | 3.24 |