From 36684b2b35e810fb1ab463ad984dc1ea86c97965 Mon Sep 17 00:00:00 2001 From: jiasheng Date: Wed, 29 Jul 2026 18:17:35 +0800 Subject: [PATCH 1/8] feat: add --introspect option to generate schema from database if no schema file is found --- packages/cli/src/actions/proxy.ts | 273 +++++++++++++++++++++++++++++- packages/cli/src/index.ts | 6 + 2 files changed, 278 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/actions/proxy.ts b/packages/cli/src/actions/proxy.ts index 16c7715d5..a2b9bd7b6 100644 --- a/packages/cli/src/actions/proxy.ts +++ b/packages/cli/src/actions/proxy.ts @@ -1,3 +1,4 @@ +import { formatDocument, ZModelCodeGenerator } from '@zenstackhq/language'; import { ConfigExpr, InvocationExpr, @@ -5,6 +6,7 @@ import { isInvocationExpr, isLiteralExpr, LiteralExpr, + type Model, } from '@zenstackhq/language/ast'; import { getStringLiteral } from '@zenstackhq/language/utils'; import { ZenStackClient, type ClientContract } from '@zenstackhq/orm'; @@ -22,11 +24,18 @@ import express from 'express'; import { createJiti } from 'jiti'; import type { createPool as MysqlCreatePool } from 'mysql2'; import { verify } from 'node:crypto'; +import fs from 'node:fs'; import path from 'node:path'; +import ora from 'ora'; +import { detect, resolveCommand } from 'package-manager-detector'; import type { Pool as PgPoolType } from 'pg'; import { CliError } from '../cli-error'; +import { execSync } from '../utils/exec-utils'; import { getVersion } from '../utils/version-utils'; import { getOutputPath, getSchemaFile, loadSchemaDocument } from './action-utils'; +import type { DataSourceProviderType } from '@zenstackhq/schema'; +import { providers as pullProviders, type IntrospectionProvider } from './pull/provider'; +import { syncEnums, syncRelation, syncTable, type Relation } from './pull'; import { z } from 'zod'; type Options = { @@ -37,6 +46,7 @@ type Options = { databaseUrl?: string; studioAuthKey?: string; signatureToleranceSecs: number; + introspect?: boolean; }; export const ProxyAuthError = { @@ -91,7 +101,7 @@ export async function run(options: Options) { const log = options.logLevel?.filter((level): level is (typeof allowedLogLevels)[number] => allowedLogLevels.includes(level as any), ); - const schemaFile = getSchemaFile(options.schema); + const schemaFile = await resolveSchema(options); console.log(colors.gray(`Loading ZModel schema from: ${schemaFile}`)); let outputPath = getOutputPath(options, schemaFile); @@ -479,3 +489,264 @@ function startServer( process.exit(0); }); } + +// --------------------------------------------------------------------------- +// --introspect helpers +// --------------------------------------------------------------------------- + +async function resolveSchema(options: Options): Promise { + try { + return getSchemaFile(options.schema); + } catch (err) { + if (!(err instanceof CliError)) throw err; + + // Schema not found – check whether we should introspect + if (!options.introspect) { + // Provide a helpful error depending on whether DATABASE_URL is set + const hasEnvUrl = !!process.env['DATABASE_URL']; + if (hasEnvUrl) { + throw new CliError( + 'No schema file found. You can either:\n' + + ' • Specify the schema explicitly: zen studio --schema \n' + + ' • Auto-generate from the database: zen studio --introspect', + ); + } else { + throw new CliError( + 'No schema file found. You can either:\n' + + ' • Specify the schema explicitly: zen studio --schema -d \n' + + ' • Auto-generate from the database: zen studio --introspect -d ', + ); + } + } + + // Resolve database URL + const databaseUrl = options.databaseUrl ?? process.env['DATABASE_URL']; + if (!databaseUrl) { + throw new CliError( + '--introspect requires a database connection — pass -d or set DATABASE_URL.', + ); + } + + // If the default schema file already exists, skip introspection + const defaultSchemaPath = path.resolve('zenstack', 'schema.zmodel'); + if (fs.existsSync(defaultSchemaPath)) { + console.log( + colors.gray( + 'Note: --introspect ignored because a schema file already exists at ' + + defaultSchemaPath, + ), + ); + return defaultSchemaPath; + } + + // Determine DB provider type from URL + const providerType = getProviderFromUrl(databaseUrl); + + // Install required packages + await installPackagesForIntrospect(providerType); + + // Introspect and generate schema + const urlFromEnv = !options.databaseUrl; + const schemaFile = await introspectAndGenerateSchema( + databaseUrl, + providerType, + urlFromEnv, + ); + + console.log( + colors.yellow( + '\n⚠ The generated schema has no access policies.\n' + + ' All data is accessible without restriction.\n' + + ' Edit ' + + colors.cyan(schemaFile) + + ' to add access policies.\n', + ), + ); + + return schemaFile; + } +} + +function getProviderFromUrl(url: string): DataSourceProviderType { + if (url.startsWith('postgres://') || url.startsWith('postgresql://')) { + return 'postgresql'; + } + if (url.startsWith('mysql://')) { + return 'mysql'; + } + if (url.startsWith('file:') || url.startsWith('sqlite:')) { + return 'sqlite'; + } + throw new CliError(`Unable to determine database type from URL: ${url}`); +} + +async function installPackagesForIntrospect(providerType: DataSourceProviderType) { + const corePackages = [ + { name: '@zenstackhq/cli@latest', dev: true }, + { name: '@zenstackhq/schema@latest', dev: false }, + { name: '@zenstackhq/orm@latest', dev: false }, + ]; + + // Add the appropriate DB driver + const driverPackage: { name: string; dev: false } = (() => { + switch (providerType) { + case 'postgresql': + return { name: 'pg', dev: false as const }; + case 'mysql': + return { name: 'mysql2', dev: false as const }; + case 'sqlite': + return { name: 'better-sqlite3', dev: false as const }; + } + })(); + + const allPackages = [...corePackages, driverPackage]; + + let pm = await detect(); + if (!pm) { + pm = { agent: 'npm', name: 'npm' }; + } + + console.log(colors.gray(`Using package manager: ${pm.agent}`)); + + for (const pkg of allPackages) { + const resolved = resolveCommand(pm.agent, 'add', [ + pkg.name, + ...(pkg.dev + ? [pm.agent.startsWith('yarn') || pm.agent === 'bun' ? '--dev' : '--save-dev'] + : []), + ]); + if (!resolved) { + throw new CliError( + `Unable to determine how to install package "${pkg.name}". Please install it manually.`, + ); + } + + const spinner = ora(`Installing "${pkg.name}"`).start(); + try { + execSync(`${resolved.command} ${resolved.args.join(' ')}`, { + cwd: process.cwd(), + }); + spinner.succeed(); + } catch (e) { + spinner.fail(); + throw e; + } + } +} + +async function introspectAndGenerateSchema( + databaseUrl: string, + providerType: DataSourceProviderType, + urlFromEnv: boolean, +): Promise { + const provider: IntrospectionProvider = pullProviders[providerType]; + if (!provider) { + throw new CliError(`No introspection provider found for: ${providerType}`); + } + + // Build a minimal datasource-only schema so we can bootstrap Langium services + const urlExpr = urlFromEnv ? 'env("DATABASE_URL")' : `'${databaseUrl}'`; + const minimalSchema = `datasource db {\n provider = '${providerType}'\n url = ${urlExpr}\n}\n`; + + // Write the minimal schema to disk first, then load it so Langium has a proper document URI + const schemaDir = path.resolve('zenstack'); + const schemaFile = path.join(schemaDir, 'schema.zmodel'); + fs.mkdirSync(schemaDir, { recursive: true }); + fs.writeFileSync(schemaFile, minimalSchema, 'utf-8'); + + // Load the minimal schema to get services + const { model, services } = await loadSchemaDocument(schemaFile, { returnServices: true }); + + // Introspect the database + const spinner = ora('Introspecting database...').start(); + let introspectionResult; + try { + introspectionResult = await provider.introspect(databaseUrl, { + schemas: ['public'], + modelCasing: 'pascal', + }); + spinner.succeed('Database introspected successfully.'); + } catch (err) { + spinner.fail('Failed to introspect database.'); + throw err; + } + + // Build a fresh AST Model with the datasource already in it from loadSchemaDocument + const newModel: Model = { + $type: 'Model', + $container: undefined, + $containerProperty: undefined, + $containerIndex: undefined, + declarations: [...model.declarations.filter((d) => d.$type === 'DataSource')], + imports: [], + }; + + // Sync enums + syncEnums({ + dbEnums: introspectionResult.enums, + model: newModel, + oldModel: model, + provider, + services, + options: { schema: schemaFile, modelCasing: 'pascal', fieldCasing: 'none', alwaysMap: false, quote: 'single', indent: 4 }, + defaultSchema: 'public', + }); + + // Sync tables and collect relations + const resolvedRelations: Relation[] = []; + for (const table of introspectionResult.tables) { + const relations = syncTable({ + table, + model: newModel, + oldModel: model, + provider, + services, + options: { schema: schemaFile, modelCasing: 'pascal', fieldCasing: 'none', alwaysMap: false, quote: 'single', indent: 4 }, + defaultSchema: 'public', + }); + resolvedRelations.push(...relations); + } + + // Normalize schema values for relation matching + const schemasMatch = (a: string | null | undefined, b: string | null | undefined) => + (a ?? '') === (b ?? ''); + + // Sync relations + for (const relation of resolvedRelations) { + const similarRelations = resolvedRelations.filter((rr) => { + return ( + rr !== relation && + ((schemasMatch(rr.schema, relation.schema) && + rr.table === relation.table && + schemasMatch(rr.references.schema, relation.references.schema) && + rr.references.table === relation.references.table) || + (schemasMatch(rr.schema, relation.references.schema) && + rr.table === relation.references.table && + schemasMatch(rr.references.schema, relation.schema) && + rr.references.table === relation.table)) + ); + }).length; + const selfRelation = + schemasMatch(relation.references.schema, relation.schema) && + relation.references.table === relation.table; + syncRelation({ + model: newModel, + relation, + services, + options: { schema: schemaFile, modelCasing: 'pascal', fieldCasing: 'none', alwaysMap: false, quote: 'single', indent: 4 }, + selfRelation, + similarRelations, + }); + } + + // Generate zmodel text + const generator = new ZModelCodeGenerator({ quote: 'single', indent: 4 }); + const generatedCode = generator.generate(newModel); + const formatted = await formatDocument(generatedCode); + + // Write the final schema + fs.writeFileSync(schemaFile, formatted, 'utf-8'); + console.log(colors.green(`\n✅ Schema generated: ${colors.cyan(schemaFile)}`)); + + return schemaFile; +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index ba458ded5..5cf2c062b 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -283,6 +283,12 @@ Arguments following -- are passed to the seed script. E.g.: "zen db seed -- --us return parsed; }), ) + .addOption( + new Option( + '--introspect', + 'Introspect the database to generate a schema when no schema file is found', + ), + ) .addOption(noVersionCheckOption) .action(proxyAction); From acea3865585868ab4981c7fdccf9ad573d3fd065 Mon Sep 17 00:00:00 2001 From: jiasheng Date: Wed, 29 Jul 2026 18:49:16 +0800 Subject: [PATCH 2/8] feat: enhance introspection support by adding databaseUrl and provider options to PullOptions and exporting related functions --- packages/cli/src/actions/db.ts | 104 +++++++++++++------ packages/cli/src/actions/proxy.ts | 162 ++++++------------------------ packages/cli/test/proxy.test.ts | 119 +++++++++++++++++++++- 3 files changed, 223 insertions(+), 162 deletions(-) diff --git a/packages/cli/src/actions/db.ts b/packages/cli/src/actions/db.ts index c8e0f8204..0bc813680 100644 --- a/packages/cli/src/actions/db.ts +++ b/packages/cli/src/actions/db.ts @@ -14,7 +14,14 @@ import { } from './action-utils'; import { consolidateEnums, syncEnums, syncRelation, syncTable, type Relation } from './pull'; import { providers as pullProviders } from './pull/provider'; -import { getDatasource, getDbName, getRelationFieldsKey, getRelationFkName, getRelationName, isDatabaseManagedAttribute } from './pull/utils'; +import { + getDatasource, + getDbName, + getRelationFieldsKey, + getRelationFkName, + getRelationName, + isDatabaseManagedAttribute, +} from './pull/utils'; import type { DataSourceProviderType } from '@zenstackhq/schema'; import { CliError } from '../cli-error'; @@ -28,6 +35,8 @@ type PushOptions = { export type PullOptions = { schema?: string; output?: string; + databaseUrl?: string; + provider?: DataSourceProviderType; modelCasing: 'pascal' | 'camel' | 'snake' | 'none'; fieldCasing: 'pascal' | 'camel' | 'snake' | 'none'; alwaysMap: boolean; @@ -101,7 +110,7 @@ async function runPush(options: PushOptions) { } } -async function runPull(options: PullOptions) { +export async function runPull(options: PullOptions) { const spinner = ora(); try { const schemaFile = getSchemaFile(options.schema); @@ -109,16 +118,34 @@ async function runPull(options: PullOptions) { // Determine early if `--out` is a single file output (combined schema) or a directory export. const outPath = options.output ? path.resolve(options.output) : undefined; const treatAsFile = - !!outPath && - ((fs.existsSync(outPath) && fs.lstatSync(outPath).isFile()) || path.extname(outPath) !== ''); + !!outPath && ((fs.existsSync(outPath) && fs.lstatSync(outPath).isFile()) || path.extname(outPath) !== ''); const { model, services } = await loadSchemaDocument(schemaFile, { returnServices: true, mergeImports: treatAsFile, }); + // When provider/databaseUrl are passed explicitly (e.g. from --introspect), + // use those directly instead of reading from the parsed datasource declaration. + let datasource: { + provider: DataSourceProviderType; + url: string; + defaultSchema: string; + allSchemas: string[]; + }; + + if (options.provider && options.databaseUrl) { + datasource = { + provider: options.provider, + url: options.databaseUrl, + defaultSchema: 'public', + allSchemas: ['public'], + }; + } else { + datasource = getDatasource(model); + } + const SUPPORTED_PROVIDERS = Object.keys(pullProviders) as DataSourceProviderType[]; - const datasource = getDatasource(model); if (!SUPPORTED_PROVIDERS.includes(datasource.provider)) { throw new CliError(`Unsupported datasource provider: ${datasource.provider}`); @@ -131,7 +158,10 @@ async function runPull(options: PullOptions) { } spinner.start('Introspecting database...'); - const { enums, tables } = await provider.introspect(datasource.url, { schemas: datasource.allSchemas, modelCasing: options.modelCasing }); + const { enums, tables } = await provider.introspect(datasource.url, { + schemas: datasource.allSchemas, + modelCasing: options.modelCasing, + }); spinner.succeed('Database introspected'); console.log(colors.blue('Syncing schema...')); @@ -170,8 +200,7 @@ async function runPull(options: PullOptions) { } // Normalize schema values: MySQL uses null for references.schema but '' for table.schema; // treat null, undefined, and '' as equivalent when comparing schemas. - const schemasMatch = (a: string | null | undefined, b: string | null | undefined) => - (a ?? '') === (b ?? ''); + const schemasMatch = (a: string | null | undefined, b: string | null | undefined) => (a ?? '') === (b ?? ''); // sync relation fields for (const relation of resolvedRelations) { @@ -276,11 +305,13 @@ async function runPull(options: PullOptions) { .filter((d) => d.$type === DataModel.$type || d.$type === Enum.$type) .forEach((_declaration) => { const newDataModel = _declaration as DataModel | Enum; - const declarations = services.shared.workspace.IndexManager.allElements(newDataModel.$type, docsSet).toArray(); + const declarations = services.shared.workspace.IndexManager.allElements( + newDataModel.$type, + docsSet, + ).toArray(); const originalDataModel = declarations.find((d) => getDbName(d.node as any) === getDbName(newDataModel)) ?.node as DataModel | Enum | undefined; if (!originalDataModel) { - if (newDataModel.$type === 'DataModel') { addedModels.push(colors.green(`+ Model ${newDataModel.name} added`)); } else if (newDataModel.$type === 'Enum') { @@ -339,9 +370,7 @@ async function runPull(options: PullOptions) { if (newRelName) { originalFields = originalDataModel.fields.filter( (d) => - isDataField(d) && - isDataField(f) && - matchesRelationNameFallback(f, newRelName, d), + isDataField(d) && isDataField(f) && matchesRelationNameFallback(f, newRelName, d), ); } } @@ -395,7 +424,9 @@ async function runPull(options: PullOptions) { const newRefName = getDbName(newType.reference.ref); const oldRefName = getDbName(oldType.reference.ref); if (newRefName !== oldRefName) { - fieldUpdates.push(`reference: ${oldType.reference.$refText} -> ${newType.reference.$refText}`); + fieldUpdates.push( + `reference: ${oldType.reference.$refText} -> ${newType.reference.$refText}`, + ); // Replace the entire reference object — Langium References // from parsed documents expose `ref` as a getter-only property. (oldType as any).reference = { @@ -444,21 +475,28 @@ async function runPull(options: PullOptions) { if (newDefaultAttr && oldDefaultAttr) { // Compare attribute arguments by serializing them (avoid circular refs with $type fallback) const serializeArgs = (args: any[]) => - args.map((arg) => { - if (arg.value?.$type === 'StringLiteral') return `"${arg.value.value}"`; - if (arg.value?.$type === 'NumberLiteral') return String(arg.value.value); - if (arg.value?.$type === 'BooleanLiteral') return String(arg.value.value); - if (arg.value?.$type === 'InvocationExpr') return arg.value.function?.$refText ?? ''; - if (arg.value?.$type === 'ReferenceExpr') return arg.value.target?.$refText ?? ''; - if (arg.value?.$type === 'ArrayExpr') { - return `[${(arg.value.items ?? []).map((item: any) => { - if (item.$type === 'ReferenceExpr') return item.target?.$refText ?? ''; - return item.$type ?? 'unknown'; - }).join(',')}]`; - } - // Fallback: use $type to avoid circular reference issues - return arg.value?.$type ?? 'unknown'; - }).join(','); + args + .map((arg) => { + if (arg.value?.$type === 'StringLiteral') return `"${arg.value.value}"`; + if (arg.value?.$type === 'NumberLiteral') return String(arg.value.value); + if (arg.value?.$type === 'BooleanLiteral') return String(arg.value.value); + if (arg.value?.$type === 'InvocationExpr') + return arg.value.function?.$refText ?? ''; + if (arg.value?.$type === 'ReferenceExpr') + return arg.value.target?.$refText ?? ''; + if (arg.value?.$type === 'ArrayExpr') { + return `[${(arg.value.items ?? []) + .map((item: any) => { + if (item.$type === 'ReferenceExpr') + return item.target?.$refText ?? ''; + return item.$type ?? 'unknown'; + }) + .join(',')}]`; + } + // Fallback: use $type to avoid circular reference issues + return arg.value?.$type ?? 'unknown'; + }) + .join(','); const newArgsStr = serializeArgs(newDefaultAttr.args ?? []); const oldArgsStr = serializeArgs(oldDefaultAttr.args ?? []); @@ -500,7 +538,7 @@ async function runPull(options: PullOptions) { originalField.attributes .filter( (attr) => - !f.attributes.find((d) => d.decl.$refText === attr.decl.$refText) && + !f.attributes.find((d) => d.decl.$refText === attr.decl.$refText) && isDatabaseManagedAttribute(attr.decl.$refText), ) .forEach((attr) => { @@ -508,7 +546,9 @@ async function runPull(options: PullOptions) { const index = field.attributes.findIndex((d) => d === attr); field.attributes.splice(index, 1); getModelChanges(originalDataModel.name).deletedAttributes.push( - colors.yellow(`- ${attr.decl.$refText} from field: ${originalDataModel.name}.${field.name}`), + colors.yellow( + `- ${attr.decl.$refText} from field: ${originalDataModel.name}.${field.name}`, + ), ); }); @@ -520,7 +560,7 @@ async function runPull(options: PullOptions) { isDatabaseManagedAttribute(attr.decl.$refText), ) .forEach((attr) => { - // attach the new attribute to the original field + // attach the new attribute to the original field const cloned = { ...attr, $container: originalField } as typeof attr; originalField.attributes.push(cloned); getModelChanges(originalDataModel.name).addedAttributes.push( diff --git a/packages/cli/src/actions/proxy.ts b/packages/cli/src/actions/proxy.ts index a2b9bd7b6..febe154c9 100644 --- a/packages/cli/src/actions/proxy.ts +++ b/packages/cli/src/actions/proxy.ts @@ -1,4 +1,3 @@ -import { formatDocument, ZModelCodeGenerator } from '@zenstackhq/language'; import { ConfigExpr, InvocationExpr, @@ -6,7 +5,6 @@ import { isInvocationExpr, isLiteralExpr, LiteralExpr, - type Model, } from '@zenstackhq/language/ast'; import { getStringLiteral } from '@zenstackhq/language/utils'; import { ZenStackClient, type ClientContract } from '@zenstackhq/orm'; @@ -34,8 +32,7 @@ import { execSync } from '../utils/exec-utils'; import { getVersion } from '../utils/version-utils'; import { getOutputPath, getSchemaFile, loadSchemaDocument } from './action-utils'; import type { DataSourceProviderType } from '@zenstackhq/schema'; -import { providers as pullProviders, type IntrospectionProvider } from './pull/provider'; -import { syncEnums, syncRelation, syncTable, type Relation } from './pull'; +import { runPull } from './db'; import { z } from 'zod'; type Options = { @@ -88,14 +85,6 @@ function normalizePublicKey(key: string): string { export async function run(options: Options) { // Resolve public key: CLI arg takes precedence, then ZENSTACK_STUDIO_AUTH_KEY env var. options = { ...options, studioAuthKey: options.studioAuthKey ?? process.env['ZENSTACK_STUDIO_AUTH_KEY'] }; - if (!options.studioAuthKey) { - console.warn( - colors.yellow( - 'Warning: This proxy has no authentication. Do not expose it to the public network.\n' + - 'To secure it, get an API key from ZenStack Studio and set it via the ZENSTACK_STUDIO_AUTH_KEY environment variable.', - ), - ); - } const allowedLogLevels = ['error', 'query'] as const; const log = options.logLevel?.filter((level): level is (typeof allowedLogLevels)[number] => @@ -173,6 +162,15 @@ export async function run(options: Options) { } startServer(db, schemaModule.schema, options, authDb); + + if (!options.studioAuthKey) { + console.warn( + colors.yellow( + 'Warning: This proxy has no authentication. Do not expose it to the public network.\n' + + 'To secure it, get an API key from ZenStack Studio and set it via the ZENSTACK_STUDIO_AUTH_KEY environment variable.', + ), + ); + } } function evaluateUrl(schemaUrl: ConfigExpr) { @@ -494,7 +492,7 @@ function startServer( // --introspect helpers // --------------------------------------------------------------------------- -async function resolveSchema(options: Options): Promise { +export async function resolveSchema(options: Options): Promise { try { return getSchemaFile(options.schema); } catch (err) { @@ -522,19 +520,14 @@ async function resolveSchema(options: Options): Promise { // Resolve database URL const databaseUrl = options.databaseUrl ?? process.env['DATABASE_URL']; if (!databaseUrl) { - throw new CliError( - '--introspect requires a database connection — pass -d or set DATABASE_URL.', - ); + throw new CliError('--introspect requires a database connection — pass -d or set DATABASE_URL.'); } // If the default schema file already exists, skip introspection const defaultSchemaPath = path.resolve('zenstack', 'schema.zmodel'); if (fs.existsSync(defaultSchemaPath)) { console.log( - colors.gray( - 'Note: --introspect ignored because a schema file already exists at ' + - defaultSchemaPath, - ), + colors.gray('Note: --introspect ignored because a schema file already exists at ' + defaultSchemaPath), ); return defaultSchemaPath; } @@ -547,11 +540,11 @@ async function resolveSchema(options: Options): Promise { // Introspect and generate schema const urlFromEnv = !options.databaseUrl; - const schemaFile = await introspectAndGenerateSchema( - databaseUrl, - providerType, - urlFromEnv, - ); + const schemaFile = await introspectAndGenerateSchema(databaseUrl, providerType, urlFromEnv); + + // Run code generation from the generated schema + const { run: runGenerate } = await import('./generate'); + await runGenerate({ schema: schemaFile, silent: false, watch: false }); console.log( colors.yellow( @@ -567,7 +560,7 @@ async function resolveSchema(options: Options): Promise { } } -function getProviderFromUrl(url: string): DataSourceProviderType { +export function getProviderFromUrl(url: string): DataSourceProviderType { if (url.startsWith('postgres://') || url.startsWith('postgresql://')) { return 'postgresql'; } @@ -611,14 +604,10 @@ async function installPackagesForIntrospect(providerType: DataSourceProviderType for (const pkg of allPackages) { const resolved = resolveCommand(pm.agent, 'add', [ pkg.name, - ...(pkg.dev - ? [pm.agent.startsWith('yarn') || pm.agent === 'bun' ? '--dev' : '--save-dev'] - : []), + ...(pkg.dev ? [pm.agent.startsWith('yarn') || pm.agent === 'bun' ? '--dev' : '--save-dev'] : []), ]); if (!resolved) { - throw new CliError( - `Unable to determine how to install package "${pkg.name}". Please install it manually.`, - ); + throw new CliError(`Unable to determine how to install package "${pkg.name}". Please install it manually.`); } const spinner = ora(`Installing "${pkg.name}"`).start(); @@ -639,113 +628,28 @@ async function introspectAndGenerateSchema( providerType: DataSourceProviderType, urlFromEnv: boolean, ): Promise { - const provider: IntrospectionProvider = pullProviders[providerType]; - if (!provider) { - throw new CliError(`No introspection provider found for: ${providerType}`); - } - - // Build a minimal datasource-only schema so we can bootstrap Langium services + // Write a minimal datasource-only schema so runPull has something to load const urlExpr = urlFromEnv ? 'env("DATABASE_URL")' : `'${databaseUrl}'`; const minimalSchema = `datasource db {\n provider = '${providerType}'\n url = ${urlExpr}\n}\n`; - // Write the minimal schema to disk first, then load it so Langium has a proper document URI const schemaDir = path.resolve('zenstack'); const schemaFile = path.join(schemaDir, 'schema.zmodel'); fs.mkdirSync(schemaDir, { recursive: true }); fs.writeFileSync(schemaFile, minimalSchema, 'utf-8'); - // Load the minimal schema to get services - const { model, services } = await loadSchemaDocument(schemaFile, { returnServices: true }); - - // Introspect the database - const spinner = ora('Introspecting database...').start(); - let introspectionResult; - try { - introspectionResult = await provider.introspect(databaseUrl, { - schemas: ['public'], - modelCasing: 'pascal', - }); - spinner.succeed('Database introspected successfully.'); - } catch (err) { - spinner.fail('Failed to introspect database.'); - throw err; - } - - // Build a fresh AST Model with the datasource already in it from loadSchemaDocument - const newModel: Model = { - $type: 'Model', - $container: undefined, - $containerProperty: undefined, - $containerIndex: undefined, - declarations: [...model.declarations.filter((d) => d.$type === 'DataSource')], - imports: [], - }; - - // Sync enums - syncEnums({ - dbEnums: introspectionResult.enums, - model: newModel, - oldModel: model, - provider, - services, - options: { schema: schemaFile, modelCasing: 'pascal', fieldCasing: 'none', alwaysMap: false, quote: 'single', indent: 4 }, - defaultSchema: 'public', + // Delegate to runPull with provider/URL overrides + await runPull({ + schema: schemaFile, + output: schemaFile, + databaseUrl, + provider: providerType, + modelCasing: 'pascal', + fieldCasing: 'none', + alwaysMap: false, + quote: 'single', + indent: 4, }); - // Sync tables and collect relations - const resolvedRelations: Relation[] = []; - for (const table of introspectionResult.tables) { - const relations = syncTable({ - table, - model: newModel, - oldModel: model, - provider, - services, - options: { schema: schemaFile, modelCasing: 'pascal', fieldCasing: 'none', alwaysMap: false, quote: 'single', indent: 4 }, - defaultSchema: 'public', - }); - resolvedRelations.push(...relations); - } - - // Normalize schema values for relation matching - const schemasMatch = (a: string | null | undefined, b: string | null | undefined) => - (a ?? '') === (b ?? ''); - - // Sync relations - for (const relation of resolvedRelations) { - const similarRelations = resolvedRelations.filter((rr) => { - return ( - rr !== relation && - ((schemasMatch(rr.schema, relation.schema) && - rr.table === relation.table && - schemasMatch(rr.references.schema, relation.references.schema) && - rr.references.table === relation.references.table) || - (schemasMatch(rr.schema, relation.references.schema) && - rr.table === relation.references.table && - schemasMatch(rr.references.schema, relation.schema) && - rr.references.table === relation.table)) - ); - }).length; - const selfRelation = - schemasMatch(relation.references.schema, relation.schema) && - relation.references.table === relation.table; - syncRelation({ - model: newModel, - relation, - services, - options: { schema: schemaFile, modelCasing: 'pascal', fieldCasing: 'none', alwaysMap: false, quote: 'single', indent: 4 }, - selfRelation, - similarRelations, - }); - } - - // Generate zmodel text - const generator = new ZModelCodeGenerator({ quote: 'single', indent: 4 }); - const generatedCode = generator.generate(newModel); - const formatted = await formatDocument(generatedCode); - - // Write the final schema - fs.writeFileSync(schemaFile, formatted, 'utf-8'); console.log(colors.green(`\n✅ Schema generated: ${colors.cyan(schemaFile)}`)); return schemaFile; diff --git a/packages/cli/test/proxy.test.ts b/packages/cli/test/proxy.test.ts index e1f958450..5038735e7 100644 --- a/packages/cli/test/proxy.test.ts +++ b/packages/cli/test/proxy.test.ts @@ -3,7 +3,7 @@ import { createTestClient } from '@zenstackhq/testtools'; import { sign } from 'node:crypto'; import http from 'node:http'; import { afterEach, describe, expect, it } from 'vitest'; -import { createProxyApp } from '../src/actions/proxy'; +import { createProxyApp, getProviderFromUrl, resolveSchema } from '../src/actions/proxy'; // ─── Ed25519 key pair for tests ─────────────────────────────────────────────── const TEST_PRIVATE_KEY = `-----BEGIN PRIVATE KEY----- @@ -800,4 +800,121 @@ describe('CLI proxy tests', () => { expect(body.data[1][0]).toMatchObject({ id: 'p1' }); }); }); + + // ─── --introspect helpers ───────────────────────────────────────────────────── + + describe('--introspect helpers', () => { + describe('getProviderFromUrl', () => { + it('should return postgresql for postgres:// URLs', () => { + expect(getProviderFromUrl('postgres://user:pass@localhost:5432/mydb')).toBe('postgresql'); + }); + + it('should return postgresql for postgresql:// URLs', () => { + expect(getProviderFromUrl('postgresql://user:pass@localhost:5432/mydb')).toBe('postgresql'); + }); + + it('should return mysql for mysql:// URLs', () => { + expect(getProviderFromUrl('mysql://user:pass@localhost:3306/mydb')).toBe('mysql'); + }); + + it('should return sqlite for file: URLs', () => { + expect(getProviderFromUrl('file:./test.db')).toBe('sqlite'); + }); + + it('should return sqlite for sqlite: URLs', () => { + expect(getProviderFromUrl('sqlite:./test.db')).toBe('sqlite'); + }); + + it('should throw for an unrecognizable URL', () => { + expect(() => getProviderFromUrl('mongodb://localhost/mydb')).toThrow( + 'Unable to determine database type from URL', + ); + }); + }); + + describe('resolveSchema', () => { + it('should throw helpful error without --introspect when DATABASE_URL is not set', async () => { + const originalEnv = process.env['DATABASE_URL']; + delete process.env['DATABASE_URL']; + try { + await expect( + resolveSchema({ + schema: '/nonexistent/path/schema.zmodel', + port: 3000, + signatureToleranceSecs: 60, + }), + ).rejects.toThrow('zen studio --introspect -d '); + } finally { + if (originalEnv !== undefined) process.env['DATABASE_URL'] = originalEnv; + } + }); + + it('should throw helpful error without --introspect when DATABASE_URL is set', async () => { + const originalEnv = process.env['DATABASE_URL']; + process.env['DATABASE_URL'] = 'postgres://localhost/testdb'; + try { + await expect( + resolveSchema({ + schema: '/nonexistent/path/schema.zmodel', + port: 3000, + signatureToleranceSecs: 60, + }), + ).rejects.toThrow('zen studio --introspect'); + } finally { + if (originalEnv !== undefined) { + process.env['DATABASE_URL'] = originalEnv; + } else { + delete process.env['DATABASE_URL']; + } + } + }); + + it('should throw when --introspect is used without a database URL', async () => { + const originalEnv = process.env['DATABASE_URL']; + delete process.env['DATABASE_URL']; + try { + await expect( + resolveSchema({ + schema: '/nonexistent/path/schema.zmodel', + port: 3000, + signatureToleranceSecs: 60, + introspect: true, + }), + ).rejects.toThrow('pass -d or set DATABASE_URL'); + } finally { + if (originalEnv !== undefined) process.env['DATABASE_URL'] = originalEnv; + } + }); + + it('should return existing schema path when default schema file exists', async () => { + const fs = await import('node:fs'); + const path = await import('node:path'); + const os = await import('node:os'); + + // Create a temporary directory to act as cwd + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zenstack-test-')); + const schemaDir = path.join(tmpDir, 'zenstack'); + fs.mkdirSync(schemaDir, { recursive: true }); + const schemaFile = path.join(schemaDir, 'schema.zmodel'); + fs.writeFileSync(schemaFile, 'datasource db { provider = "sqlite" url = "file:./test.db" }'); + + const originalCwd = process.cwd(); + process.chdir(tmpDir); + try { + const result = await resolveSchema({ + // no schema path — force the "not found" path + schema: '/nonexistent/path/schema.zmodel', + port: 3000, + signatureToleranceSecs: 60, + introspect: true, + databaseUrl: 'file:./test.db', + }); + expect(result).toBe(path.resolve('zenstack', 'schema.zmodel')); + } finally { + process.chdir(originalCwd); + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + }); + }); }); From bd122328cb4bd51e86ddbd63285193e282f97153 Mon Sep 17 00:00:00 2001 From: jiasheng Date: Wed, 29 Jul 2026 21:33:56 +0800 Subject: [PATCH 3/8] feat: add package loading utilities and update introspection providers to use them --- packages/cli/src/actions/action-utils.ts | 32 +++++++ packages/cli/src/actions/proxy.ts | 95 +++++++++---------- .../cli/src/actions/pull/provider/mysql.ts | 7 +- .../src/actions/pull/provider/postgresql.ts | 10 +- .../cli/src/actions/pull/provider/sqlite.ts | 48 ++++++++-- packages/cli/test/action-utils.test.ts | 14 ++- 6 files changed, 143 insertions(+), 63 deletions(-) diff --git a/packages/cli/src/actions/action-utils.ts b/packages/cli/src/actions/action-utils.ts index 16fae2e15..972a5586b 100644 --- a/packages/cli/src/actions/action-utils.ts +++ b/packages/cli/src/actions/action-utils.ts @@ -382,3 +382,35 @@ export function startUsageTipsFetch() { } }; } + +export function isPackageInstalled(pkgName: string): boolean { + try { + const projectRequire = createRequire(path.resolve(process.cwd(), 'package.json')); + projectRequire.resolve(pkgName); + return true; + } catch { + return false; + } +} + +export async function loadPackage(pkgName: string) { + try { + return await import(pkgName); + } catch { + // If zenstack CLI is running directly using npx/pnpm in a temp folder, + // dynamic package import will fail. Also check the current project folder. + try { + const projectRequire = createRequire(path.resolve(process.cwd(), 'package.json')); + const resolvedPath = projectRequire.resolve(pkgName); + return await import(pathToFileURL(resolvedPath).href); + } catch { + try { + const projectRequire = createRequire(path.resolve(process.cwd(), 'package.json')); + return projectRequire(pkgName); + } catch { + return null; + } + } + } +} + diff --git a/packages/cli/src/actions/proxy.ts b/packages/cli/src/actions/proxy.ts index febe154c9..ae46afa22 100644 --- a/packages/cli/src/actions/proxy.ts +++ b/packages/cli/src/actions/proxy.ts @@ -30,7 +30,7 @@ import type { Pool as PgPoolType } from 'pg'; import { CliError } from '../cli-error'; import { execSync } from '../utils/exec-utils'; import { getVersion } from '../utils/version-utils'; -import { getOutputPath, getSchemaFile, loadSchemaDocument } from './action-utils'; +import { getOutputPath, getSchemaFile, isPackageInstalled, loadPackage, loadSchemaDocument } from './action-utils'; import type { DataSourceProviderType } from '@zenstackhq/schema'; import { runPull } from './db'; import { z } from 'zod'; @@ -206,13 +206,14 @@ function redactDatabaseUrl(url: string): string { } } -async function createDialect(provider: string, databaseUrl: string, outputPath: string) { +export async function createDialect(provider: string, databaseUrl: string, outputPath: string) { switch (provider) { case 'sqlite': { let SQLite: typeof BetterSqlite3; - try { - SQLite = (await import('better-sqlite3')).default; - } catch { + const mod = await loadPackage('better-sqlite3'); + if (mod) { + SQLite = mod.default || mod; + } else { throw new CliError( `Package "better-sqlite3" is required for SQLite support. Please install it with: npm install better-sqlite3`, ); @@ -231,9 +232,10 @@ async function createDialect(provider: string, databaseUrl: string, outputPath: } case 'postgresql': { let PgPool: typeof PgPoolType; - try { - PgPool = (await import('pg')).Pool; - } catch { + const mod = await loadPackage('pg'); + if (mod) { + PgPool = mod.Pool || mod.default?.Pool; + } else { throw new CliError( `Package "pg" is required for PostgreSQL support. Please install it with: npm install pg`, ); @@ -247,9 +249,10 @@ async function createDialect(provider: string, databaseUrl: string, outputPath: } case 'mysql': { let createMysqlPool: typeof MysqlCreatePool; - try { - createMysqlPool = (await import('mysql2')).createPool; - } catch { + const mod = await loadPackage('mysql2'); + if (mod) { + createMysqlPool = mod.createPool || mod.default?.createPool; + } else { throw new CliError( `Package "mysql2" is required for MySQL support. Please install it with: npm install mysql2`, ); @@ -502,19 +505,13 @@ export async function resolveSchema(options: Options): Promise { if (!options.introspect) { // Provide a helpful error depending on whether DATABASE_URL is set const hasEnvUrl = !!process.env['DATABASE_URL']; - if (hasEnvUrl) { - throw new CliError( - 'No schema file found. You can either:\n' + - ' • Specify the schema explicitly: zen studio --schema \n' + - ' • Auto-generate from the database: zen studio --introspect', - ); - } else { - throw new CliError( - 'No schema file found. You can either:\n' + - ' • Specify the schema explicitly: zen studio --schema -d \n' + - ' • Auto-generate from the database: zen studio --introspect -d ', - ); - } + const dbFlag = hasEnvUrl ? '' : '-d '; + + throw new CliError( + 'No ZModel schema file found in default locations.\n' + + ` • If you have an existing schema file, specify it explicitly:\n npx @zenstackhq/cli studio --schema ${dbFlag}\n` + + ` • Otherwise, auto-generate from the database:\n npx @zenstackhq/cli studio --introspect ${dbFlag}`, + ); } // Resolve database URL @@ -536,7 +533,7 @@ export async function resolveSchema(options: Options): Promise { const providerType = getProviderFromUrl(databaseUrl); // Install required packages - await installPackagesForIntrospect(providerType); + await installDbDriverPackage(providerType); // Introspect and generate schema const urlFromEnv = !options.databaseUrl; @@ -573,13 +570,7 @@ export function getProviderFromUrl(url: string): DataSourceProviderType { throw new CliError(`Unable to determine database type from URL: ${url}`); } -async function installPackagesForIntrospect(providerType: DataSourceProviderType) { - const corePackages = [ - { name: '@zenstackhq/cli@latest', dev: true }, - { name: '@zenstackhq/schema@latest', dev: false }, - { name: '@zenstackhq/orm@latest', dev: false }, - ]; - +export async function installDbDriverPackage(providerType: DataSourceProviderType) { // Add the appropriate DB driver const driverPackage: { name: string; dev: false } = (() => { switch (providerType) { @@ -592,7 +583,9 @@ async function installPackagesForIntrospect(providerType: DataSourceProviderType } })(); - const allPackages = [...corePackages, driverPackage]; + if (isPackageInstalled(driverPackage.name)) { + return; + } let pm = await detect(); if (!pm) { @@ -601,25 +594,25 @@ async function installPackagesForIntrospect(providerType: DataSourceProviderType console.log(colors.gray(`Using package manager: ${pm.agent}`)); - for (const pkg of allPackages) { - const resolved = resolveCommand(pm.agent, 'add', [ - pkg.name, - ...(pkg.dev ? [pm.agent.startsWith('yarn') || pm.agent === 'bun' ? '--dev' : '--save-dev'] : []), - ]); - if (!resolved) { - throw new CliError(`Unable to determine how to install package "${pkg.name}". Please install it manually.`); - } + const resolved = resolveCommand(pm.agent, 'add', [ + driverPackage.name, + ...(driverPackage.dev ? [pm.agent.startsWith('yarn') || pm.agent === 'bun' ? '--dev' : '--save-dev'] : []), + ]); + if (!resolved) { + throw new CliError( + `Unable to determine how to install package "${driverPackage.name}". Please install it manually.`, + ); + } - const spinner = ora(`Installing "${pkg.name}"`).start(); - try { - execSync(`${resolved.command} ${resolved.args.join(' ')}`, { - cwd: process.cwd(), - }); - spinner.succeed(); - } catch (e) { - spinner.fail(); - throw e; - } + const spinner = ora(`Installing "${driverPackage.name}"`).start(); + try { + execSync(`${resolved.command} ${resolved.args.join(' ')}`, { + cwd: process.cwd(), + }); + spinner.succeed(); + } catch (e) { + spinner.fail(); + throw e; } } diff --git a/packages/cli/src/actions/pull/provider/mysql.ts b/packages/cli/src/actions/pull/provider/mysql.ts index 1c8124435..8de1baa24 100644 --- a/packages/cli/src/actions/pull/provider/mysql.ts +++ b/packages/cli/src/actions/pull/provider/mysql.ts @@ -4,6 +4,7 @@ import { getAttributeRef, getDbName, getFunctionRef, normalizeDecimalDefault, no import type { IntrospectedEnum, IntrospectedSchema, IntrospectedTable, IntrospectionProvider } from './provider'; import { CliError } from '../../../cli-error'; import { resolveNameCasing } from '../casing'; +import { loadPackage } from '../../action-utils'; // Note: We dynamically import mysql2 inside the async function to avoid // requiring it at module load time for environments that don't use MySQL. @@ -131,7 +132,11 @@ export const mysql: IntrospectionProvider = { } }, async introspect(connectionString: string, options: { schemas: string[]; modelCasing: 'pascal' | 'camel' | 'snake' | 'none' }): Promise { - const mysql = await import('mysql2/promise'); + const mod = await loadPackage('mysql2/promise'); + if (!mod) { + throw new CliError('Package "mysql2" is required for MySQL support. Please install it with: npm install mysql2'); + } + const mysql = mod.createConnection ? mod : (mod.default ?? mod); const connection = await mysql.createConnection(connectionString); try { diff --git a/packages/cli/src/actions/pull/provider/postgresql.ts b/packages/cli/src/actions/pull/provider/postgresql.ts index 7cc470640..d3d52dff4 100644 --- a/packages/cli/src/actions/pull/provider/postgresql.ts +++ b/packages/cli/src/actions/pull/provider/postgresql.ts @@ -1,7 +1,9 @@ import type { ZModelServices } from '@zenstackhq/language'; import type { Attribute, BuiltinType, Enum, Expression } from '@zenstackhq/language/ast'; import { AstFactory, DataFieldAttributeFactory, ExpressionBuilder } from '@zenstackhq/language/factory'; +import type { Client as PgClient } from 'pg'; import { CliError } from '../../../cli-error'; +import { loadPackage } from '../../action-utils'; import { getAttributeRef, getDbName, getFunctionRef, normalizeDecimalDefault, normalizeFloatDefault } from '../utils'; import type { IntrospectedEnum, IntrospectedSchema, IntrospectedTable, IntrospectionProvider } from './provider'; @@ -179,8 +181,12 @@ export const postgresql: IntrospectionProvider = { connectionString: string, options: { schemas: string[]; modelCasing: 'pascal' | 'camel' | 'snake' | 'none' }, ): Promise { - const { Client } = await import('pg'); - const client = new Client({ connectionString }); + const pg = await loadPackage('pg'); + const ClientClass: typeof PgClient = pg?.Client || pg?.default?.Client; + if (!ClientClass) { + throw new CliError('Package "pg" is required for PostgreSQL support. Please install it with: npm install pg'); + } + const client = new ClientClass({ connectionString }); await client.connect(); try { diff --git a/packages/cli/src/actions/pull/provider/sqlite.ts b/packages/cli/src/actions/pull/provider/sqlite.ts index 435da9766..3e9cc0694 100644 --- a/packages/cli/src/actions/pull/provider/sqlite.ts +++ b/packages/cli/src/actions/pull/provider/sqlite.ts @@ -1,6 +1,9 @@ import { DataFieldAttributeFactory } from '@zenstackhq/language/factory'; +import { CliError } from '../../../cli-error'; +import { loadPackage } from '../../action-utils'; import { getAttributeRef, getDbName, getFunctionRef, normalizeDecimalDefault, normalizeFloatDefault } from '../utils'; import type { IntrospectedEnum, IntrospectedSchema, IntrospectedTable, IntrospectionProvider } from './provider'; +import path from 'path'; // Note: We dynamically import better-sqlite3 inside the async function to avoid // requiring it at module load time for environments that don't use SQLite. @@ -21,7 +24,11 @@ export const sqlite: IntrospectionProvider = { }, getBuiltinType(type) { // Strip parenthesized constraints (e.g., VARCHAR(255) → varchar, DECIMAL(10,2) → decimal) - const t = (type || '').toLowerCase().trim().replace(/\(.*\)$/, '').trim(); + const t = (type || '') + .toLowerCase() + .trim() + .replace(/\(.*\)$/, '') + .trim(); // SQLite has no array types const isArray = false; @@ -122,9 +129,26 @@ export const sqlite: IntrospectionProvider = { return undefined; }, - async introspect(connectionString: string, _options: { schemas: string[]; modelCasing: 'pascal' | 'camel' | 'snake' | 'none' }): Promise { - const SQLite = (await import('better-sqlite3')).default; - const db = new SQLite(connectionString, { readonly: true }); + async introspect( + connectionString: string, + _options: { schemas: string[]; modelCasing: 'pascal' | 'camel' | 'snake' | 'none' }, + ): Promise { + const mod = await loadPackage('better-sqlite3'); + const SQLite = mod?.default || mod; + if (!SQLite) { + throw new CliError( + 'Package "better-sqlite3" is required for SQLite support. Please install it with: npm install better-sqlite3', + ); + } + + let resolvedUrl = connectionString.trim(); + if (resolvedUrl.startsWith('file:')) { + const filePath = resolvedUrl.substring('file:'.length); + if (!path.isAbsolute(filePath)) { + resolvedUrl = path.join(process.cwd(), filePath); + } + } + const db = new SQLite(resolvedUrl, { readonly: true }); try { const all = (sql: string): T[] => { @@ -256,7 +280,9 @@ export const sqlite: IntrospectionProvider = { if (constraintName && columnList) { // Split the column list on commas and strip quotes/whitespace // to extract each individual column name. - const columns = columnList.split(',').map((col) => col.trim().replace(/^["'`]|["'`]$/g, '')); + const columns = columnList + .split(',') + .map((col) => col.trim().replace(/^["'`]|["'`]$/g, '')); for (const col of columns) { if (col) { fkConstraintNames.set(col, constraintName); @@ -358,7 +384,8 @@ export const sqlite: IntrospectionProvider = { } }, - getDefaultValue({ defaultValue, fieldType, services, enums }) { // datatype and datatype_name not used for SQLite + getDefaultValue({ defaultValue, fieldType, services, enums }) { + // datatype and datatype_name not used for SQLite const val = defaultValue.trim(); switch (fieldType) { @@ -401,7 +428,9 @@ export const sqlite: IntrospectionProvider = { return (ab) => ab.StringLiteral.setValue(val); } - console.warn(`Unsupported default value type: "${defaultValue}" for field type "${fieldType}". Skipping default value.`); + console.warn( + `Unsupported default value type: "${defaultValue}" for field type "${fieldType}". Skipping default value.`, + ); return null; }, @@ -409,7 +438,10 @@ export const sqlite: IntrospectionProvider = { const factories: DataFieldAttributeFactory[] = []; // Add @updatedAt for DateTime fields named updatedAt or updated_at - if (fieldType === 'DateTime' && (fieldName.toLowerCase() === 'updatedat' || fieldName.toLowerCase() === 'updated_at')) { + if ( + fieldType === 'DateTime' && + (fieldName.toLowerCase() === 'updatedat' || fieldName.toLowerCase() === 'updated_at') + ) { factories.push(new DataFieldAttributeFactory().setDecl(getAttributeRef('@updatedAt', services))); } diff --git a/packages/cli/test/action-utils.test.ts b/packages/cli/test/action-utils.test.ts index c8dd277ad..f2db32cc2 100644 --- a/packages/cli/test/action-utils.test.ts +++ b/packages/cli/test/action-utils.test.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; -import { generateTempPrismaSchema } from '../src/actions/action-utils'; +import { generateTempPrismaSchema, isPackageInstalled, loadPackage } from '../src/actions/action-utils'; import { createProject } from './utils'; const model = ` @@ -50,3 +50,15 @@ describe('generateTempPrismaSchema', () => { } }); }); + +describe('loadPackage and isPackageInstalled', () => { + it('detects installed packages', () => { + expect(isPackageInstalled('better-sqlite3')).toBe(true); + expect(isPackageInstalled('non-existent-pkg-999')).toBe(false); + }); + + it('loads installed package successfully', async () => { + const mod = await loadPackage('better-sqlite3'); + expect(mod).toBeDefined(); + }); +}); From 49ea72f1c96cad345556293b3d4df4fc27ce0fa0 Mon Sep 17 00:00:00 2001 From: jiasheng Date: Thu, 30 Jul 2026 07:52:21 +0800 Subject: [PATCH 4/8] fix(tests): update expected error messages in CLI proxy tests to include npx command --- packages/cli/test/proxy.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/test/proxy.test.ts b/packages/cli/test/proxy.test.ts index 5038735e7..7041a7672 100644 --- a/packages/cli/test/proxy.test.ts +++ b/packages/cli/test/proxy.test.ts @@ -843,7 +843,7 @@ describe('CLI proxy tests', () => { port: 3000, signatureToleranceSecs: 60, }), - ).rejects.toThrow('zen studio --introspect -d '); + ).rejects.toThrow('npx @zenstackhq/cli studio --introspect -d '); } finally { if (originalEnv !== undefined) process.env['DATABASE_URL'] = originalEnv; } @@ -859,7 +859,7 @@ describe('CLI proxy tests', () => { port: 3000, signatureToleranceSecs: 60, }), - ).rejects.toThrow('zen studio --introspect'); + ).rejects.toThrow('npx @zenstackhq/cli studio --introspect'); } finally { if (originalEnv !== undefined) { process.env['DATABASE_URL'] = originalEnv; From 844e92c577f90c14f4bbcb7c0f165a7efafb0e4e Mon Sep 17 00:00:00 2001 From: Jiasheng Date: Thu, 30 Jul 2026 07:54:14 +0800 Subject: [PATCH 5/8] Update packages/cli/src/actions/action-utils.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- packages/cli/src/actions/action-utils.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/actions/action-utils.ts b/packages/cli/src/actions/action-utils.ts index 972a5586b..eb8cc483f 100644 --- a/packages/cli/src/actions/action-utils.ts +++ b/packages/cli/src/actions/action-utils.ts @@ -396,18 +396,21 @@ export function isPackageInstalled(pkgName: string): boolean { export async function loadPackage(pkgName: string) { try { return await import(pkgName); - } catch { + } catch (importErr) { // If zenstack CLI is running directly using npx/pnpm in a temp folder, // dynamic package import will fail. Also check the current project folder. try { const projectRequire = createRequire(path.resolve(process.cwd(), 'package.json')); const resolvedPath = projectRequire.resolve(pkgName); return await import(pathToFileURL(resolvedPath).href); - } catch { + } catch (resolvedImportErr) { try { const projectRequire = createRequire(path.resolve(process.cwd(), 'package.json')); return projectRequire(pkgName); - } catch { + } catch (requireErr) { + if (process.env.DEBUG) { + console.error(importErr, resolvedImportErr, requireErr); + } return null; } } From 0a25e4bf9accf39ca0e4ccb463ceeb0560eca68a Mon Sep 17 00:00:00 2001 From: jiasheng Date: Thu, 30 Jul 2026 09:40:46 +0800 Subject: [PATCH 6/8] test: add SQLite database setup and schema introspection test --- packages/cli/src/actions/action-utils.ts | 3 +- packages/cli/src/actions/db.ts | 8 ++- packages/cli/src/actions/proxy.ts | 68 +++++++++++++------ .../cli/src/actions/pull/provider/sqlite.ts | 10 +-- packages/cli/test/proxy.test.ts | 38 +++++++++++ 5 files changed, 94 insertions(+), 33 deletions(-) diff --git a/packages/cli/src/actions/action-utils.ts b/packages/cli/src/actions/action-utils.ts index eb8cc483f..c9b0aefd7 100644 --- a/packages/cli/src/actions/action-utils.ts +++ b/packages/cli/src/actions/action-utils.ts @@ -408,7 +408,7 @@ export async function loadPackage(pkgName: string) { const projectRequire = createRequire(path.resolve(process.cwd(), 'package.json')); return projectRequire(pkgName); } catch (requireErr) { - if (process.env.DEBUG) { + if (process.env['DEBUG']) { console.error(importErr, resolvedImportErr, requireErr); } return null; @@ -416,4 +416,3 @@ export async function loadPackage(pkgName: string) { } } } - diff --git a/packages/cli/src/actions/db.ts b/packages/cli/src/actions/db.ts index 0bc813680..b2546de3d 100644 --- a/packages/cli/src/actions/db.ts +++ b/packages/cli/src/actions/db.ts @@ -135,9 +135,15 @@ export async function runPull(options: PullOptions) { }; if (options.provider && options.databaseUrl) { + let databaseUrl = options.databaseUrl; + // Handle SQLite file URLs for local files + if (options.provider === 'sqlite' && databaseUrl.startsWith('file:')) { + databaseUrl = new URL(databaseUrl, `file:${model.$document!.uri.path}`).pathname; + if (process.platform === 'win32' && databaseUrl[0] === '/') databaseUrl = databaseUrl.slice(1); + } datasource = { provider: options.provider, - url: options.databaseUrl, + url: databaseUrl, defaultSchema: 'public', allSchemas: ['public'], }; diff --git a/packages/cli/src/actions/proxy.ts b/packages/cli/src/actions/proxy.ts index ae46afa22..bc1ec1f03 100644 --- a/packages/cli/src/actions/proxy.ts +++ b/packages/cli/src/actions/proxy.ts @@ -220,10 +220,8 @@ export async function createDialect(provider: string, databaseUrl: string, outpu } let resolvedUrl = databaseUrl.trim(); if (resolvedUrl.startsWith('file:')) { - const filePath = resolvedUrl.substring('file:'.length); - if (!path.isAbsolute(filePath)) { - resolvedUrl = path.join(outputPath, filePath); - } + resolvedUrl = new URL(resolvedUrl, `file:${outputPath}`).pathname; + if (process.platform === 'win32' && resolvedUrl[0] === '/') resolvedUrl = resolvedUrl.slice(1); } console.log(colors.gray(`Connecting to SQLite database at: ${resolvedUrl}`)); return new SqliteDialect({ @@ -496,13 +494,12 @@ function startServer( // --------------------------------------------------------------------------- export async function resolveSchema(options: Options): Promise { - try { - return getSchemaFile(options.schema); - } catch (err) { - if (!(err instanceof CliError)) throw err; + if (!options.introspect) { + try { + return getSchemaFile(options.schema); + } catch (err) { + if (!(err instanceof CliError)) throw err; - // Schema not found – check whether we should introspect - if (!options.introspect) { // Provide a helpful error depending on whether DATABASE_URL is set const hasEnvUrl = !!process.env['DATABASE_URL']; const dbFlag = hasEnvUrl ? '' : '-d '; @@ -513,22 +510,13 @@ export async function resolveSchema(options: Options): Promise { ` • Otherwise, auto-generate from the database:\n npx @zenstackhq/cli studio --introspect ${dbFlag}`, ); } - + } else { // Resolve database URL const databaseUrl = options.databaseUrl ?? process.env['DATABASE_URL']; if (!databaseUrl) { throw new CliError('--introspect requires a database connection — pass -d or set DATABASE_URL.'); } - // If the default schema file already exists, skip introspection - const defaultSchemaPath = path.resolve('zenstack', 'schema.zmodel'); - if (fs.existsSync(defaultSchemaPath)) { - console.log( - colors.gray('Note: --introspect ignored because a schema file already exists at ' + defaultSchemaPath), - ); - return defaultSchemaPath; - } - // Determine DB provider type from URL const providerType = getProviderFromUrl(databaseUrl); @@ -616,13 +604,51 @@ export async function installDbDriverPackage(providerType: DataSourceProviderTyp } } +function adjustSqliteUrlForSchema(url: string): { isRelative: boolean; adjustedUrl: string } { + let prefix = ''; + let filePath = url; + + if (url.startsWith('file:')) { + prefix = 'file:'; + filePath = url.substring('file:'.length); + } else if (url.startsWith('sqlite:')) { + prefix = 'sqlite:'; + filePath = url.substring('sqlite:'.length); + } + + if (!path.isAbsolute(filePath)) { + const adjustedPath = path.join('..', filePath).replace(/\\/g, '/'); + return { + isRelative: true, + adjustedUrl: `${prefix}${adjustedPath}`, + }; + } + + return { + isRelative: false, + adjustedUrl: url, + }; +} + async function introspectAndGenerateSchema( databaseUrl: string, providerType: DataSourceProviderType, urlFromEnv: boolean, ): Promise { // Write a minimal datasource-only schema so runPull has something to load - const urlExpr = urlFromEnv ? 'env("DATABASE_URL")' : `'${databaseUrl}'`; + let urlExpr: string; + if (providerType === 'sqlite') { + const { isRelative, adjustedUrl } = adjustSqliteUrlForSchema(databaseUrl); + if (isRelative) { + urlExpr = `'${adjustedUrl}'`; + databaseUrl = adjustedUrl; // Update databaseUrl to the adjusted relative path for runPull + } else { + urlExpr = urlFromEnv ? 'env("DATABASE_URL")' : `'${databaseUrl}'`; + } + } else { + urlExpr = urlFromEnv ? 'env("DATABASE_URL")' : `'${databaseUrl}'`; + } + const minimalSchema = `datasource db {\n provider = '${providerType}'\n url = ${urlExpr}\n}\n`; const schemaDir = path.resolve('zenstack'); diff --git a/packages/cli/src/actions/pull/provider/sqlite.ts b/packages/cli/src/actions/pull/provider/sqlite.ts index 3e9cc0694..360929f94 100644 --- a/packages/cli/src/actions/pull/provider/sqlite.ts +++ b/packages/cli/src/actions/pull/provider/sqlite.ts @@ -3,7 +3,6 @@ import { CliError } from '../../../cli-error'; import { loadPackage } from '../../action-utils'; import { getAttributeRef, getDbName, getFunctionRef, normalizeDecimalDefault, normalizeFloatDefault } from '../utils'; import type { IntrospectedEnum, IntrospectedSchema, IntrospectedTable, IntrospectionProvider } from './provider'; -import path from 'path'; // Note: We dynamically import better-sqlite3 inside the async function to avoid // requiring it at module load time for environments that don't use SQLite. @@ -141,14 +140,7 @@ export const sqlite: IntrospectionProvider = { ); } - let resolvedUrl = connectionString.trim(); - if (resolvedUrl.startsWith('file:')) { - const filePath = resolvedUrl.substring('file:'.length); - if (!path.isAbsolute(filePath)) { - resolvedUrl = path.join(process.cwd(), filePath); - } - } - const db = new SQLite(resolvedUrl, { readonly: true }); + const db = new SQLite(connectionString, { readonly: true }); try { const all = (sql: string): T[] => { diff --git a/packages/cli/test/proxy.test.ts b/packages/cli/test/proxy.test.ts index 7041a7672..2c828cda9 100644 --- a/packages/cli/test/proxy.test.ts +++ b/packages/cli/test/proxy.test.ts @@ -898,6 +898,11 @@ describe('CLI proxy tests', () => { const schemaFile = path.join(schemaDir, 'schema.zmodel'); fs.writeFileSync(schemaFile, 'datasource db { provider = "sqlite" url = "file:./test.db" }'); + const SQLite = (await import('better-sqlite3')).default; + const db = new SQLite(path.join(tmpDir, 'test.db')); + db.exec('CREATE TABLE test (id INTEGER PRIMARY KEY);'); + db.close(); + const originalCwd = process.cwd(); process.chdir(tmpDir); try { @@ -915,6 +920,39 @@ describe('CLI proxy tests', () => { fs.rmSync(tmpDir, { recursive: true, force: true }); } }); + + it('should adjust relative sqlite database url when introspecting schema', async () => { + const fs = await import('node:fs'); + const path = await import('node:path'); + const os = await import('node:os'); + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zenstack-test-introspect-')); + const dbFile = path.join(tmpDir, 'test.db'); + + // Create dummy sqlite db + const SQLite = (await import('better-sqlite3')).default; + const db = new SQLite(dbFile); + db.exec('CREATE TABLE test (id INTEGER PRIMARY KEY);'); + db.close(); + + const originalCwd = process.cwd(); + process.chdir(tmpDir); + try { + const result = await resolveSchema({ + schema: '/nonexistent/path/schema.zmodel', + port: 3000, + signatureToleranceSecs: 60, + introspect: true, + databaseUrl: 'file:./test.db', + }); + expect(result).toBe(path.resolve('zenstack', 'schema.zmodel')); + const content = fs.readFileSync(result, 'utf-8'); + expect(content).toContain("url = 'file:../test.db'"); + } finally { + process.chdir(originalCwd); + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); }); }); }); From e7c4a29d3d2a398e4c732056133c7042d1e56cf1 Mon Sep 17 00:00:00 2001 From: jiasheng Date: Thu, 30 Jul 2026 21:48:43 +0800 Subject: [PATCH 7/8] fix(sqlite): adjust nullable property to account for PRIMARY KEY columns --- packages/cli/src/actions/pull/provider/sqlite.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/actions/pull/provider/sqlite.ts b/packages/cli/src/actions/pull/provider/sqlite.ts index 360929f94..1590c0214 100644 --- a/packages/cli/src/actions/pull/provider/sqlite.ts +++ b/packages/cli/src/actions/pull/provider/sqlite.ts @@ -358,7 +358,7 @@ export const sqlite: IntrospectionProvider = { foreign_key_on_delete: fk?.foreign_key_on_delete ?? null, pk: !!c.pk, computed: isGenerated, - nullable: c.notnull !== 1, + nullable: c.notnull !== 1 && !c.pk, // SQLite treats PRIMARY KEY columns as NOT NULL default: defaultValue, unique: uniqueSingleColumn.has(c.name), unique_name: null, From 8d749066fc055a9f5cdbcd96ab5b8dabefff4751 Mon Sep 17 00:00:00 2001 From: jiasheng Date: Thu, 30 Jul 2026 22:12:38 +0800 Subject: [PATCH 8/8] fix(sqlite): update nullable property handling for PRIMARY KEY columns --- packages/cli/src/actions/pull/provider/sqlite.ts | 2 +- packages/cli/test/proxy.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/actions/pull/provider/sqlite.ts b/packages/cli/src/actions/pull/provider/sqlite.ts index 1590c0214..360929f94 100644 --- a/packages/cli/src/actions/pull/provider/sqlite.ts +++ b/packages/cli/src/actions/pull/provider/sqlite.ts @@ -358,7 +358,7 @@ export const sqlite: IntrospectionProvider = { foreign_key_on_delete: fk?.foreign_key_on_delete ?? null, pk: !!c.pk, computed: isGenerated, - nullable: c.notnull !== 1 && !c.pk, // SQLite treats PRIMARY KEY columns as NOT NULL + nullable: c.notnull !== 1, default: defaultValue, unique: uniqueSingleColumn.has(c.name), unique_name: null, diff --git a/packages/cli/test/proxy.test.ts b/packages/cli/test/proxy.test.ts index 2c828cda9..30509be01 100644 --- a/packages/cli/test/proxy.test.ts +++ b/packages/cli/test/proxy.test.ts @@ -900,7 +900,7 @@ describe('CLI proxy tests', () => { const SQLite = (await import('better-sqlite3')).default; const db = new SQLite(path.join(tmpDir, 'test.db')); - db.exec('CREATE TABLE test (id INTEGER PRIMARY KEY);'); + db.exec('CREATE TABLE test (id INTEGER NOT NULL PRIMARY KEY);'); db.close(); const originalCwd = process.cwd(); @@ -932,7 +932,7 @@ describe('CLI proxy tests', () => { // Create dummy sqlite db const SQLite = (await import('better-sqlite3')).default; const db = new SQLite(dbFile); - db.exec('CREATE TABLE test (id INTEGER PRIMARY KEY);'); + db.exec('CREATE TABLE test (id INTEGER NOT NULL PRIMARY KEY);'); db.close(); const originalCwd = process.cwd();