feat: add npm workspace indexing support

.prettierignore

@@ -2,3 +2,7 @@ dist
 node_modules
 src/scip.ts
 snapshots/output
+# Snapshot fixtures intentionally have no trailing newline so the snapshot
+# formatter (which mirrors input line count) does not emit a trailing blank
+# line at EOF in snapshots/output, which would trip `git diff --check`.
+snapshots/input/npm-workspaces/packages/*/src/*.ts

README.md

@@ -53,6 +53,16 @@ pnpm install
 scip-typescript index --pnpm-workspaces
 ```
 
+### Index a TypeScript project using npm workspaces
+
+Navigate to the project root, containing `package.json` with a `workspaces` field.
+
+```sh
+npm install
+
+scip-typescript index --npm-workspaces
+```
+
 ### Indexing in CI
 
 Add the following run steps to your CI pipeline:

snapshots/input/npm-workspaces/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "npm-workspaces",
+  "version": "1.0.0",
+  "description": "Example TS/JS project",
+  "main": "src/main.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "private": true,
+  "packageManager": "npm@10.0.0",
+  "workspaces": [
+    "packages/*"
+  ]
+}

snapshots/input/npm-workspaces/packages/a/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@example/a",
+  "version": "1.0.0",
+  "description": "Example TS/JS project",
+  "main": "src/a.ts",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC"
+}

snapshots/input/npm-workspaces/packages/a/src/a.ts

@@ -0,0 +1,3 @@
+export function a(): string {
+  return ''
+}

snapshots/input/npm-workspaces/packages/a/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "baseUrl": ".",
+    "outDir": "dist"
+  },
+  "include": ["src/*"]
+}

snapshots/input/npm-workspaces/packages/b/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@example/b",
+  "version": "1.0.0",
+  "description": "Example TS/JS project",
+  "main": "src/b.ts",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@example/a": "1.0.0"
+  }
+}

snapshots/input/npm-workspaces/packages/b/src/b.ts

@@ -0,0 +1,5 @@
+import { a } from '@example/a'
+
+export function b() {
+  return a()
+}

snapshots/input/npm-workspaces/packages/b/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "baseUrl": "./src",
+    "sourceRoot": "src",
+    "outDir": "dist",
+    "paths": {
+      "@example/a": ["../../a/src/a.ts"]
+    }
+  },
+  "include": ["src/*"],
+  "references": [{ "path": "../a" }]
+}

snapshots/input/npm-workspaces/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "extends": "@sourcegraph/tsconfig",
+  "compilerOptions": {
+    "target": "es2020",
+    "module": "commonjs",
+    "allowJs": false,
+    "moduleResolution": "node",
+    "esModuleInterop": true,
+    "lib": ["esnext", "dom", "dom.iterable"],
+    "sourceMap": true,
+    "declaration": true,
+    "declarationMap": true,
+    "skipLibCheck": true,
+    "skipDefaultLibCheck": true,
+    "noErrorTruncation": true,
+    "importHelpers": true,
+    "resolveJsonModule": true,
+    "composite": true,
+    "outDir": "out",
+    "rootDir": "."
+  },
+  "include": [],
+  "exclude": ["out", "node_modules", "dist"]
+}

snapshots/output/npm-workspaces/packages/a/src/a.ts

@@ -0,0 +1,6 @@
+// < definition @example/a 1.0.0 src/`a.ts`/
+
+export function a(): string {
+//              ^ definition @example/a 1.0.0 src/`a.ts`/a().
+  return ''
+}

snapshots/output/npm-workspaces/packages/b/src/b.ts

@@ -0,0 +1,11 @@
+// < definition @example/b 1.0.0 src/`b.ts`/
+
+import { a } from '@example/a'
+//       ^ reference @example/a 1.0.0 src/`a.ts`/a().
+//                ^^^^^^^^^^^^ reference @example/a 1.0.0 src/`a.ts`/
+
+export function b() {
+//              ^ definition @example/b 1.0.0 src/`b.ts`/b().
+  return a()
+//       ^ reference @example/a 1.0.0 src/`a.ts`/a().
+}

src/CommandLineOptions.test.ts

@@ -33,6 +33,7 @@ checkIndexParser([], {
 checkIndexParser(['--cwd', 'qux'], { cwd: 'qux' })
 checkIndexParser(['--yarn-workspaces'], { yarnWorkspaces: true })
 checkIndexParser(['--pnpm-workspaces'], { pnpmWorkspaces: true })
+checkIndexParser(['--npm-workspaces'], { npmWorkspaces: true })
 checkIndexParser(['--infer-tsconfig'], { inferTsconfig: true })
 checkIndexParser(['--no-progress-bar'], { progressBar: false })
 checkIndexParser(['--progress-bar'], { progressBar: true })

src/CommandLineOptions.ts

@@ -13,6 +13,7 @@ export interface MultiProjectOptions {
   yarnWorkspaces: boolean
   yarnBerryWorkspaces: boolean
   pnpmWorkspaces: boolean
+  npmWorkspaces: boolean
   globalCaches: boolean
   maxFileByteSize?: string
   maxFileByteSizeNumber?: number
@@ -51,6 +52,7 @@ export function mainCommand(
     .command('index')
     .option('--cwd <path>', 'the working directory', process.cwd())
     .option('--pnpm-workspaces', 'whether to index all pnpm workspaces', false)
+    .option('--npm-workspaces', 'whether to index all npm workspaces', false)
     .option('--yarn-workspaces', 'whether to index all yarn workspaces', false)
     .option(
       '--yarn-berry-workspaces',

src/listNpmWorkspaces.test.ts

@@ -0,0 +1,74 @@
+import * as fs from 'fs'
+import * as os from 'os'
+import * as path from 'path'
+
+import { test } from 'uvu'
+import * as assert from 'uvu/assert'
+
+import { listNpmWorkspaces } from './main'
+
+function makeWorkspace(
+  workspaces: unknown,
+  packages: string[]
+): { dir: string; cleanup: () => void } {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'scip-npm-ws-'))
+  fs.writeFileSync(
+    path.join(dir, 'package.json'),
+    JSON.stringify({ name: 'root', private: true, workspaces })
+  )
+  for (const p of packages) {
+    const full = path.join(dir, p)
+    fs.mkdirSync(full, { recursive: true })
+    fs.writeFileSync(
+      path.join(full, 'package.json'),
+      JSON.stringify({ name: `@example/${path.basename(p)}` })
+    )
+  }
+  return {
+    dir,
+    cleanup: () => fs.rmSync(dir, { recursive: true, force: true }),
+  }
+}
+
+test('npm workspaces: array form', () => {
+  const { dir, cleanup } = makeWorkspace(
+    ['packages/*'],
+    ['packages/a', 'packages/b']
+  )
+  try {
+    const got = listNpmWorkspaces(dir)
+      .map(p => path.relative(dir, p))
+      .sort()
+    assert.equal(got, [path.join('packages', 'a'), path.join('packages', 'b')])
+  } finally {
+    cleanup()
+  }
+})
+
+test('npm workspaces: object form { packages: [...] }', () => {
+  const { dir, cleanup } = makeWorkspace({ packages: ['packages/*'] }, [
+    'packages/a',
+    'packages/b',
+  ])
+  try {
+    const got = listNpmWorkspaces(dir)
+      .map(p => path.relative(dir, p))
+      .sort()
+    assert.equal(got, [path.join('packages', 'a'), path.join('packages', 'b')])
+  } finally {
+    cleanup()
+  }
+})
+
+test('npm workspaces: object form without packages key returns nothing', () => {
+  const { dir, cleanup } = makeWorkspace({ nohmm: ['packages/*'] }, [
+    'packages/a',
+  ])
+  try {
+    assert.equal(listNpmWorkspaces(dir), [])
+  } finally {
+    cleanup()
+  }
+})
+
+test.run()

src/main.test.ts

@@ -51,13 +51,17 @@ for (const snapshotDirectory of snapshotDirectories) {
     const tsconfigJsonPath = path.join(inputRoot, 'tsconfig.json')
     const inferTsconfig = !fs.existsSync(tsconfigJsonPath)
     const output = path.join(inputRoot, 'index.scip')
+    const isPnpm = Boolean(packageJson.packageManager?.includes('pnpm'))
+    const isNpm =
+      !isPnpm && Boolean(packageJson.packageManager?.startsWith('npm@'))
     indexCommand([], {
       cwd: inputRoot,
       inferTsconfig,
       output,
-      yarnWorkspaces: Boolean(packageJson.workspaces),
+      yarnWorkspaces: !isNpm && !isPnpm && Boolean(packageJson.workspaces),
       yarnBerryWorkspaces: false,
-      pnpmWorkspaces: Boolean(packageJson.packageManager?.includes('pnpm')),
+      pnpmWorkspaces: isPnpm,
+      npmWorkspaces: isNpm,
       progressBar: false,
       indexedProjects: new Set(),
       globalCaches: true,
@@ -73,6 +77,22 @@ for (const snapshotDirectory of snapshotDirectories) {
     if (index.documents.length === 0) {
       throw new Error('empty LSIF index')
     }
+    // Normalize backslashes so a Windows-emitted `packages\a\src\a.ts`
+    // and a forward-slash `packages/a/src/a.ts` are recognized as the
+    // same document (this was the npm-workspaces dedup bug).
+    const documentPaths = index.documents.map(d =>
+      d.relative_path.replaceAll('\\', '/')
+    )
+    const duplicatePaths = documentPaths.filter(
+      (p, i) => documentPaths.indexOf(p) !== i
+    )
+    if (duplicatePaths.length > 0) {
+      throw new Error(
+        `duplicate documents in index: ${JSON.stringify([
+          ...new Set(duplicatePaths),
+        ])}`
+      )
+    }
     for (const document of index.documents) {
       const inputPath = path.join(inputRoot, document.relative_path)
       const relativeToInputDirectory = path.relative(inputDirectory, inputPath)

src/main.ts

@@ -36,6 +36,8 @@ export function indexCommand(
     projects.push(...listYarnWorkspaces(options.cwd, 'yarn2Plus'))
   } else if (options.pnpmWorkspaces) {
     projects.push(...listPnpmWorkspaces(options.cwd))
+  } else if (options.npmWorkspaces) {
+    projects.push(...listNpmWorkspaces(options.cwd))
   } else if (projects.length === 0) {
     projects.push(options.cwd)
   }
@@ -107,11 +109,16 @@ function makeAbsolutePath(cwd: string, relativeOrAbsolutePath: string): string {
 }
 
 function indexSingleProject(options: ProjectOptions, cache: GlobalCache): void {
-  if (options.indexedProjects.has(options.projectRoot)) {
+  // Normalize the project root before deduping: workspace listings may use
+  // OS-native separators (e.g. `packages\a` on Windows) while TypeScript
+  // resolves projectReferences with forward slashes (`packages/a`), so a
+  // raw Set lookup would miss and re-index the same project.
+  const normalizedRoot = path.resolve(options.projectRoot).replaceAll('\\', '/')
+  if (options.indexedProjects.has(normalizedRoot)) {
     return
   }
 
-  options.indexedProjects.add(options.projectRoot)
+  options.indexedProjects.add(normalizedRoot)
   let config = ts.parseCommandLine(
     ['-p', options.projectRoot],
     (relativePath: string) => path.resolve(options.projectRoot, relativePath)
@@ -217,6 +224,63 @@ function defaultCompilerOptions(configFileName?: string): ts.CompilerOptions {
   return options
 }
 
+export function listNpmWorkspaces(directory: string): string[] {
+  // npm workspaces are declared in the root package.json. The `workspaces`
+  // field is either an array of patterns (`["packages/*"]`) or the object
+  // form `{ "packages": ["packages/*"] }`. Patterns commonly use a single
+  // `*` segment; we expand those directly to avoid pulling in a glob dep.
+  // ponytail: single-* glob covers the common case; if users need `**` or
+  // brace expansion, swap in a glob library later.
+  const packageJsonPath = path.join(directory, 'package.json')
+  const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8')) as {
+    workspaces?: unknown
+  }
+  const raw = pkg.workspaces
+  let patterns: string[] = []
+  if (Array.isArray(raw)) {
+    patterns = raw as string[]
+  } else if (
+    raw &&
+    typeof raw === 'object' &&
+    Array.isArray((raw as { packages?: unknown }).packages)
+  ) {
+    patterns = (raw as { packages: string[] }).packages
+  }
+  const result: string[] = []
+  for (const pattern of patterns) {
+    for (const candidate of expandNpmWorkspacePattern(directory, pattern)) {
+      if (fs.existsSync(path.join(candidate, 'package.json'))) {
+        result.push(candidate)
+      }
+    }
+  }
+  return result
+}
+
+function expandNpmWorkspacePattern(root: string, pattern: string): string[] {
+  const segments = pattern.split(/[/\\]/).filter(s => s.length > 0)
+  let dirs: string[] = [root]
+  for (const segment of segments) {
+    const next: string[] = []
+    for (const dir of dirs) {
+      if (segment === '*') {
+        if (!fs.existsSync(dir)) {
+          continue
+        }
+        for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+          if (entry.isDirectory() && entry.name !== 'node_modules') {
+            next.push(path.join(dir, entry.name))
+          }
+        }
+      } else {
+        next.push(path.join(dir, segment))
+      }
+    }
+    dirs = next
+  }
+  return dirs
+}
+
 function listPnpmWorkspaces(directory: string): string[] {
   /**
    * Returns the list of projects formatted as: